tor-commits August 2014

tor-commits@lists.torproject.org

21 participants
1599 discussions

[metrics-web/master] Add permanent redirects for unused pages.
by karsten＠torproject.org 14 Aug '14

14 Aug '14

commit 517af625d899d8a3cf86443032e9a3c1f307ccb3 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Aug 14 09:40:38 2014 +0200 Add permanent redirects for unused pages. --- .../org/torproject/metrics/web/GraphsServlet.java | 11 +- .../torproject/metrics/web/ResearchServlet.java | 11 +- .../metrics/web/research/ResearchDataServlet.java | 253 +---- .../web/research/ResearchFormatsServlet.java | 12 +- .../web/research/ResearchPapersServlet.java | 12 +- .../metrics/web/research/ResearchToolsServlet.java | 12 +- .../metrics/web/status/ConsensusHealthServlet.java | 9 +- .../metrics/web/status/ExoneraTorServlet.java | 10 +- .../metrics/web/status/RelaySearchServlet.java | 11 +- website/web/WEB-INF/data.jsp | 303 ------ website/web/WEB-INF/formats.jsp | 992 -------------------- website/web/WEB-INF/graphs.jsp | 34 - website/web/WEB-INF/papers.jsp | 81 -- website/web/WEB-INF/relay-search.jsp | 28 - website/web/WEB-INF/research.jsp | 32 - website/web/WEB-INF/status.jsp | 28 - website/web/WEB-INF/tools.jsp | 116 --- website/web/images/tor-metrics-overview.png | Bin 309306 -> 0 bytes 18 files changed, 39 insertions(+), 1916 deletions(-) diff --git a/website/src/org/torproject/metrics/web/GraphsServlet.java b/website/src/org/torproject/metrics/web/GraphsServlet.java index eb5eaf4..228a14a 100644 --- a/website/src/org/torproject/metrics/web/GraphsServlet.java +++ b/website/src/org/torproject/metrics/web/GraphsServlet.java @@ -1,4 +1,4 @@ -/* Copyright 2011, 2012 The Tor Project +/* Copyright 2011, 2012, 2014 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.web; @@ -9,16 +9,13 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +@SuppressWarnings("serial") public class GraphsServlet extends HttpServlet { - private static final long serialVersionUID = 7615715032362498151L; - public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { - - /* Forward the request to the JSP that does all the hard work. */ - request.getRequestDispatcher("WEB-INF/graphs.jsp").forward(request, - response); + response.setStatus(HttpServletResponse.SC_MOVED_PERMANENTLY); + response.setHeader("Location", "https://metrics.torproject.org/"); } } diff --git a/website/src/org/torproject/metrics/web/ResearchServlet.java b/website/src/org/torproject/metrics/web/ResearchServlet.java index f69c45d..020d9c5 100644 --- a/website/src/org/torproject/metrics/web/ResearchServlet.java +++ b/website/src/org/torproject/metrics/web/ResearchServlet.java @@ -1,4 +1,4 @@ -/* Copyright 2011, 2012 The Tor Project +/* Copyright 2011--2014 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.web; @@ -9,16 +9,13 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +@SuppressWarnings("serial") public class ResearchServlet extends HttpServlet { - private static final long serialVersionUID = -9151727188925700665L; - public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { - - /* Forward the request to the JSP that does all the hard work. */ - request.getRequestDispatcher("WEB-INF/research.jsp").forward(request, - response); + response.setStatus(HttpServletResponse.SC_MOVED_PERMANENTLY); + response.setHeader("Location", "https://research.torproject.org/"); } } diff --git a/website/src/org/torproject/metrics/web/research/ResearchDataServlet.java b/website/src/org/torproject/metrics/web/research/ResearchDataServlet.java index 9006ebf..578d052 100644 --- a/website/src/org/torproject/metrics/web/research/ResearchDataServlet.java +++ b/website/src/org/torproject/metrics/web/research/ResearchDataServlet.java @@ -1,266 +1,21 @@ -/* Copyright 2011, 2012 The Tor Project +/* Copyright 2011--2014 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.web.research; -import java.io.File; import java.io.IOException; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.SortedMap; -import java.util.TreeMap; import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -/** - * Controller servlet for the Data page. Prepares the various lists of - * downloadable metrics data files by parsing a file with URLs on other - * servers and looking at a local directory with files served by local - * Apache HTTP server. The file with URLs on other servers may contain - * comment lines starting with #. Recognizes metrics data file types from - * the file names. - */ +@SuppressWarnings("serial") public class ResearchDataServlet extends HttpServlet { - private static final long serialVersionUID = -5168280373350515577L; - public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { - - /* Read local directory with files served by the local Apache HTTP - * server and add the URLs to the list. */ - List<String> dataFileUrls = new ArrayList<String>(); - String localDataDir = getServletConfig().getInitParameter( - "localDataDir"); - if (localDataDir != null) { - try { - File localDataDirFile = new File(localDataDir); - if (localDataDirFile.exists() && localDataDirFile.isDirectory()) { - for (File localDataFile : localDataDirFile.listFiles()) { - if (!localDataFile.isDirectory()) { - dataFileUrls.add("/data/" + localDataFile.getName()); - } - } - } - } catch (SecurityException e) { - /* We're not permitted to read the directory with metrics data - * files. Ignore. */ - } - } - - /* Prepare data structures that we're going to pass to the JSP. All - * data structures are (nested) maps with the map keys being used for - * displaying the files in tables and map values being 2-element - * arrays containing the file url and optional signature file. */ - SortedMap<Date, Map<String, String[]>> relayDescriptors = - new TreeMap<Date, Map<String, String[]>>( - java.util.Collections.reverseOrder()); - String[] certs = new String[2]; - SortedMap<Date, String[]> bridgeDescriptors = - new TreeMap<Date, String[]>(java.util.Collections.reverseOrder()); - String[] relayStatistics = new String[2]; - SortedMap<Date, String[]> torperfTarballs = - new TreeMap<Date, String[]>(java.util.Collections.reverseOrder()); - SortedMap<String, Map<String, String[]>> torperfData = - new TreeMap<String, Map<String, String[]>>(); - SortedMap<Date, String[]> exitLists = - new TreeMap<Date, String[]>(java.util.Collections.reverseOrder()); - SortedMap<Date, String[]> torperfExperiments = - new TreeMap<Date, String[]>(); - SortedMap<Date, String[]> bridgePoolAssignments = - new TreeMap<Date, String[]>(java.util.Collections.reverseOrder()); - - /* Prepare rewriting Torperf sources. */ - Map<String, String> torperfSources = new HashMap<String, String>(); - torperfSources.put("torperffast", "torperf, fastest"); - torperfSources.put("torperffastratio", "torperf, best ratio"); - torperfSources.put("torperfslow", "torperf, slowest"); - torperfSources.put("torperfslowratio", "torperf, worst ratio"); - - /* Go through the file list, decide for each file what metrics data - * type it is, and put it in the appropriate map. */ - SimpleDateFormat monthFormat = new SimpleDateFormat("yyyy-MM"); - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - List<String> torperfFilesizes = Arrays.asList("50kb,1mb,5mb". - split(",")); - for (String url : dataFileUrls) { - if (!url.contains("/")) { - continue; - } - String filename = url.substring(url.lastIndexOf("/") + 1); - /* TODO Remove the following once we stop providing .tar.bz2 - * files. */ - if (filename.endsWith(".tar.bz2")) { - continue; - } - - /* URL contains relay descriptors. */ - if (filename.startsWith("tor-20") || - filename.startsWith("statuses-20") || - filename.startsWith("server-descriptors-20") || - filename.startsWith("extra-infos-20") || - filename.startsWith("votes-20") || - filename.startsWith("consensuses-20") || - filename.startsWith("microdescs-20")) { - String type = filename.substring(0, filename.indexOf("-20")); - String yearMonth = filename.substring(filename.indexOf("20")); - yearMonth = yearMonth.substring(0, 7); - Date month = null; - try { - month = monthFormat.parse(yearMonth); - } catch (ParseException e) { - /* Ignore this URL. */ - continue; - } - int index = filename.endsWith(".asc") ? 1 : 0; - if (!relayDescriptors.containsKey(month)) { - relayDescriptors.put(month, new HashMap<String, String[]>()); - } - if (!relayDescriptors.get(month).containsKey(type)) { - relayDescriptors.get(month).put(type, new String[2]); - } - relayDescriptors.get(month).get(type)[index] = url; - - /* URL contains v3 certificates. */ - } else if (filename.startsWith("certs.tar")) { - int index = filename.endsWith(".asc") ? 1 : 0; - certs[index] = url; - - /* URL contains bridge descriptors. */ - } else if (filename.startsWith("bridge-descriptors-20")) { - String yearMonth = filename.substring(filename.indexOf("20")); - yearMonth = yearMonth.substring(0, 7); - Date month = null; - try { - month = monthFormat.parse(yearMonth); - } catch (ParseException e) { - /* Ignore this URL. */ - continue; - } - int index = filename.endsWith(".asc") ? 1 : 0; - if (!bridgeDescriptors.containsKey(month)) { - bridgeDescriptors.put(month, new String[2]); - } - bridgeDescriptors.get(month)[index] = url; - - /* URL contains relay statistics. */ - } else if (filename.startsWith("relay-statistics.tar")) { - int index = filename.endsWith(".asc") ? 1 : 0; - relayStatistics[index] = url; - - /* URL contains Torperf tarball. */ - } else if (filename.startsWith("torperf-20")) { - String yearMonth = filename.substring(filename.indexOf("20")); - yearMonth = yearMonth.substring(0, 7); - Date month = null; - try { - month = monthFormat.parse(yearMonth); - } catch (ParseException e) { - /* Ignore this URL. */ - continue; - } - if (!torperfTarballs.containsKey(month)) { - torperfTarballs.put(month, new String[2]); - } - torperfTarballs.get(month)[0] = url; - - /* URL contains Torperf data file. */ - } else if (filename.endsWith("b.data") || - filename.endsWith("b.extradata")) { - boolean isExtraData = filename.endsWith("b.extradata"); - String[] parts = filename.split("-"); - if (parts.length != 2) { - continue; - } - String source = parts[0]; - if (torperfSources.containsKey(source)) { - source = torperfSources.get(source); - } - String filesize = parts[1]; - filesize = filesize.substring(0, filesize.length() - - (isExtraData ? 10 : 5)); - if (!torperfFilesizes.contains(filesize)) { - continue; - } - if (!torperfData.containsKey(source)) { - torperfData.put(source, new HashMap<String, String[]>()); - } - if (!torperfData.get(source).containsKey(filesize)) { - torperfData.get(source).put(filesize, new String[2]); - } - torperfData.get(source).get(filesize)[isExtraData ? 1 : 0] = url; - - /* URL contains Torperf experiment tarball. */ - } else if (filename.startsWith("torperf-experiment-20")) { - String dateString = filename.substring(filename.indexOf("20")); - dateString = dateString.substring(0, 10); - Date date = null; - try { - date = dateFormat.parse(dateString); - } catch (ParseException e) { - /* Ignore this URL. */ - continue; - } - if (!torperfExperiments.containsKey(date)) { - torperfExperiments.put(date, new String[2]); - } - torperfExperiments.get(date)[0] = url; - - /* URL contains exit list. */ - } else if (filename.startsWith("exit-list-20")) { - String yearMonth = filename.substring(filename.indexOf("20")); - yearMonth = yearMonth.substring(0, 7); - Date month = null; - try { - month = monthFormat.parse(yearMonth); - } catch (ParseException e) { - /* Ignore this URL. */ - continue; - } - if (!exitLists.containsKey(month)) { - exitLists.put(month, new String[2]); - } - exitLists.get(month)[0] = url; - - /* URL contains bridge pool assignments. */ - } else if (filename.startsWith("bridge-pool-assignments-20")) { - String yearMonth = filename.substring(filename.indexOf("20")); - yearMonth = yearMonth.substring(0, 7); - Date month = null; - try { - month = monthFormat.parse(yearMonth); - } catch (ParseException e) { - /* Ignore this URL. */ - continue; - } - if (!bridgePoolAssignments.containsKey(month)) { - bridgePoolAssignments.put(month, new String[2]); - } - bridgePoolAssignments.get(month)[0] = url; - } - } - - /* Add the maps to the request and forward it to the JSP to display - * the page. */ - request.setAttribute("relayDescriptors", relayDescriptors); - request.setAttribute("certs", certs); - request.setAttribute("bridgeDescriptors", bridgeDescriptors); - request.setAttribute("relayStatistics", relayStatistics); - request.setAttribute("torperfData", torperfData); - request.setAttribute("exitLists", exitLists); - request.setAttribute("torperfTarballs", torperfTarballs); - request.setAttribute("torperfExperiments", torperfExperiments); - request.setAttribute("bridgePoolAssignments", bridgePoolAssignments); - request.getRequestDispatcher("WEB-INF/data.jsp").forward(request, - response); + response.setStatus(HttpServletResponse.SC_MOVED_PERMANENTLY); + response.setHeader("Location", "https://collector.torproject.org/"); } } diff --git a/website/src/org/torproject/metrics/web/research/ResearchFormatsServlet.java b/website/src/org/torproject/metrics/web/research/ResearchFormatsServlet.java index 447ad44..95ce1bb 100644 --- a/website/src/org/torproject/metrics/web/research/ResearchFormatsServlet.java +++ b/website/src/org/torproject/metrics/web/research/ResearchFormatsServlet.java @@ -1,4 +1,4 @@ -/* Copyright 2011, 2012 The Tor Project +/* Copyright 2011--2014 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.web.research; @@ -9,16 +9,14 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +@SuppressWarnings("serial") public class ResearchFormatsServlet extends HttpServlet { - private static final long serialVersionUID = 5666493868675314116L; - public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { - - /* Forward the request to the JSP that does all the hard work. */ - request.getRequestDispatcher("WEB-INF/formats.jsp").forward(request, - response); + response.setStatus(HttpServletResponse.SC_MOVED_PERMANENTLY); + response.setHeader("Location", + "https://collector.torproject.org/formats.html"); } } diff --git a/website/src/org/torproject/metrics/web/research/ResearchPapersServlet.java b/website/src/org/torproject/metrics/web/research/ResearchPapersServlet.java index 88dc6fe..2aaa14a 100644 --- a/website/src/org/torproject/metrics/web/research/ResearchPapersServlet.java +++ b/website/src/org/torproject/metrics/web/research/ResearchPapersServlet.java @@ -1,4 +1,4 @@ -/* Copyright 2011, 2012 The Tor Project +/* Copyright 2011--2014 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.web.research; @@ -9,16 +9,14 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +@SuppressWarnings("serial") public class ResearchPapersServlet extends HttpServlet { - private static final long serialVersionUID = -8135459207158536268L; - public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { - - /* Forward the request to the JSP that does all the hard work. */ - request.getRequestDispatcher("WEB-INF/papers.jsp").forward(request, - response); + response.setStatus(HttpServletResponse.SC_MOVED_PERMANENTLY); + response.setHeader("Location", + "https://research.torproject.org/techreports.html"); } } diff --git a/website/src/org/torproject/metrics/web/research/ResearchToolsServlet.java b/website/src/org/torproject/metrics/web/research/ResearchToolsServlet.java index a7de0f4..45346c5 100644 --- a/website/src/org/torproject/metrics/web/research/ResearchToolsServlet.java +++ b/website/src/org/torproject/metrics/web/research/ResearchToolsServlet.java @@ -1,4 +1,4 @@ -/* Copyright 2011, 2012 The Tor Project +/* Copyright 2011--2014 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.web.research; @@ -9,16 +9,14 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +@SuppressWarnings("serial") public class ResearchToolsServlet extends HttpServlet { - private static final long serialVersionUID = -3344204426180358872L; - public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { - - /* Forward the request to the JSP that does all the hard work. */ - request.getRequestDispatcher("WEB-INF/tools.jsp").forward(request, - response); + response.setStatus(HttpServletResponse.SC_MOVED_PERMANENTLY); + response.setHeader("Location", + "https://collector.torproject.org/#references"); } } diff --git a/website/src/org/torproject/metrics/web/status/ConsensusHealthServlet.java b/website/src/org/torproject/metrics/web/status/ConsensusHealthServlet.java index 3c74acd..fea0c91 100644 --- a/website/src/org/torproject/metrics/web/status/ConsensusHealthServlet.java +++ b/website/src/org/torproject/metrics/web/status/ConsensusHealthServlet.java @@ -9,15 +9,14 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +@SuppressWarnings("serial") public class ConsensusHealthServlet extends HttpServlet { - private static final long serialVersionUID = 8349991221914797433L; - public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { - - /* Forward to the actual consensus-health service. */ - response.sendRedirect("https://consensus-health.torproject.org"); + response.setStatus(HttpServletResponse.SC_MOVED_PERMANENTLY); + response.setHeader("Location", + "https://consensus-health.torproject.org"); } } diff --git a/website/src/org/torproject/metrics/web/status/ExoneraTorServlet.java b/website/src/org/torproject/metrics/web/status/ExoneraTorServlet.java index 980a2ba..fdcb177 100644 --- a/website/src/org/torproject/metrics/web/status/ExoneraTorServlet.java +++ b/website/src/org/torproject/metrics/web/status/ExoneraTorServlet.java @@ -1,4 +1,4 @@ -/* Copyright 2011, 2012 The Tor Project +/* Copyright 2011--2014 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.web.status; @@ -9,15 +9,13 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +@SuppressWarnings("serial") public class ExoneraTorServlet extends HttpServlet { - private static final long serialVersionUID = -6227541092325776626L; - public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { - - /* Forward to the actual ExoneraTor service. */ - response.sendRedirect("https://exonerator.torproject.org"); + response.setStatus(HttpServletResponse.SC_MOVED_PERMANENTLY); + response.setHeader("Location", "https://exonerator.torproject.org"); } } diff --git a/website/src/org/torproject/metrics/web/status/RelaySearchServlet.java b/website/src/org/torproject/metrics/web/status/RelaySearchServlet.java index aac7f10..d3b189b 100644 --- a/website/src/org/torproject/metrics/web/status/RelaySearchServlet.java +++ b/website/src/org/torproject/metrics/web/status/RelaySearchServlet.java @@ -1,4 +1,4 @@ -/* Copyright 2011, 2012 The Tor Project +/* Copyright 2011--2014 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.web.status; @@ -9,17 +9,14 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +@SuppressWarnings("serial") public class RelaySearchServlet extends HttpServlet { - private static final long serialVersionUID = -1772662230310611806L; - public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { - - /* Let the JSP display that we're out of service. */ - request.getRequestDispatcher("WEB-INF/relay-search.jsp").forward( - request, response); + response.setStatus(HttpServletResponse.SC_MOVED_PERMANENTLY); + response.setHeader("Location", "https://atlas.torproject.org"); } } diff --git a/website/web/WEB-INF/data.jsp b/website/web/WEB-INF/data.jsp deleted file mode 100644 index 1828c03..0000000 --- a/website/web/WEB-INF/data.jsp +++ /dev/null @@ -1,303 +0,0 @@ -<%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core" %> -<%@ taglib prefix="fmt" uri="http://java.sun.com/jsp/jstl/fmt" %> -<fmt:setLocale value="en_US"/> -<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> -<html> -<head> - <title>Tor Metrics: Data</title> - <meta http-equiv="content-type" content="text/html; charset=ISO-8859-1"> - <link href="/css/stylesheet-ltr.css" type="text/css" rel="stylesheet"> - <link href="/images/favicon.ico" type="image/x-icon" rel="shortcut icon"> -</head> -<body> - <div class="center"> - <%@ include file="banner.jsp"%> - <div class="main-column"> - <h2>Tor Metrics: Data</h2> - <br> - <p><font color="red"><b>Notice:</b> The files linked on this site - are now available on: - <a href="https://collector.torproject.org/">https://collector.torproject.org/</a>. - This page will be automatically redirected there after August 3, - 2014.</font> - </p> - <p>One of the main goals of the Tor Metrics Project is to make all - gathered data available to the public. This approach enables - privacy researchers to perform their own analyses using real data - on the Tor network, and it acts as a safeguard to not gather data - that are too sensitive to publish. The following data are - available (see the <a href="tools.html">Tools</a> section for - details on processing the files):</p> - <ul> - <li><a href="#relaydesc">Relay descriptor archives</a></li> - <li><a href="#bridgedesc">Bridge descriptor archives</a></li> - <li><a href="#bridgeassignments">Bridge pool assignments</a></li> - <li><a href="#performance">Performance data</a></li> - <li><a href="#exitlist">Exit lists</a></li> - </ul> - <p>The tarballs listed on this page and the raw files that were - published on the last three days are also available via - "rsync metrics.torproject.org::".</p> - <p>Note that most descriptors compress really well. You should - expect tarballs to decompress to 20 times the compressed size or - even more.</p> - <br> - <a name="relaydesc"></a> - <h3><a href="#relaydesc" class="anchor">Relay descriptor - archives</a></h3> - <br> - <p>The relay descriptor archives contain all documents that the - directory authorities make available about the network of relays. - These documents include network statuses, server (relay) - descriptors, and extra-info descriptors. - The data formats are described <a href="formats.html">here</a>.</p> - <table width="100%" border="0" cellpadding="5" cellspacing="0" summary=""> - <c:forEach var="item" items="${relayDescriptors}" > - <fmt:formatDate var="longDate" pattern="MMMM yyyy" - value="${item.key}"/> - <tr> - <td>${longDate}</td> - <td> - <c:if test="${item.value['tor'] ne null}" > - <a href="${item.value['tor'][0]}">v1 directories</a> - <c:if test="${item.value['tor'][1] ne null}"> - (<a href="${item.value['tor'][1]}">sig</a>) - </c:if> - </c:if> - </td> - <td> - <c:if test="${item.value['statuses'] ne null}" > - <a href="${item.value['statuses'][0]}">v2 statuses</a> - <c:if test="${item.value['statuses'][1] ne null}"> - (<a href="${item.value['statuses'][1]}">sig</a>) - </c:if> - </c:if> - </td> - <td> - <c:if test="${item.value['server-descriptors'] ne null}" > - <a href="${item.value['server-descriptors'][0]}">server descriptors</a> - <c:if test="${item.value['server-descriptors'][1] ne null}"> - (<a href="${item.value['server-descriptors'][1]}">sig</a>) - </c:if> - </c:if> - </td> - <td> - <c:if test="${item.value['extra-infos'] ne null}" > - <a href="${item.value['extra-infos'][0]}">extra-infos</a> - <c:if test="${item.value['extra-infos'][1] ne null}"> - (<a href="${item.value['extra-infos'][1]}">sig</a>) - </c:if> - </c:if> - </td> - <td> - <c:if test="${item.value['votes'] ne null}" > - <a href="${item.value['votes'][0]}">v3 votes</a> - <c:if test="${item.value['votes'][1] ne null}"> - (<a href="${item.value['votes'][1]}">sig</a>) - </c:if> - </c:if> - </td> - <td> - <c:if test="${item.value['consensuses'] ne null}" > - <a href="${item.value['consensuses'][0]}">v3 consensuses</a> - <c:if test="${item.value['consensuses'][1] ne null}"> - (<a href="${item.value['consensuses'][1]}">sig</a>) - </c:if> - </c:if> - </td> - <td> - <c:if test="${item.value['microdescs'] ne null}" > - <a href="${item.value['microdescs'][0]}">microdescriptors</a> - <c:if test="${item.value['microdescs'][1] ne null}"> - (<a href="${item.value['microdescs'][1]}">sig</a>) - </c:if> - </c:if> - </td> - </tr> - </c:forEach> - </table> - <c:if test="${certs[0] ne null}"> - <br> - <p>In order to verify the v3 votes and v3 consensuses, download - the tarball of <a href="${certs[0]}">v3 certificates</a> - <c:if test="${certs[1] ne null}"> - (<a href="${certs[1]}">sig</a>) - </c:if> - which is updated whenever new v3 certificates become available.</p> - </c:if> - <c:if test="${relayStatistics[0] ne null}"> - <br> - <p>Some of the relays are configured to gather statistics on the - number of requests or connecting clients, the number of - processed cells per queue, or the number of exiting bytes per - port. Relays running version 0.2.2.4-alpha or higher can include - these statistics in extra-info descriptors, so that they are - included in the relay descriptor archives. This - <a href="${relayStatistics[0]}">archive</a> - <c:if test="${relayStatistics[1] ne null}"> - (<a href="${relayStatistics[1]}">sig</a>) - </c:if> - contains the statistics produced by relays running earlier - versions.</p> - </c:if> - <br> - <a name="bridgedesc"></a> - <h3><a href="#bridgedesc" class="anchor">Bridge descriptor - archives</a></h3> - <br> - <p>The bridge descriptor archives contain similar documents as the - relay descriptor archives, but for the non-public bridges. The - descriptors have been sanitized before publication to remove all - information that could otherwise be used to locate bridges. The - files below contain all documents of a given month, including - bridge network statuses, bridge server descriptors, and bridge - extra-info descriptors. The sanitizing process is described - <a href="formats.html#bridgedesc">here</a>.</p> - <table width="100%" border="0" cellpadding="5" cellspacing="0" summary=""> - <c:forEach var="item" items="${bridgeDescriptors}" > - <fmt:formatDate var="longDate" pattern="MMMM yyyy" - value="${item.key}"/> - <tr> - <td> - <a href="${item.value[0]}">${longDate}</a> - <c:if test="${item.value[1] ne null}"> - (<a href="${item.value[1]}">sig</a>) - </c:if> - </td> - </tr> - </c:forEach> - </table> - <p></p> - <br> - <a name="bridgeassignments"></a> - <h3><a href="#bridgeassignments" class="anchor">Bridge pool - assignments</a></h3> - <br> - <p>BridgeDB periodically dumps the list of running bridges with - information about the rings, subrings, and file buckets to which - they are assigned to a local file. We are archiving sanitized - versions of these files here to analyze how the pool assignment - affects a bridge's usage.</p> - The data format and sanitizing process is described - <a href="formats.html#bridgepool">here</a>.</p> - <table width="100%" border="0" cellpadding="5" cellspacing="0" summary=""> - <c:forEach var="item" items="${bridgePoolAssignments}" > - <fmt:formatDate var="longDate" pattern="MMMM yyyy" - value="${item.key}"/> - <tr> - <td> - <a href="${item.value[0]}">${longDate}</a> - </td> - </tr> - </c:forEach> - </table> - <br> - <a name="performance"></a> - <h3><a href="#performance" class="anchor">Performance - data</a></h3> - <br> - <p>We are continuously measuring the performance of the Tor - network by periodically requesting files of different sizes and - recording the time needed to do so. These measurements take place - on moria, siv, and torperf and use an unmodified Tor client. - The files below contain the output of the torperf application. - The data format is described - <a href="formats.html#torperf">here</a>.</p> - <table width="100%" border="0" cellpadding="5" cellspacing="0" summary=""> - <c:forEach var="item" items="${torperfTarballs}" > - <fmt:formatDate var="longDate" pattern="MMMM yyyy" - value="${item.key}"/> - <tr> - <td> - <a href="${item.value[0]}">${longDate}</a> - </td> - </tr> - </c:forEach> - </table> - <br> - <p>The output above is the result of combining torperf request - data with information about used paths. - The raw files are also available below.</p> - <table width="100%" border="0" cellpadding="5" cellspacing="0" summary=""> - <c:forEach var="item" items="${torperfData}" > - <tr> - <td>${item.key}</td> - <td> - <c:if test="${item.value['50kb'] ne null}" > - <c:if test="${item.value['50kb'][0] ne null}" > - <a href="${item.value['50kb'][0]}">50 KiB requests</a> - </c:if> - <c:if test="${item.value['50kb'][1] ne null}" > - <a href="${item.value['50kb'][1]}">50 KiB path info</a> - </c:if> - </c:if> - </td> - <td> - <c:if test="${item.value['1mb'] ne null}" > - <c:if test="${item.value['1mb'][0] ne null}" > - <a href="${item.value['1mb'][0]}">1 MiB requests</a> - </c:if> - <c:if test="${item.value['1mb'][1] ne null}" > - <a href="${item.value['1mb'][1]}">1 MiB path info</a> - </c:if> - </c:if> - </td> - <td> - <c:if test="${item.value['5mb'] ne null}" > - <c:if test="${item.value['5mb'][0] ne null}" > - <a href="${item.value['5mb'][0]}">5 MiB requests</a> - </c:if> - <c:if test="${item.value['5mb'][1] ne null}" > - <a href="${item.value['5mb'][1]}">5 MiB path info</a> - </c:if> - </c:if> - </td> - </tr> - </c:forEach> - </table> - <br> - <p>We further conducted additional experiments with Torperf in the - past by modifying the guard node selection strategies or circuit - build timeouts. The modified guard node selection strategies are - to pick guard nodes from sets of the a) absolute fastest, b) - absolute slowest, c) best rated vs. advertised ratio or d) worst - rated vs. advertised ratio nodes. The ratio mechanisms provide a - way to select the nodes that the bandwidth authorities think stand - out in their measurement. Experiments are listed by the date when - they ended. Details about the experiment setup are contained in a - README file in the tarballs.</p> - <table width="100%" border="0" cellpadding="5" cellspacing="0" summary=""> - <c:forEach var="item" items="${torperfExperiments}" > - <fmt:formatDate var="endDate" pattern="MMMM dd, yyyy" - value="${item.key}"/> - <tr><td><a href="${item.value[0]}">${endDate}</a></td></tr> - </c:forEach> - </table> - <br> - <a name="exitlist"></a> - <h3><a href="#exitlist" class="anchor">Exit lists</a></h3> - <br> - <p>We are archiving the bulk exit lists used by - <a href="https://check.torproject.org/">Tor Check</a> - containing the IP addresses that exit relays exit from. - The data format is described in - <a href="https://www.torproject.org/tordnsel/exitlist-spec.txt">exitlist-spec.txt</a> - and <a href="formats.html#exitlist">here</a>.</p> - <table width="100%" border="0" cellpadding="5" cellspacing="0" summary=""> - <c:forEach var="item" items="${exitLists}" > - <fmt:formatDate var="longDate" pattern="MMMM yyyy" - value="${item.key}"/> - <tr> - <td> - <a href="${item.value[0]}">${longDate}</a> - </td> - </tr> - </c:forEach> - </table> - </div> - </div> - <div class="bottom" id="bottom"> - <%@ include file="footer.jsp"%> - </div> -</body> -</html> diff --git a/website/web/WEB-INF/formats.jsp b/website/web/WEB-INF/formats.jsp deleted file mode 100644 index 92fd699..0000000 --- a/website/web/WEB-INF/formats.jsp +++ /dev/null @@ -1,992 +0,0 @@ -<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> -<html> -<head> - <title>Tor Metrics: Data Formats</title> - <meta http-equiv="content-type" content="text/html; charset=ISO-8859-1"> - <link href="/css/stylesheet-ltr.css" type="text/css" rel="stylesheet"> - <link href="/images/favicon.ico" type="image/x-icon" rel="shortcut icon"> -</head> -<body> -<div class="center"> -<%@ include file="banner.jsp"%> -<div class="main-column"> -<h2>Tor Metrics: Data Formats</h2> -<br> - -<p><font color="red"><b>Notice:</b> The file formats explained on this -site are now available on: -<a href="https://collector.torproject.org/formats.html">https://collector.torproject.org/formats.html</a>. -This page will be automatically redirected there after August 3, -2014.</font> -</p> -<p> -Statistical analysis in the Tor network can be performed using various -kinds of data. -This page gives an overview of three major data sources for -statistics in the Tor network:</p> - -<ol> -<li>First, we recap measuring the Tor network from public directory -information (<a href="https://research.torproject.org/techreports/metrics-2009-08-07.pdf">PDF</a>) -by describing the data format of -<a href="#serverdesc">server descriptors and network statuses</a>, -and we explain the sanitzation process of (non-public) -<a href="#bridgedesc">bridge directory information</a>.</li> -<li>Second, we describe the numerous aggregate statistics that relays -publish about their usage (<a href="http://freehaven.net/anonbib/cache/wecsr10measuring-tor.pdf">PDF</a>), -including -<a href="#bytehist">byte histories</a>, -<a href="#dirreqstats">directory request statistics</a>, -<a href="#entrystats">connecting client statistics</a>, -<a href="#bridgestats">bridge user statistics</a>, -<a href="#cellstats">cell-queue statistics</a>, -<a href="#exitstats">exit-port statistics</a>, and -<a href="#bidistats">bidirectional connection use</a>.</li> -<li>Third, we delineate the output of various Tor services like -<a href="#bridgepool">BridgeDB</a>, or -<a href="#exitlist">Tor Check</a> as well as specific measurement tools like -<a href="#torperf">Torperf</a>.</li> -</ol> - -<p> -All data described on this page are available for download on the -<a href="data.html">data</a> page. -This page is based on a technical report -(<a href="https://research.torproject.org/techreports/data-2011-03-14.pdf">PDF</a>) -and is very likely more recent than the report. -</p> -<hr> -<br> - -<a name="descriptortypes"></a> -<h3><a href="#descriptortypes" class="anchor">Descriptor types</a></h3> -<br> -<p> -Any file containing descriptors described on this page may contain meta -data in its first text line using the format -<tt>@type $descriptortype $major.$minor</tt>. -Any tool that processes these descriptors may parse files without meta -data or with an unknown descriptor type at its own risk, can safely parse -files with known descriptor type and same major version number, and should -not parse files with known descriptor type and higher major version -number. -</p> - -<p> -The following descriptor types and versions are known. -Gray entries are deprecated, black entries are recent: -</p> - -<ul> -<li><tt>@type server-descriptor 1.0</tt></li> -<li><tt>@type microdescriptor 1.0</tt></li> -<li><tt>@type extra-info 1.0</tt></li> -<li><tt>@type directory 1.0</tt></li> -<li><tt>@type network-status-2 1.0</tt></li> -<li><tt>@type dir-key-certificate-3 1.0</tt></li> -<li><tt>@type network-status-consensus-3 1.0</tt></li> -<li><tt>@type network-status-vote-3 1.0</tt></li> -<li><tt>@type network-status-microdesc-consensus-3 1.0</tt></li> -<li><tt>@type bridge-network-status 1.0</tt></li> -<li><tt>@type bridge-server-descriptor 1.0</tt></li> -<li><font color="gray"><tt>@type bridge-extra-info 1.0</tt></font></li> -<li><font color="gray"><tt>@type bridge-extra-info 1.1</tt> contain - sanitized <tt>transport</tt> lines</font></li> -<li><tt>@type bridge-extra-info 1.2</tt> contain <tt>ntor-onion-key</tt> - lines</li> -<li><tt>@type torperf 1.0</tt></li> -<li><tt>@type bridge-pool-assignment 1.0</tt></li> -<li><tt>@type tordnsel 1.0</tt></li> -</ul> - -<hr> -<br> - -<a name="serverdesc"></a> -<h3><a href="#serverdesc" class="anchor">Server descriptors and network -statuses</a></h3> -<br> -<p> -Relays in the Tor network report their capabilities by publishing server -descriptors to the directory authorities. -The directory authorities confirm reachability of relays and assign flags -to help clients make good path selections. -Every hour, the directory authorities publish a network status consensus -with all known running relays at the time. -Both server descriptors and network statuses constitute a solid data basis -for statistical analysis in the Tor network. -We described the approach to measure the Tor network from public directory -information in the HotPETS 2009 paper -(<a href="https://research.torproject.org/techreports/metrics-2009-08-07.pdf">PDF</a>) -and provide interactive -<a href="graphs.html">graphs</a> on the metrics -website. -We briefly describe the most interesting pieces of the -two descriptor formats that can be used for statistics. -</p> - -<p> -The server descriptors published by relays at least once every 18 hours -contain the necessary information for clients to build circuits using a -given relay. -These server descriptors can also be useful for statistical analysis of -the Tor network infrastructure. -</p> - -<p> -We assume that the majority of server descriptors are correct. -But when performing statistical analysis on server descriptors, one has to -keep in mind that only a small subset of the information written to server -descriptors is confirmed by the trusted directory authorities. -In theory, relays can provide false information in their server -descriptors, even though the incentive to do so is probably low. -</p> - -<blockquote> -<p> -<i>Server descriptor published by relay <tt>blutmagie</tt> (without -cryptographic keys and hashes):</i> -</p> -<p> -<tt>router blutmagie 192.251.226.206 443 0 80</tt><br> -<tt>platform Tor 0.2.2.20-alpha on Linux x86_64</tt><br> -<tt>opt protocols Link 1 2 Circuit 1</tt><br> -<tt>published 2010-12-27 14:35:27</tt><br> -<tt>opt fingerprint 6297 B13A 687B 521A 59C6 BD79 188A 2501 EC03 A065</tt><br> -<tt>uptime 445412</tt><br> -<tt>bandwidth 14336000 18432000 15905178</tt><br> -<tt>opt extra-info-digest 5C1D5D6F8B243304079BC15CD96C7FCCB88322D4</tt><br> -<tt>opt caches-extra-info</tt><br> -<tt>onion-key</tt><br> -<tt>[...]</tt><br> -<tt>signing-key</tt><br> -<tt>[...]</tt><br> -<tt>family $66CA87E164F1CFCE8C3BB5C095217A28578B8BAF - $67EC84376D9C4C467DCE8621AACA109160B5264E - $7B698D327F1695590408FED95CDEE1565774D136</tt><br> -<tt>opt hidden-service-dir</tt><br> -<tt>contact abuse(a)blutmagie.de</tt><br> -<tt>reject 0.0.0.0/8:*</tt><br> -<tt>reject 169.254.0.0/16:*</tt><br> -<tt>reject 127.0.0.0/8:*</tt><br> -<tt>reject 192.168.0.0/16:*</tt><br> -<tt>reject 10.0.0.0/8:*</tt><br> -<tt>reject 172.16.0.0/12:*</tt><br> -<tt>reject 192.251.226.206:*</tt><br> -<tt>reject *:25</tt><br> -<tt>reject *:119</tt><br> -<tt>reject *:135-139</tt><br> -<tt>reject *:445</tt><br> -<tt>reject *:465</tt><br> -<tt>reject *:563</tt><br> -<tt>reject *:587</tt><br> -<tt>reject *:1214</tt><br> -<tt>reject *:4661-4666</tt><br> -<tt>reject *:6346-6429</tt><br> -<tt>reject *:6660-6999</tt><br> -<tt>accept *:*</tt><br> -<tt>router-signature</tt><br> -<tt>[...]</tt><br> -</p> -</blockquote> - -<p> -The document above shows an example server descriptor. -The following data fields in server descriptors may be relevant to -statistical analysis: -</p> - -<ul> -<li><b>IP address and ports:</b> Relays provide their IP address -and ports where they accept requests to build circuits and directory -requests. -These data fields are contained in the first line of a server descriptor -starting with <tt>router</tt>. -Note that in rare cases, the IP address provided here can be different -from the IP address used for exiting to the Internet. -The latter can be found in the exit lists produced by Tor Check as -described in the <a href="#exitlist">Tor Check exit lists</a> section below. -</li> -<li><b>Operating system and Tor software version:</b> Relays include -their operating system and Tor software version in their server -descriptors in the <tt>platform</tt> line. -While this information is very likely correct in most cases, a few relay -operators may try to impede hacking attempts by providing false platform -strings.</li> -<li><b>Uptime:</b> Relays include the number of seconds since the -last restart in their server descriptor in the <tt>uptime</tt> line.</li> -<li><b>Own measured bandwidth:</b> Relays report the bandwidth that -they are willing to provide on average and for short periods of time. -Relays also perform periodic bandwidth self-tests and report their actual -available bandwidth. -The latter was used by clients to weight relays in the path selection -algorithm and was sometimes subject to manipulation by malicious relays. -All three bandwidth values can be found in a server descriptor's -<tt>bandwidth</tt> line. -With the introduction of -<a href="http://gitweb.torproject.org/torflow.git/">bandwidth scanners</a>, the self-reported relay -bandwidth in server descriptors has become less -relevant.</li> -<li><b>Relay family:</b> Some relay operators who run more than one -relay organize their relays in relay families, so that clients don't pick -more than one of these relays for a single circuit. -Each relay belonging to a relay family lists the members of that family -either by nickname or fingerprint in its server descriptor in the -<tt>family</tt> line.</li> -<li><b>Exit policy:</b> Relays define their exit policy by including -firewall-like rules which outgoing connections they reject or accept in -the <tt>reject</tt> and <tt>accept</tt> lines.</li> -</ul> - -<p> -These are just a subset of the fields in a server descriptor that seem -relevant for statistical analysis. -For a complete list of fields in server descriptors, see the <a href="https://gitweb.torproject.org/torspec.git/blob/HEAD:/dir-spec.txt">directory -protocol specification</a>. -</p> - -<p> -Every hour, the directory authorities publish a new network status that -contains a list of all running relays. -The directory authorities confirm reachability of the contained relays and -assign flags based on the relays' characteristics. -The entries in a network status reference the last published server -descriptor of a relay. -</p> - -<p> -The network statuses are relevant for statistical analysis, because they -constitute trusted snapshots of the Tor network. -Anyone can publish as many server descriptors as they want, but only the -directory authorities can confirm that a relay was running at a given -time. -Most statistics on the Tor network infrastructure rely on network statuses -and possibly combine them with the referenced server descriptors. -The document below shows the network status entry referencing -the server descriptor above. -In addition to the reachability information, network statuses contain the -following fields that may be relevant for statistical analysis: -</p> - -<blockquote> -<p> -<i>Network status entry of relay <tt>blutmagie</tt>:</i> -</p> -<p> -<tt>r blutmagie YpexOmh7UhpZxr15GIolAewDoGU - lFY7WmD/yvVFp9drmZzNeTxZ6dw 2010-12-27 14:35:27 192.251.226.206 - 443 80</tt><br> -<tt>s Exit Fast Guard HSDir Named Running Stable V2Dir Valid</tt><br> -<tt>v Tor 0.2.2.20-alpha</tt><br> -<tt>w Bandwidth=30800</tt><br> -<tt>p reject 25,119,135-139,445,465,563,587,1214,4661-4666,6346-6429,6660-6999</tt><br> -</p> -</blockquote> - -<ul> -<li><b>Relay flags:</b> The directory authorities assign flags to -relays based on their characteristics to the line starting with <tt>s</tt>. -Examples are the <tt>Exit</tt> flag if a relay permits exiting to the -Internet and the <tt>Guard</tt> flag if a relay is stable enough to be -picked as guard node</li> -<li><b>Relay version:</b> The directory authorities include the -version part of the platform string written to server descriptors in the -network status in the line starting with <tt>v</tt>.</li> -<li><b>Bandwidth weights:</b> The network status contains a bandwidth -weight for every relay in the lines with <tt>w</tt> that clients shall use -for weighting relays in their path selection algorithm. -This bandwidth weight is either the self-reported bandwidth of the relay -or the bandwidth measured by the bandwidth scanners.</li> -<li><b>Exit policy summary:</b> Every entry in a network status -contains a summary version of a relay's exit policy in the line starting -with <tt>p</tt>. -This summary is a list of accepted or rejected ports for exit to most IP -addresses.</li> -</ul> -<hr> -<br> - -<a name="bridgedesc"></a> -<h3><a href="#bridgedesc" class="anchor">Sanitized bridge -descriptors</a></h3> -<br> -Bridges in the Tor network publish server descriptors to the bridge -authority which in turn generates a bridge network status. -We cannot, however, make the bridge server descriptors and bridge network -statuses available for statistical analysis as we do with the relay server -descriptors and relay network statuses. -The problem is that bridge server descriptors and network statuses contain -bridge IP addresses and other sensitive information that shall not be made -publicly available. -We therefore sanitize bridge descriptors by removing all potentially -identifying information and publish sanitized versions of the descriptors. -The processing steps for sanitizing bridge descriptors are as follows: - -<ol> -<li><b>Replace the bridge identity with its SHA1 value:</b> Clients -can request a bridge's current descriptor by sending its identity string -to the bridge authority. -This is a feature to make bridges on dynamic IP addresses useful. -Therefore, the original identities (and anything that could be used to -derive them) need to be removed from the descriptors. -The bridge identity is replaced with its SHA1 hash value. -The idea is to have a consistent replacement that remains stable over -months or even years (without keeping a secret for a keyed hash function).</li> -<li><b>Remove all cryptographic keys and signatures:</b> It would be -straightforward to learn about the bridge identity from the bridge's -public key. -Replacing keys by newly generated ones seemed to be unnecessary (and would -involve keeping a state over months/years), so that all cryptographic -objects have simply been removed.</li> -<li><b>Replace IP address with IP address hash:</b> Of course, IP -addresses need to be sanitized, too. -<ul><li>IPv4 addresses are replaced with <tt>10.x.x.x</tt> with -<tt>x.x.x</tt> being the 3 byte output of -<tt>H(IP address | bridge identity | secret)[:3]</tt>. -The input <tt>IP address</tt> is the 4-byte long binary representation of -the bridge's current IP address. -The <tt>bridge identity</tt> is the 20-byte long binary representation of -the bridge's long-term identity fingerprint. -The <tt>secret</tt> is a 31-byte long secure random string that changes once -per month for all descriptors and statuses published in that month. -<tt>H()</tt> is SHA-256. -The <tt>[:3]</tt> operator means that we pick the 3 most significant bytes -of the result.</li> -<li>IPv6 addresses are replaced with <tt>[fd9f:2e19:3bcf::xx:xxxx]</tt> -with <tt>xx:xxxx</tt> being the hex-formatted 3 byte output of a similar -hash function as described for IPv4 addresses. -The only differences are that the input <tt>IP address</tt> is 16 bytes -long and the <tt>secret</tt> is only 19 bytes long.</li></ul> -<li><b>Replace contact information:</b> If there is contact -information in a descriptor, the contact line is changed to -<tt>somebody</tt>.</li> -<li><b>Remove pluggable transport addresses and arguments:</b> Bridges may -provide transports in addition to the onion-routing protocol and include -information about these transports in their extra-info descriptors for -BridgeDB. In that case, any IP addresses, TCP ports, or additional -arguments are removed, only leaving in the supported transport names.</li> -</ol> - -<p> -Apart from these processing steps, sanitized bridge server descriptors -follow the same format as relay server descriptors. -The same applies to sanitized bridge extra-info descriptors. -Sanitized bridge network statuses are similar to version 2 relay network -statuses, but with only a <tt>published</tt> line in the header and -without any lines in the footer. -</p> - -<p> -The two documents below show an example bridge server -descriptor that is referenced from a bridge network status. -For more details about this process, see the -<a href="https://gitweb.torproject.org/metrics-db.git">metrics data processor</a> software. -</p> - -<blockquote> -<p> -<i>Sanitized bridge server descriptor:</i> -</p> -<p> -<tt>@type bridge-server-descriptor 1.0</tt><br> -<tt>router Hawthorne 10.175.105.22 443 0 0</tt><br> -<tt>platform Tor 0.2.2.19-alpha (git-1988927edecce4c7) on Linux i686</tt><br> -<tt>opt protocols Link 1 2 Circuit 1</tt><br> -<tt>published 2010-12-27 18:55:01</tt><br> -<tt>opt fingerprint A5FA 7F38 B02A 415E 72FE 614C 64A1 E5A9 2BA9 9BBD</tt><br> -<tt>uptime 2347112</tt><br> -<tt>bandwidth 5242880 10485760 1016594</tt><br> -<tt>opt extra-info-digest E729BCB5E06A5657A73151B55354EB003D2BAE0F</tt><br> -<tt>opt hidden-service-dir</tt><br> -<tt>contact somebody</tt><br> -<tt>reject *:*</tt><br> -<tt>router-digest 46DFDBE7B67B7C90A1962B0B5AA4526FAF406979</tt><br> -</p> -</blockquote> - -<blockquote> -<p> -<i>Sanitized bridge network status:</i> -</p> -<p> -<tt>@type bridge-network-status 1.0</tt><br> -<tt>published 2010-12-27 22:07:03</tt><br> -<tt>[...status entries...]</tt><br> -<tt>r Hawthorne pfp/OLAqQV5y/mFMZKHlqSupm70 Rt/b57Z7fJChlisLWqRSb69AaXk - 2010-12-27 18:55:01 10.175.105.22 443 0</tt><br> -<tt>s Fast Guard HSDir Running Stable Valid</tt><br> -<tt>[...status entries...]</tt><br> -</p> -</blockquote> -<hr> -<br> - -<a name="bytehist"></a> -<h3><a href="#bytehist" class="anchor">Byte histories</a></h3> -<br> -<p> -Relays include aggregate statistics in their descriptors that they upload -to the directory authorities. -These aggregate statistics are contained in extra-info descriptors that -are published in companion with server descriptors. -Extra-info descriptors are not required for clients to build circuits. -An extra-info descriptor belonging to a server descriptor is referenced by -its SHA1 hash value. -</p> - -<p> -Byte histories were the first statistical data that relays published about -their usage. -Relays report the number of written and read bytes in 15-minute intervals -throughout the last 24 hours. -The extra-info descriptor in the document below contains the byte -histories in the two lines starting with <tt>write-history</tt> and -<tt>read-history</tt>. -More details about these statistics can be found in the <a href="https://gitweb.torproject.org/torspec.git/blob/HEAD:/dir-spec.txt">directory protocol -specification</a>. -</p> - -<blockquote> -<p> -<i>Extra-info descriptor published by relay <tt>blutmagie</tt> -(without cryptographic signature and with long lines being truncated):</i> -</p> -<p> -<tt>extra-info blutmagie 6297B13A687B521A59C6BD79188A2501EC03A065</tt><br> -<tt>published 2010-12-27 14:35:27</tt><br> -<tt>write-history 2010-12-27 14:34:05 (900 s) 12902389760,12902402048,12859373568,12894131200,[...]</tt><br> -<tt>read-history 2010-12-27 14:34:05 (900 s) 12770249728,12833485824,12661140480,12872439808,[...]</tt><br> -<tt>dirreq-write-history 2010-12-27 14:26:13 (900 s) 51731456,60808192,56740864,54948864,[...]</tt><br> -<tt>dirreq-read-history 2010-12-27 14:26:13 (900 s) 4747264,4767744,4511744,4752384,[...]</tt><br> -<tt>dirreq-stats-end 2010-12-27 10:51:09 (86400 s)</tt><br> -<tt>dirreq-v3-ips us=2000,de=1344,fr=744,kr=712,[...]</tt><br> -<tt>dirreq-v2-ips ??=8,au=8,cn=8,cz=8,[...]</tt><br> -<tt>dirreq-v3-reqs us=2368,de=1680,kr=1048,fr=800,[...]</tt><br> -<tt>dirreq-v2-reqs id=48,??=8,au=8,cn=8,[...]</tt><br> -<tt>dirreq-v3-resp ok=12504,not-enough-sigs=0,unavailable=0,not-found=0,not-modified=0,busy=128</tt><br> -<tt>dirreq-v2-resp ok=64,unavailable=0,not-found=8,not-modified=0,busy=8</tt><br> -<tt>dirreq-v2-share 1.03%</tt><br> -<tt>dirreq-v3-share 1.03%</tt><br> -<tt>dirreq-v3-direct-dl complete=316,timeout=4,running=0,min=4649,d1=36436,d2=68056,q1=76600,d3=87891,d4=131294,md=173579,d6=229695,d7=294528,q3=332053,d8=376301,d9=530252,max=2129698</tt><br> -<tt>dirreq-v2-direct-dl complete=16,timeout=52,running=0,min=9769,d1=9769,d2=9844,q1=9981,d3=9981,d4=27297,md=33640,d6=60814,d7=205884,q3=205884,d8=361137,d9=628256,max=956009</tt><br> -<tt>dirreq-v3-tunneled-dl complete=12088,timeout=92,running=4,min=534,d1=31351,d2=49166,q1=58490,d3=70774,d4=88192,md=109778,d6=152389,d7=203435,q3=246377,d8=323837,d9=559237,max=26601000</tt><br> -<tt>dirreq-v2-tunneled-dl complete=0,timeout=0,running=0</tt><br> -<tt>entry-stats-end 2010-12-27 10:51:09 (86400 s)</tt><br> -<tt>entry-ips de=11024,us=10672,ir=5936,fr=5040,[...]</tt><br> -<tt>exit-stats-end 2010-12-27 10:51:09 (86400 s)</tt><br> -<tt>exit-kibibytes-written 80=6758009,443=498987,4000=227483,5004=1182656,11000=22767,19371=1428809,31551=8212,41500=965584,51413=3772428,56424=1912605,other=175227777</tt><br> -<tt>exit-kibibytes-read 80=197075167,443=5954607,4000=1660990,5004=1808563,11000=1893893,19371=130360,31551=7588414,41500=756287,51413=2994144,56424=1646509,other=288412366</tt><br> -<tt>exit-streams-opened 80=5095484,443=359256,4000=4508,5004=22288,11000=124,19371=24,31551=40,41500=96,51413=16840,56424=28,other=1970964</tt><br> -</p> -</blockquote> -<hr> -<br> - -<a name="dirreqstats"></a> -<h3><a href="#dirreqstats" class="anchor">Directory requests</a></h3> -<br> -<p> -The directory authorities and directory mirrors report statistical data -about processed directory requests. -Starting with Tor version 0.2.2.15-alpha, all directories report the -number of written and read bytes for answering directory requests. -The format is similar to the format of byte histories as described in the -previous section. -The relevant lines are <tt>dirreq-write-history</tt> and -<tt>dirreq-read-history</tt> in the document listed in the -<a href="#bytehist">Byte histories</a> section above. -These two lines contain the subset of total read and written bytes that -the directory mirror spent on responding to any kind of directory request, -including network statuses, server descriptors, extra-info descriptors, -authority certificates, etc. -</p> - -<p> -The directories further report statistics on answering directory requests -for network statuses only. -For Tor versions before 0.2.3.x, relay operators had to manually enable -these statistics, which is why only a few directories report them. -The lines starting with <tt>dirreq-v3-</tt> all belong to the directory -request statistics (the lines starting with <tt>dirreq-v2-</tt> report -similar statistics for version 2 of the directory protocol which is -deprecated at the time of writing this report). -The following fields may be relevant for statistical analysis: -</p> - -<ul> -<li><b>Unique IP addresses:</b> The numbers in <tt>dirreq-v3-ips</tt> -denote the unique IP addresses of clients requesting network statuses by -country.</li> -<li><b>Network status requests:</b> The numbers in -<tt>dirreq-v3-reqs</tt> constitute the total network status requests by -country.</li> -<li><b>Request share:</b> The percentage in <tt>dirreq-v3-share</tt> is -an estimate of the share of directory requests that the reporting relay -expects to see in the Tor network. -In a tech report (<a href="https://research.torproject.org/techreports/countingusers-2010-11-30.pdf">PDF</a>) -we found that this estimate isn't very useful -for statistical analysis because of the different approaches that clients -take to select directory mirrors. -The fraction of written directory bytes (<tt>dirreq-write-history</tt>) can -be used to derive a better metric for the share of directory requests.</li> -<li><b>Network status responses:</b> The directories also report -whether they could provide the requested network status to clients in -<tt>dirreq-v3-resp</tt>. -This information was mostly used to diagnose error rates in version 2 of -the directory protocol where a lot of directories replied to network -status requests with <tt>503 Busy</tt>. -In version 3 of the directory protocol, most responses contain the status -code <tt>200 OK</tt>.</li> -<li><b>Network status download times:</b> The line -<tt>dirreq-v3-direct-dl</tt> contains statistics on the download of network -statuses via the relay's directory port. -The line <tt>dirreq-v3-tunneled-dl</tt> contains similar statistics on -downloads via a 1-hop circuit between client and directory (which is the -common approach in version 3 of the directory protocol). -Relays report how many requests have been completed, have timed out, and -are still running at the end of a 24-hour time interval as well as the -minimum, maximum, median, quartiles, and deciles of download times.</li> -</ul> - -<p> -More details about these statistics can be found in the <a href="https://gitweb.torproject.org/torspec.git/blob/HEAD:/dir-spec.txt">directory protocol -specification</a>. -</p> -<hr> -<br> - -<a name="entrystats"></a> -<h3><a href="#entrystats" class="anchor">Connecting clients</a></h3> -<br> -<p> -Relays can be configured to report per-country statistics on directly -connecting clients. -This metric includes clients connecting to a relay in order to build -circuits and clients creating a 1-hop circuit to request directory -information. -In practice, the latter number outweighs the former number. -The <tt>entry-ips</tt> line in the document listed in the -<a href="#bytehist">Byte histories</a> section above -shows the number -of unique IP addresses connecting to the relay by country. -More details about these statistics can be found in the <a href="https://gitweb.torproject.org/torspec.git/blob/HEAD:/dir-spec.txt">directory protocol -specification</a>. -</p> -<hr> -<br> - -<a name="bridgestats"></a> -<h3><a href="#bridgestats" class="anchor">Bridge users</a></h3> -<br> -<p> -Bridges report statistics on connecting bridge clients in their extra-info -descriptors. -The document below shows a bridge extra-info descriptor -with the bridge user statistics in the <tt>bridge-ips</tt> line. - -<blockquote> -<p> -<i>Sanitized bridge extra-info descriptor:</i> -</p> -<p> -<tt>extra-info Unnamed A5FA7F38B02A415E72FE614C64A1E5A92BA99BBD</tt><br> -<tt>published 2010-12-27 18:55:01</tt><br> -<tt>write-history 2010-12-27 18:43:50 (900 s) 151712768,176698368,180030464,163150848,[...]</tt><br> -<tt>read-history 2010-12-27 18:43:50 (900 s) 148109312,172274688,172168192,161094656,[...]</tt><br> -<tt>bridge-stats-end 2010-12-27 14:56:29 (86400 s)</tt><br> -<tt>bridge-ips sa=48,us=40,de=32,ir=32,[...]</tt><br> -</p> -</blockquote> - -<p> -Bridges running Tor version 0.2.2.3-alpha or earlier report bridge users -in a similar line starting with <tt>geoip-client-origins</tt>. -The reason for switching to <tt>bridge-ips</tt> was that the measurement -interval in <tt>geoip-client-origins</tt> had a variable length, whereas the -measurement interval in 0.2.2.4-alpha and later is set to exactly -24 hours. -In order to clearly distinguish the new measurement intervals from the old -ones, the new keywords have been introduced. -More details about these statistics can be found in the <a href="https://gitweb.torproject.org/torspec.git/blob/HEAD:/dir-spec.txt">directory protocol -specification</a>. -</p> -<hr> -<br> - -<a name="cellstats"></a> -<h3><a href="#cellstats" class="anchor">Cell-queue statistics</a></h3> -<br> -<p> -Relays can be configured to report aggregate statistics on their cell -queues. -These statistics include average processed cells, average number of queued -cells, and average time that cells spend in circuits. -Circuits are split into deciles based on the number of processed cells. -The statistics are provided for circuit deciles from loudest to quietest -circuits. -The document below shows the cell statistics contained in an -extra-info descriptor by relay <tt>gabelmoo</tt>. -An early analysis of cell-queue statistics can be found in a tech report -(<a href="https://research.torproject.org/techreports/bufferstats-2009-08-25.pdf">PDF</a>). -More details about these statistics can be found in the <a href="https://gitweb.torproject.org/torspec.git/blob/HEAD:/dir-spec.txt">directory protocol -specification</a>. -</p> - -<blockquote> -<p> -<i>Cell statistics in extra-info descriptor by relay <tt>gabelmoo</tt>:</i> -</p> -<p> -<tt>cell-stats-end 2010-12-27 09:59:50 (86400 s)</tt><br> -<tt>cell-processed-cells 4563,153,42,15,7,7,6,5,4,2</tt><br> -<tt>cell-queued-cells 9.39,0.98,0.09,0.01,0.00,0.00,0.00,0.01,0.00, - 0.01</tt><br> -<tt>cell-time-in-queue 2248,807,277,92,49,22,52,55,81,148</tt><br> -<tt>cell-circuits-per-decile 7233</tt><br> -</p> -</blockquote> -<hr> -<br> - -<a name="exitstats"></a> -<h3><a href="#exitstats" class="anchor">Exit-port statistics</a></h3> -<br> -<p> -Exit relays running Tor version 0.2.1.1-alpha or higher can be configured -to report aggregate statistics on exiting connections. -These relays report the number of opened streams, written and read bytes -by exiting port. -Until version 0.2.2.19-alpha, relays reported all ports exceeding a -threshold of 0.01 % of all written and read exit bytes. -Starting with version 0.2.2.20-alpha, relays only report the top 10 ports -in exit-port statistics in order not to exceed the maximum extra-info -descriptor length of 50 KB. -The document listed in the -<a href="#bytehist">Byte histories</a> section above contains -exit-port statistics in the lines starting with <tt>exit-</tt>. -More details about these statistics can be found in the <a href="https://gitweb.torproject.org/torspec.git/blob/HEAD:/dir-spec.txt">directory protocol -specification</a>. -</p> -<hr> -<br> - -<a name="bidistats"></a> -<h3><a href="#bidistats" class="anchor">Bidirectional connection -use</a></h3> -<br> -<p> -Relays running Tor version 0.2.3.1-alpha or higher can be configured to -report what fraction of connections is used uni- or bi-directionally. -Every 10 seconds, relays determine for every connection whether they read -and wrote less than a threshold of 20 KiB. -Connections below this threshold are labeled as "Below Threshold". -For the remaining connections, relays report whether they read/wrote at -least 10 times as many bytes as they wrote/read. -If so, they classify a connection as "Mostly reading" or "Mostly -writing," respectively. -All other connections are classified as "Both reading and writing." -After classifying connections, read and write counters are reset for the -next 10-second interval. -Statistics are aggregated over 24 hours. -The document below shows the bidirectional connection use -statistics in an extra-info descriptor by relay <tt>zweifaltigkeit</tt>. -The four numbers denote the number of connections "Below threshold," -"Mostly reading," "Mostly writing," and "Both reading and writing." -More details about these statistics can be found in the <a href="https://gitweb.torproject.org/torspec.git/blob/HEAD:/dir-spec.txt">directory protocol -specification</a>. -</p> - -<blockquote> -<p> -<i>Bidirectional connection use statistic in extra-info descriptor -by relay <tt>zweifaltigkeit</tt>:</i> -</p> -<p> -<tt>conn-bi-direct 2010-12-28 15:55:11 (86400 s) 387465,45285,55361,81786</tt> -</p> -</blockquote> -<hr> -<br> - -<a name="torperf"></a> -<h3><a href="#torperf" class="anchor">Torperf output files</a></h3> -<br> -<p> -Torperf is a little tool that measures Tor's performance as users -experience it. -Torperf uses a trivial SOCKS client to download files of various sizes -over the Tor network and notes how long substeps take. -Torperf can be -<a href="https://metrics.torproject.org/tools.html">downloaded</a> -from the metrics -website. -A Torperf results file contains a single line per Torperf run with -<tt>key=value</tt> pairs. -Such a result line is sufficient to learn about 1) the Tor and Torperf -configuration, 2) measurement results, and 3) additional information that -might help explain the results. -Known keys are explained below. -</p> -<ul> -<li>Configuration -<ul> -<li><tt>SOURCE:</tt> Configured name of the data source; required.</li> -<li><tt>FILESIZE:</tt> Configured file size in bytes; required.</li> -<li>Other meta data describing the Tor or Torperf configuration, e.g., -GUARD for a custom guard choice; optional.</li> -</ul> -<li>Measurement results -<ul> -<li><tt>START:</tt> Time when the connection process starts; -required.</li> -<li><tt>SOCKET:</tt> Time when the socket was created; required.</li> -<li><tt>CONNECT:</tt> Time when the socket was connected; required.</li> -<li><tt>NEGOTIATE:</tt> Time when SOCKS 5 authentication methods have been -negotiated; required.</li> -<li><tt>REQUEST:</tt> Time when the SOCKS request was sent; required.</li> -<li><tt>RESPONSE:</tt> Time when the SOCKS response was received; -required.</li> -<li><tt>DATAREQUEST:</tt> Time when the HTTP request was written; -required.</li> -<li><tt>DATARESPONSE:</tt> Time when the first response was received; -required.</li> -<li><tt>DATACOMPLETE:</tt> Time when the payload was complete; -required.</li> -<li><tt>WRITEBYTES:</tt> Total number of bytes written; required.</li> -<li><tt>READBYTES:</tt> Total number of bytes read; required.</li> -<li><tt>DIDTIMEOUT:</tt> 1 if the request timed out, 0 otherwise; -optional.</li> -<li><tt>DATAPERCx:</tt> Time when x% of expected bytes were read for -x = { 10, 20, 30, 40, 50, 60, 70, 80, 90 }; optional.</li> -<li>Other measurement results, e.g., START_RENDCIRC, GOT_INTROCIRC, etc. -for hidden-service measurements; optional.</li> -</ul> -<li>Additional information -<ul> -<li><tt>LAUNCH:</tt> Time when the circuit was launched; optional.</li> -<li><tt>USED_AT:</tt> Time when this circuit was used; optional.</li> -<li><tt>PATH:</tt> List of relays in the circuit, separated by commas; -optional.</li> -<li><tt>BUILDTIMES:</tt> List of times when circuit hops were built, -separated by commas; optional.</li> -<li><tt>TIMEOUT:</tt> Circuit build timeout that the Tor client used when -building this circuit; optional.</li> -<li><tt>QUANTILE:</tt> Circuit build time quantile that the Tor client -uses to determine its circuit-build timeout; optional.</li> -<li><tt>CIRC_ID:</tt> Circuit identifier of the circuit used for this -measurement; optional.</li> -<li><tt>USED_BY:</tt> Stream identifier of the stream used for this -measurement; optional.</li> -<li>Other fields containing additional information; optional.</li> -</ul> -</ul> - -<blockquote> -<p> -<i>Torperf <tt>.tpf</tt> output lines for a single request to download a -50 KiB file (reformatted):</i> -</p> - -<p> -<tt>BUILDTIMES=1.16901898384,1.86555600166,2.13295292854</tt><br> -<tt>CIRC_ID=9878</tt><br> -<tt>CONNECT=1338357901.42</tt><br> -<tt>DATACOMPLETE=1338357902.91</tt><br> -<tt>DATAPERC10=1338357902.48</tt><br> -<tt>DATAPERC20=1338357902.48</tt><br> -<tt>DATAPERC30=1338357902.61</tt><br> -<tt>DATAPERC40=1338357902.64</tt><br> -<tt>DATAPERC50=1338357902.65</tt><br> -<tt>DATAPERC60=1338357902.74</tt><br> -<tt>DATAPERC70=1338357902.74</tt><br> -<tt>DATAPERC80=1338357902.75</tt><br> -<tt>DATAPERC90=1338357902.79</tt><br> -<tt>DATAREQUEST=1338357901.83</tt><br> -<tt>DATARESPONSE=1338357902.25</tt><br> -<tt>DIDTIMEOUT=0</tt><br> -<tt>FILESIZE=51200</tt><br> -<tt>LAUNCH=1338357661.74</tt><br> -<tt>NEGOTIATE=1338357901.42</tt><br> -<tt>PATH=$980D326017CEF4CBBF4089FBABE767DC83D059AF,$03545609092A24C71CCAD2F4523F5CCC6714F159,$CAC3CF7154AE9C656C4096DC38B4EFA145905654</tt><br> -<tt>QUANTILE=0.800000</tt><br> -<tt>READBYTES=51442</tt><br> -<tt>REQUEST=1338357901.42</tt><br> -<tt>RESPONSE=1338357901.83</tt><br> -<tt>SOCKET=1338357901.42</tt><br> -<tt>SOURCE=torperf</tt><br> -<tt>START=1338357901.42</tt><br> -<tt>TIMEOUT=5049</tt><br> -<tt>USED_AT=1338357902.91</tt><br> -<tt>USED_BY=18869</tt><br> -<tt>WRITEBYTES=75</tt><br> -</p> -</blockquote> -<br> - -<p> -Torperf can produce two output files: <tt>.data</tt> and -<tt>.extradata</tt>. -The <tt>.data</tt> file contains timestamps for request substeps and the -byte summaries for downloading a test file via Tor. -The document below shows an example output of a Torperf run. -The timestamps are seconds and microseconds since 1970-01-01 -00:00:00.000000. -Torperf can be configured to write <tt>.extradata</tt> files by attaching -a Tor controller and writing certain controller events to disk. -The format of a <tt>.extradata</tt> line is similar to the combined format -as specified above, except that it can only contain "Additional -information" keywords. -</p> - -<blockquote> -<p> -<i>Torperf <tt>.data</tt> and <tt>.extradata</tt> output lines for a -single request to download a 50 KiB file (reformatted and annotated with -comments):</i> -</p> - -<p> -<tt># Timestamps and byte summaries contained in .data files:</tt><br> -<tt>1338357901 422336 # Connection process started</tt><br> -<tt>1338357901 422346 # After socket is created</tt><br> -<tt>1338357901 422521 # After socket is connected</tt><br> -<tt>1338357901 422604 # After authentication methods are negotiated (SOCKS 5 only)</tt><br> -<tt>1338357901 423550 # After SOCKS request is sent</tt><br> -<tt>1338357901 839639 # After SOCKS response is received</tt><br> -<tt>1338357901 839849 # After HTTP request is written</tt><br> -<tt>1338357902 258157 # After first response is received</tt><br> -<tt>1338357902 914263 # After payload is complete</tt><br> -<tt>75 # Written bytes</tt><br> -<tt>51442 # Read bytes</tt><br> -<tt>0 # Timeout (optional field)</tt><br> -<tt>1338357902 481591 # After 10% of expected bytes are read (optional field)</tt><br> -<tt>1338357902 482719 # After 20% of expected bytes are read (optional field)</tt><br> -<tt>1338357902 613169 # After 30% of expected bytes are read (optional field)</tt><br> -<tt>1338357902 647108 # After 40% of expected bytes are read (optional field)</tt><br> -<tt>1338357902 651764 # After 50% of expected bytes are read (optional field)</tt><br> -<tt>1338357902 743705 # After 60% of expected bytes are read (optional field)</tt><br> -<tt>1338357902 743876 # After 70% of expected bytes are read (optional field)</tt><br> -<tt>1338357902 757475 # After 80% of expected bytes are read (optional field)</tt><br> -<tt>1338357902 795100 # After 90% of expected bytes are read (optional field)</tt><br> -</p> - -<p> -<tt># Path information contained in .extradata files:</tt><br> -<tt>CIRC_ID=9878</tt><br> -<tt>LAUNCH=1338357661.74</tt><br> -<tt>PATH=$980D326017CEF4CBBF4089FBABE767DC83D059AF,$03545609092A24C71CCAD2F4523F5CCC6714F159,$CAC3CF7154AE9C656C4096DC38B4EFA145905654</tt><br> -<tt>BUILDTIMES=1.16901898384,1.86555600166,2.13295292854</tt><br> -<tt>USED_AT=1338357902.91</tt><br> -<tt>USED_BY=18869</tt><br> -<tt>TIMEOUT=5049</tt><br> -<tt>QUANTILE=0.800000</tt><br> -</p> -</blockquote> -<hr> -<br> - -<a name="bridgepool"></a> -<h3><a href="#bridgepool" class="anchor">BridgeDB pool assignment -files</a></h3> -<br> -<p> -BridgeDB is the software that receives bridge network statuses containing -the information which bridges are running from the bridge authority, -assigns these bridges to persistent distribution rings, and hands them out -to bridge users. -BridgeDB periodically dumps the list of running bridges with information -about the rings, subrings, and file buckets to which they are assigned to -a local file. -The sanitized versions of these lists containing SHA-1 hashes of bridge -fingerprints instead of the original fingerprints are available for -statistical analysis. -</p> - -<blockquote> -<p> -<i>BridgeDB pool assignment file from March 13, 2011:</i> -</p> -<p> -<tt>bridge-pool-assignment 2011-03-13 14:38:03</tt><br> -<tt>00b834117566035736fc6bd4ece950eace8e057a unallocated</tt><br> -<tt>00e923e7a8d87d28954fee7503e480f3a03ce4ee email port=443 flag=stable</tt><br> -<tt>0103bb5b00ad3102b2dbafe9ce709a0a7c1060e4 https ring=2 port=443 flag=stable</tt><br> -<tt>[...]</tt><br> -</p> -</blockquote> - -<p> -The document above shows a BridgeDB pool assignment file -from March 13, 2011. -Every such file begins with a line containing the timestamp when BridgeDB -wrote this file. -Subsequent lines always start with the SHA-1 hash of a bridge fingerprint, -followed by ring, subring, and/or file bucket information. -There are currently three distributor ring types in BridgeDB: -</p> - -<ol> -<li><b>unallocated:</b> These bridges are not distributed by BridgeDB, -but are either reserved for manual distribution or are written to file -buckets for distribution via an external tool. -If a bridge in the <tt>unallocated</tt> ring is assigned to a file bucket, -this is noted by <tt>bucket=$bucketname</tt>.</li> -<li><b>email:</b> These bridges are distributed via an e-mail -autoresponder. Bridges can be assigned to subrings by their OR port or -relay flag which is defined by <tt>port=$port</tt> and/or <tt>flag=$flag</tt>. -</li> -<li><b>https:</b> These bridges are distributed via https server. -There are multiple https rings to further distribute bridges by IP address -ranges, which is denoted by <tt>ring=$ring</tt>. -Bridges in the <tt>https</tt> ring can also be assigned to subrings by -OR port or relay flag which is defined by <tt>port=$port</tt> and/or -<tt>flag=$flag</tt>.</li> -</ol> -<hr> -<br> - -<a name="exitlist"> -<h3><a href="#exitlist" class="anchor">Tor Check exit lists</a></h3> -<br> -<p> -<a href="https://www.torproject.org/tordnsel/dist/">TorDNSEL</a> is an -implementation of the active testing, DNS-based exit list -for Tor exit -nodes. -Tor Check makes the list of known exits and corresponding exit IP -addresses available in a specific format. -The document below shows an entry of the exit list written on -December 28, 2010 at 15:21:44 UTC. -This entry means that the relay with fingerprint <tt>63BA..</tt> which -published a descriptor at 07:35:55 and was contained in a version 2 -network status from 08:10:11 uses two different IP addresses for exiting. -The first address <tt>91.102.152.236</tt> was found in a test performed at -07:10:30. -When looking at the corresponding server descriptor, one finds that this -is also the IP address on which the relay accepts connections from inside -the Tor network. -A second test performed at 10:35:30 reveals that the relay also uses IP -address <tt>91.102.152.227</tt> for exiting. -</p> - -<blockquote> -<p> -<i>Exit list entry written on December 28, 2010 at 15:21:44 UTC:</i> -</p> -<p> -<tt>ExitNode 63BA28370F543D175173E414D5450590D73E22DC</tt><br> -<tt>Published 2010-12-28 07:35:55</tt><br> -<tt>LastStatus 2010-12-28 08:10:11</tt><br> -<tt>ExitAddress 91.102.152.236 2010-12-28 07:10:30</tt><br> -<tt>ExitAddress 91.102.152.227 2010-12-28 10:35:30</tt><br> -</p> -</blockquote> - -</div> -</div> -<div class="bottom" id="bottom"> -<%@ include file="footer.jsp"%> -</div> -</body> -</html> - diff --git a/website/web/WEB-INF/graphs.jsp b/website/web/WEB-INF/graphs.jsp deleted file mode 100644 index 084d085..0000000 --- a/website/web/WEB-INF/graphs.jsp +++ /dev/null @@ -1,34 +0,0 @@ -<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> -<html> -<head> - <title>Tor Metrics: Graphs</title> - <meta http-equiv="content-type" content="text/html; charset=ISO-8859-1"> - <link href="/css/stylesheet-ltr.css" type="text/css" rel="stylesheet"> - <link href="/images/favicon.ico" type="image/x-icon" rel="shortcut icon"> -</head> -<body> - <div class="center"> - <%@ include file="banner.jsp"%> - <div class="main-column"> - <h2>Tor Metrics: Graphs</h2> - <br> - <p>The graphs on this page visualize a small portion of the data - gathered in the Tor Metrics Project. The following graphs are - available:</p> - <ul> - <li>The <a href="network.html">Network page</a> has numerous - statistics on the network of relays and bridges.</li> - <li>The <a href="users.html">Users page</a> attempts to estimate - the number of users in the network.</li> - <li>There are active and passive performance measurements of the - Tor network available on the - <a href="performance.html">Performance page</a>.</li> - </ul> - <br> - </div> - </div> - <div class="bottom" id="bottom"> - <%@ include file="footer.jsp"%> - </div> -</body> -</html> diff --git a/website/web/WEB-INF/papers.jsp b/website/web/WEB-INF/papers.jsp deleted file mode 100644 index a9a8d04..0000000 --- a/website/web/WEB-INF/papers.jsp +++ /dev/null @@ -1,81 +0,0 @@ -<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> -<html> -<head> - <title>Tor Metrics: Papers</title> - <meta http-equiv="content-type" content="text/html; charset=ISO-8859-1"> - <link href="/css/stylesheet-ltr.css" type="text/css" rel="stylesheet"> - <link href="/images/favicon.ico" type="image/x-icon" rel="shortcut icon"> -</head> -<body> - <div class="center"> - <%@ include file="banner.jsp"%> - <div class="main-column"> - <h2>Tor Metrics: Papers</h2> - <br> - <p>The <a href="#papers">papers</a>, - <a href="#techreports">technical reports</a>, and - <a href="#blogposts">blog posts</a> listed on this page originate - from, are based on, or are related to work performed in the Tor - Metrics Project.</p> - <p><font color="red">This page will be removed soon. - Its main purpose was to list metrics-related tech reports, but - those have now moved - <a href="https://research.torproject.org/techreports.html">here</a>. - </font></p> - <br> - <a name="papers"></a> - <h3><a href="#papers" class="anchor">Papers</a></h3> - <br> - These papers summarize some of the results of of the Tor Metrics - Project and have been accepted for publication at academic - conferences or workshops. - <ul> - <li>Karsten Loesing, Steven J. Murdoch, Roger Dingledine. A Case - Study on Measuring Statistical Data in the Tor Anonymity - Network. Workshop on Ethics in - Computer Security Research (WECSR 2010), Tenerife, Spain, - January 2010. (<a href="http://freehaven.net/anonbib/cache/wecsr10measuring-tor.pdf">PDF</a>)</li> - <li>Karsten Loesing. Measuring the Tor Network from Public - Directory Information. 2nd Hot Topics in Privacy Enhancing - Technologies (HotPETs 2009), Seattle, WA, USA, August 2009. - (<a href="https://research.torproject.org/techreports/metrics-2009-08-07.pdf">PDF</a>)</li> - </ul> - <br> - <a name="techreports"></a> - <h3><a href="#techreports" class="anchor">Technical - reports</a></h3> - <br> - <p> - Some of the - <a href="https://research.torproject.org/techreports.html">Tor - Technical Reports</a> have been the first place to - publish novel kinds of statistics on the Tor network. Some, but - not all, of the results contained in those technical reports have - been included in the <a href="#papers">papers</a> above or in the - daily updated <a href="graphs.html">graphs</a>. - </p> - <br> - <a name="blogposts"></a> - <h3><a href="#blogposts" class="anchor">Blog posts</a></h3> - <br> - The following blog posts are either the results of metrics - research or describe new interesting research questions that can - (partly) be answered with metrics data. - <ul> - <li>Research problems: Ten ways to discover Tor bridges - (<a href="https://blog.torproject.org/blog/research-problems-ten-ways-discover-tor-br…">link</a>, - October 31, 2011).</li> - <li>Research problem: better guard rotation parameters - (<a href="https://blog.torproject.org/blog/research-problem-better-guard-rotation-par…">link</a>, - August 20, 2011).</li> - <li>Research problem: measuring the safety of the Tor network - (<a href="https://blog.torproject.org/blog/research-problem-measuring-safety-tor-netw…">link</a>, - February 5, 2011).</li> - </ul> - </div> - </div> - <div class="bottom" id="bottom"> - <%@ include file="footer.jsp"%> - </div> -</body> -</html> diff --git a/website/web/WEB-INF/relay-search.jsp b/website/web/WEB-INF/relay-search.jsp deleted file mode 100644 index 7f15a3c..0000000 --- a/website/web/WEB-INF/relay-search.jsp +++ /dev/null @@ -1,28 +0,0 @@ -<%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core" %> -<%@ taglib prefix="fmt" uri="http://java.sun.com/jsp/jstl/fmt" %> -<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> -<html> -<head> - <title>Tor Metrics: Relay Search</title> - <meta http-equiv="content-type" content="text/html; charset=ISO-8859-1"> - <link href="/css/stylesheet-ltr.css" type="text/css" rel="stylesheet"> - <link href="/images/favicon.ico" type="image/x-icon" rel="shortcut icon"> -</head> -<body> - <div class="center"> - <%@ include file="banner.jsp"%> - <div class="main-column"> - <h2>Tor Metrics: Relay Search</h2> - <p><font color="red">Sorry, this service has been shut down - on June 10, 2014. Possible alternatives are - <a href="https://atlas.torproject.org">Atlas</a> and - <a href="https://globe.torproject.org">Globe</a>. - </font></p> - </div> - </div> - <div class="bottom" id="bottom"> - <%@ include file="footer.jsp"%> - </div> -</body> -</html> - diff --git a/website/web/WEB-INF/research.jsp b/website/web/WEB-INF/research.jsp deleted file mode 100644 index c203619..0000000 --- a/website/web/WEB-INF/research.jsp +++ /dev/null @@ -1,32 +0,0 @@ -<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> -<html> -<head> - <title>Tor Metrics: Research</title> - <meta http-equiv="content-type" content="text/html; charset=ISO-8859-1"> - <link href="/css/stylesheet-ltr.css" type="text/css" rel="stylesheet"> - <link href="/images/favicon.ico" type="image/x-icon" rel="shortcut icon"> -</head> -<body> - <div class="center"> - <%@ include file="banner.jsp"%> - <div class="main-column"> - <h2>Tor Metrics: Research</h2> - <br> - <p>The Tor Metrics Project aims at supporting privacy enhancing - technologies research by making gathered network - <a href="data.html">data</a>, and - <a href="tools.html">tools</a> - for processing these data available to the public. - Some results from analyzing these data can be found in - <a href="https://research.torproject.org/techreports.html">Tor - Tech Reports</a>. If you are - missing anything for your Tor-related research or want to share - your research results with others, please - <a href="mailto:tor-assistants@torproject.org">let us know</a>! - </div> - </div> - <div class="bottom" id="bottom"> - <%@ include file="footer.jsp"%> - </div> -</body> -</html> diff --git a/website/web/WEB-INF/status.jsp b/website/web/WEB-INF/status.jsp deleted file mode 100644 index 5142b7f..0000000 --- a/website/web/WEB-INF/status.jsp +++ /dev/null @@ -1,28 +0,0 @@ -<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> -<html> -<head> - <title>Tor Metrics: Status</title> - <meta http-equiv="content-type" content="text/html; charset=ISO-8859-1"> - <link href="/css/stylesheet-ltr.css" type="text/css" rel="stylesheet"> - <link href="/images/favicon.ico" type="image/x-icon" rel="shortcut icon"> -</head> -<body> - <div class="center"> - <%@ include file="banner.jsp"%> - <div class="main-column"> - <h2>Tor Metrics: Status</h2> - <br> - <p>The network data collected by the Tor Metrics Project can be - used to analyze the Tor network status from a few years ago until - an hour ago. There is currently one application for this data - available on this site: - The - <a href="relay-search.html">Relay Search</a> lets you search the - descriptor archive for a relay. - </div> - </div> - <div class="bottom" id="bottom"> - <%@ include file="footer.jsp"%> - </div> -</body> -</html> diff --git a/website/web/WEB-INF/tools.jsp b/website/web/WEB-INF/tools.jsp deleted file mode 100644 index 0e1a5fc..0000000 --- a/website/web/WEB-INF/tools.jsp +++ /dev/null @@ -1,116 +0,0 @@ -<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> -<html> -<head> - <title>Tor Metrics: Tools</title> - <meta http-equiv="content-type" content="text/html; charset=ISO-8859-1"> - <link href="/css/stylesheet-ltr.css" type="text/css" rel="stylesheet"> - <link href="/images/favicon.ico" type="image/x-icon" rel="shortcut icon"> -</head> -<body> - <div class="center"> - <%@ include file="banner.jsp"%> - <div class="main-column"> - <h2>Tor Metrics: Tools</h2> - <br> - <p><font color="red"><b>Notice:</b> The tools listed on this site - are now available on: - <a href="https://collector.torproject.org/#references">https://collector.torproject.org/#references</a>. - This page will be automatically redirected there after August 3, - 2014.</font> - </p> - <p>The metrics website and related websites depend on a collection - of services that measure, archive, process, and finally present Tor - network data. - This page gives an overview of these services.</p> - <img src="/images/tor-metrics-overview.png"/> - <br> - <a name="measure"></a> - <h3><a href="#measure" class="anchor">Measuring Tor network - data</a></h3> - <br> - <p>Tor network data is measured at various places: - <ul> - <li><a href="https://gitweb.torproject.org/torperf.git">Torperf</a> - is a set of utilities for testing Tor performance from a client - perspective.</li> - <li><a href="https://gitweb.torproject.org/tor.git">tor</a> - relays and bridges gather aggregate usage statistics and publish - descriptors containing data about Tor network structure and - usage.</li> - <li><a href="https://gitweb.torproject.org/tordnsel.git">TorDNSEL</a> - is a Tor DNS-based exit list that runs periodic checks whether - relays use different IP addresses for exiting to the Internet - than they use to register in the Tor network.</li> - <li><a href="https://gitweb.torproject.org/bridgedb.git">BridgeDB</a> - assigns bridges to distributors and gives them out via HTTPS or - email.</li> - </ul> - <br> - <a name="archive"></a> - <h3><a href="#archive" class="anchor">Archiving Tor network - data</a></h3> - <br> - <p>All Tor network data is downloaded, possibly sanitized, and - then archived by a single tool:</p> - <ul> - <li><a href="https://gitweb.torproject.org/metrics-db.git">metrics-db</a> - contains five components for archiving relay descriptors, bridge - descriptors, Torperf results, TorDNSEL exit lists, and BridgeDB - pool assignments.</li> - </ul> - <br> - <a name="process"></a> - <h3><a href="#process" class="anchor">Processing Tor network - data</a></h3> - <br> - <p>In some cases, processing and presenting Tor network data is - separated for maximum flexibility. - In particular, there are currently two main tools that process Tor - network data and write an intermediate data format, but don't - directly present results:</p> - <ul> - <li><a href="https://gitweb.torproject.org/metrics-web.git">metrics-web</a> - is the software behind this website, including aggregation code - that produces statistics files.</li> - <li><a href="https://gitweb.torproject.org/metrics-tasks.git/tree/HEAD:/task-6498">task-6498</a> - is a submodule of metric-web that - aggregates data to visualize fast exits in the Tor network.</li> - <li><a href="https://gitweb.torproject.org/metrics-tasks.git/tree/HEAD:/task-8462">task-8462</a> - is another submodule of metric-web that - estimates daily users from reported directory request - statistics.</li> - <li><a href="https://gitweb.torproject.org/onionoo.git">Onionoo</a> - provides Tor network status information in JSON format via a - RESTful web service.</li> - </ul> - <br> - <a name="present"></a> - <h3><a href="#present" class="anchor">Presenting Tor network - data</a></h3> - <br> - <p>There are a few websites and additional tools presenting Tor - network data: - <ul> - <li><a href="https://gitweb.torproject.org/metrics-web.git">metrics-web</a> - also contains the code that presents aggregate statistics on - this website.</li> - <li><a href="https://gitweb.torproject.org/exonerator.git">ExoneraTor</a> - is a website that tells you whether a given IP address was a Tor - relay.</li> - <li><a href="https://gitweb.torproject.org/atlas.git">Atlas</a> - is a web application to discover relays that uses Onionoo as its - data back-end.</li> - <li><a href="https://gitweb.torproject.org/globe.git">Globe</a> - is a Tor relay and bridge explorer that also uses Onionoo as its - data back-end.</li> - <li><a href="https://gitweb.torproject.org/compass.git">Compass</a> - is a web application that uses Onionoo's data to display - information about fast exits in the Tor network.</li> - </ul> - </div> - </div> - <div class="bottom" id="bottom"> - <%@ include file="footer.jsp"%> - </div> -</body> -</html> diff --git a/website/web/images/tor-metrics-overview.png b/website/web/images/tor-metrics-overview.png deleted file mode 100644 index ee2bc07..0000000 Binary files a/website/web/images/tor-metrics-overview.png and /dev/null differ

1 0

[onionoo/master] Add koninoo to list of Onionoo clients.
by karsten＠torproject.org 14 Aug '14

14 Aug '14

commit 29f613d3f4befdb29a2ea68c544096ca63dd57c8 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Aug 14 16:31:30 2014 +0200 Add koninoo to list of Onionoo clients. --- web/index.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/web/index.html b/web/index.html index a6b99d1..5171b80 100644 --- a/web/index.html +++ b/web/index.html @@ -56,6 +56,8 @@ is an Android app that helps find and explore Tor relays and bridges.</li> <li><a href="https://oniontip.com/">OnionTip</a> uses Onionoo's data to distribute donations to relays that have a bitcoin address in their contact line.</li> +<li><a href="https://savannah.nongnu.org/projects/koninoo/">koninoo</a> is +a simple Java command line interface for querying Onionoo data.</li> </ul> <p>The following library facilitates development of Onionoo clients:</p>

1 0

[check/master] Add Norwegian Nynorsk translation
by arlo＠torproject.org 14 Aug '14

14 Aug '14

commit 830077bc8b156c6cd9ab69efbf588b5c72a534f2 Author: Arlo Breault <arlolra(a)gmail.com> Date: Thu Aug 14 06:07:06 2014 -0700 Add Norwegian Nynorsk translation --- utils.go | 1 + 1 file changed, 1 insertion(+) diff --git a/utils.go b/utils.go index 9ab9289..33fa54c 100644 --- a/utils.go +++ b/utils.go @@ -182,6 +182,7 @@ func GetLocaleList(base string) map[string]string { "nb": "Norsk bokmål", "nl": "Nederlands", "nl_BE": "Vlaams", + "nn": "Norsk nynorsk", "pa": "ਪੰਜਾਬੀ", "pl": "Język polski", "pl_PL": "Język polski (Poland)",

1 0

[check/master] Expose host ip in JSON response
by arlo＠torproject.org 14 Aug '14

14 Aug '14

commit 21e22361dd76cbef423352cb9b84d9b7abe34440 Author: Arlo Breault <arlolra(a)gmail.com> Date: Thu Aug 14 05:54:43 2014 -0700 Expose host ip in JSON response --- handlers.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/handlers.go b/handlers.go index 3b220e0..179aa98 100644 --- a/handlers.go +++ b/handlers.go @@ -95,16 +95,23 @@ func RootHandler(Layout *template.Template, Exits *Exits, domain *gettext.Domain } -type IPResp struct{ IsTor bool } +type IPResp struct { + IsTor bool + IP string +} func APIHandler(Exits *Exits) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - var isTor bool - if host, err := GetHost(r); err == nil { + var ( + err error + isTor bool + host string + ) + if host, err = GetHost(r); err == nil { _, isTor = Exits.IsTor(host) } - ip, _ := json.Marshal(IPResp{isTor}) + ip, _ := json.Marshal(IPResp{isTor, host}) w.Write(ip) } }

1 0

[ooni-probe/master] Fix bug that lead to sniffer not being unregistered properly.
by art＠torproject.org 14 Aug '14

14 Aug '14

commit fd95a46128a3fb4779498df5e615b419d1750fd0 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Aug 14 12:59:52 2014 +0200 Fix bug that lead to sniffer not being unregistered properly. --- ooni/director.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index 43269cc..7496ba8 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -195,10 +195,11 @@ class Director(object): self.successfulMeasurements += 1 measurement.result = result test_name = test_class_name_to_name(measurement.testInstance.name) - sniffer = self.sniffers[test_name] - config.scapyFactory.unRegisterProtocol(sniffer) - sniffer.close() - del self.sniffers[test_name] + if test_name in self.sniffers: + sniffer = self.sniffers[test_name] + config.scapyFactory.unRegisterProtocol(sniffer) + sniffer.close() + del self.sniffers[test_name] return measurement def measurementFailed(self, failure, measurement):

1 0

[ooni-probe/master] Make it possible to run ooniprobe also without setting the control port.
by art＠torproject.org 14 Aug '14

14 Aug '14

commit 8d28d75a63fba098f98e144773399b7c28f063d2 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Aug 14 12:59:10 2014 +0200 Make it possible to run ooniprobe also without setting the control port. Simplify a bit the tests that are done to ensure that tor related settings are consistent. --- ooni/settings.py | 41 +++++++++++++++++------------------------ 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/ooni/settings.py b/ooni/settings.py index 60d0b50..dd31801 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -3,12 +3,11 @@ import sys import yaml import getpass -from twisted.internet import defer, reactor -from twisted.internet.endpoints import TCP4ClientEndpoint +from twisted.internet import defer, reactor, protocol +from twisted.internet.endpoints import TCP4ClientEndpoint, connectProtocol from os.path import abspath, expanduser from scapy.all import get_if_list -import txtorcon from ooni import otime, geoip from ooni.utils import Storage, log @@ -141,33 +140,27 @@ class OConfig(object): Called only when we must start tor by director.start """ incoherent = [] - d = defer.Deferred() - if not self.advanced.start_tor: if self.tor.socks_port is None: incoherent.append('tor:socks_port') - if self.tor.control_port is None: - incoherent.append('tor:control_port') - if self.tor.socks_port is not None and self.tor.control_port is not None: - # Check if tor is listening in these ports - def timeout(): - incoherent.append('tor:control_port') - if not d.called: - d.errback() - - connection = TCP4ClientEndpoint(reactor, "localhost", self.tor.control_port) - timeout_call = reactor.callLater(30, timeout) + else: + socks_port_ep = TCP4ClientEndpoint(reactor, + "localhost", + self.tor.socks_port) try: - d = txtorcon.build_tor_connection(connection) - state = yield d - result = yield state.protocol.get_info("net/listeners/socks") - if result["net/listeners/socks"].split(':')[1] != str(self.tor.socks_port): - incoherent.append('tor:socks_port') + yield connectProtocol(socks_port_ep, protocol.Protocol()) except Exception: incoherent.append('tor:socks_port') - finally: - if timeout_call.active: - timeout_call.cancel() + + if self.tor.control_port is not None: + control_port_ep = TCP4ClientEndpoint(reactor, + "localhost", + self.tor.control_port) + try: + yield connectProtocol(control_port_ep, protocol.Protocol()) + except Exception: + incoherent.append('tor:control_port') + self.log_incoherences(incoherent) def generate_pcap_filename(self, testDetails):

1 0

[ooni-probe/master] Merge pull request #339 from TheTorProject/fix/12706
by art＠torproject.org 14 Aug '14

14 Aug '14

commit 426e9d46d876ede63d26d488698591e006f24a4d Merge: 3f68f15 c6eb5c4 Author: aagbsn <aagbsn(a)extc.org> Date: Wed Aug 13 16:56:29 2014 +0000 Merge pull request #339 from TheTorProject/fix/12706 Fix/12706 ooni/nettest.py | 8 ++++---- ooni/utils/__init__.py | 11 +++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-)

1 0

[ooni-probe/master] Probe Report should not leak the full PATH.
by art＠torproject.org 14 Aug '14

14 Aug '14

commit c6eb5c44ac817ed1704b77ab416b2df833e3daa6 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Aug 7 17:47:34 2014 +0200 Probe Report should not leak the full PATH. This fixes: https://trac.torproject.org/projects/tor/ticket/12706 --- ooni/nettest.py | 8 ++++---- ooni/utils/__init__.py | 11 +++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 4cc453d..54a7d28 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -9,7 +9,7 @@ from twisted.trial.runner import filenameToModule from twisted.python import usage, reflect from ooni.tasks import Measurement -from ooni.utils import log, checkForRoot +from ooni.utils import log, checkForRoot, sanitize_options from ooni.settings import config from ooni import geoip @@ -219,6 +219,7 @@ class NetTestLoader(object): for input_file in self.inputFiles: input_file_hashes.append(input_file['hash']) + options = sanitize_options(self.options) test_details = {'start_time': time.time(), 'probe_asn': config.probe_ip.geodata['asn'], 'probe_cc': config.probe_ip.geodata['countrycode'], @@ -228,9 +229,8 @@ class NetTestLoader(object): 'test_version': self.testVersion, 'software_name': 'ooniprobe', 'software_version': software_version, - 'options': self.options, - 'input_hashes': input_file_hashes, - 'geoip_database_version': geoip.database_version() + 'options': options, + 'input_hashes': input_file_hashes } return test_details diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index e879d7b..ec1b1ce 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -131,3 +131,14 @@ def generate_filename(testDetails, prefix=None, extension=None, filename=None): return final_filename +def sanitize_options(options): + """ + Strips all possible user identifying information from the ooniprobe test + options. + Currently only strips leading directories from filepaths. + """ + sanitized_options = [] + for option in options: + option = os.path.basename(option) + sanitized_options.append(option) + return sanitized_options

1 0

[ooni-probe/master] Handle exceptions better inside of httpt test template
by art＠torproject.org 14 Aug '14

14 Aug '14

commit dc7cc8fea556ed9391147defb7624110ab48133d Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Aug 14 13:26:24 2014 +0200 Handle exceptions better inside of httpt test template --- ooni/nettests/blocking/http_requests.py | 6 ++++-- ooni/templates/httpt.py | 8 ++++++++ ooni/utils/net.py | 1 - 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/ooni/nettests/blocking/http_requests.py b/ooni/nettests/blocking/http_requests.py index dd74b6c..625f34c 100644 --- a/ooni/nettests/blocking/http_requests.py +++ b/ooni/nettests/blocking/http_requests.py @@ -115,13 +115,15 @@ class HTTPRequestsTest(httpt.HTTPTest): def postProcessor(self, measurements): experiment = control = None for status, measurement in measurements: - if 'experiment' in str(measurement.netTestMethod): + net_test_method = measurement.netTestMethod.im_func.func_name + + if net_test_method == "test_get_experiment": if isinstance(measurement.result, failure.Failure): self.report['experiment_failure'] = failureToString( measurement.result) else: experiment = measurement.result - elif 'control' in str(measurement.netTestMethod): + elif net_test_method == "test_get_control": if isinstance(measurement.result, failure.Failure): self.report['control_failure'] = failureToString( measurement.result) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 59b550e..18b0d81 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -154,6 +154,12 @@ class HTTPTest(NetTestCase): response.body = response_body return response + def _processResponseBodyFail(self, failure, request): + failure_string = handleAllFailures(failure) + HTTPTest.addToReport(self, request, response, + failure_string=failure_string) + return response + def processResponseBody(self, body): """ Overwrite this method if you wish to interact with the response body of @@ -242,6 +248,8 @@ class HTTPTest(NetTestCase): response.deliverBody(BodyReceiver(finished, content_length)) finished.addCallback(self._processResponseBody, request, response, body_processor) + finished.addErrback(self._processResponseBodyFail, request, + response) return finished def doRequest(self, url, method="GET", diff --git a/ooni/utils/net.py b/ooni/utils/net.py index 69e39d6..7580616 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -4,7 +4,6 @@ from random import randint from zope.interface import implements from twisted.internet import protocol, defer -from twisted.internet import threads, reactor from twisted.web.iweb import IBodyProducer from ooni.utils import log

1 0

[ooni-probe/master] Don't waste memory on something we don't currently use
by art＠torproject.org 14 Aug '14

14 Aug '14

commit 6218decfac085341e015047ad53d628f7f29faad Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Aug 14 13:27:23 2014 +0200 Don't waste memory on something we don't currently use --- ooni/director.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ooni/director.py b/ooni/director.py index 7496ba8..6ef4143 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -207,7 +207,6 @@ class Director(object): self.totalMeasurementRuntime += measurement.runtime self.failedMeasurements += 1 - self.failures.append((failure, measurement)) measurement.result = failure return measurement

1 0

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits August 2014