[tor-commits] [metrics-web/master] Add .tpf tarballs and specify the new format.

karsten at torproject.org karsten at torproject.org
Wed May 30 08:59:01 UTC 2012


commit 252e368c441cad7c1c9e61755731db15a5c5259a
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Wed May 30 10:58:00 2012 +0200

    Add .tpf tarballs and specify the new format.
---
 .../torproject/ernie/web/ResearchDataServlet.java  |   19 ++
 web/WEB-INF/data.jsp                               |   20 ++-
 web/WEB-INF/formats.jsp                            |  189 +++++++++++++++-----
 3 files changed, 177 insertions(+), 51 deletions(-)

diff --git a/src/org/torproject/ernie/web/ResearchDataServlet.java b/src/org/torproject/ernie/web/ResearchDataServlet.java
index 1df5eee..e8ae79d 100644
--- a/src/org/torproject/ernie/web/ResearchDataServlet.java
+++ b/src/org/torproject/ernie/web/ResearchDataServlet.java
@@ -91,6 +91,8 @@ public class ResearchDataServlet extends HttpServlet {
     SortedMap<Date, String[]> bridgeDescriptors =
         new TreeMap<Date, String[]>(java.util.Collections.reverseOrder());
     String[] relayStatistics = new String[2];
+    SortedMap<Date, String[]> torperfTarballs =
+        new TreeMap<Date, String[]>(java.util.Collections.reverseOrder());
     SortedMap<String, Map<String, String[]>> torperfData =
         new TreeMap<String, Map<String, String[]>>();
     SortedMap<Date, String[]> exitLists =
@@ -172,6 +174,22 @@ public class ResearchDataServlet extends HttpServlet {
         int index = filename.endsWith(".asc") ? 1 : 0;
         relayStatistics[index] = url;
 
+      /* URL contains Torperf tarball. */
+      } else if (filename.startsWith("torperf-20")) {
+        String yearMonth = filename.substring(filename.indexOf("20"));
+        yearMonth = yearMonth.substring(0, 7);
+        Date month = null;
+        try {
+          month = monthFormat.parse(yearMonth);
+        } catch (ParseException e) {
+          /* Ignore this URL. */
+          continue;
+        }
+        if (!torperfTarballs.containsKey(month)) {
+          torperfTarballs.put(month, new String[2]);
+        }
+        torperfTarballs.get(month)[0] = url;
+
       /* URL contains Torperf data file. */
       } else if (filename.endsWith("b.data") ||
           filename.endsWith("b.extradata")) {
@@ -256,6 +274,7 @@ public class ResearchDataServlet extends HttpServlet {
     request.setAttribute("relayStatistics", relayStatistics);
     request.setAttribute("torperfData", torperfData);
     request.setAttribute("exitLists", exitLists);
+    request.setAttribute("torperfTarballs", torperfTarballs);
     request.setAttribute("torperfExperiments", torperfExperiments);
     request.setAttribute("bridgePoolAssignments", bridgePoolAssignments);
     request.getRequestDispatcher("WEB-INF/data.jsp").forward(request,
diff --git a/web/WEB-INF/data.jsp b/web/WEB-INF/data.jsp
index a4e88f2..06176db 100644
--- a/web/WEB-INF/data.jsp
+++ b/web/WEB-INF/data.jsp
@@ -182,12 +182,26 @@
         <p>We are continuously measuring the performance of the Tor
         network by periodically requesting files of different sizes and
         recording the time needed to do so. These measurements take place
-        on moria, siv, and torperf and use an unmodified Tor client.  The
-        files below contain the output of the torperf application and are
-        updated every hour.
+        on moria, siv, and torperf and use an unmodified Tor client.
+        The files below contain the output of the torperf application.
         The data format is described
         <a href="formats.html#torperf">here</a>.</p>
         <table width="100%" border="0" cellpadding="5" cellspacing="0" summary="">
+          <c:forEach var="item" items="${torperfTarballs}" >
+            <fmt:formatDate var="longDate" pattern="MMMM yyyy"
+                            value="${item.key}"/>
+            <tr>
+              <td>
+                <a href="${item.value[0]}">${longDate}</a>
+              </td>
+            </tr>
+          </c:forEach>
+        </table>
+        <br>
+        <p>The output above is the result of combining torperf request
+        data with information about used paths.
+        The raw files are also available below.</p>
+        <table width="100%" border="0" cellpadding="5" cellspacing="0" summary="">
           <c:forEach var="item" items="${torperfData}" >
             <tr>
               <td>${item.key}</td>
diff --git a/web/WEB-INF/formats.jsp b/web/WEB-INF/formats.jsp
index 24647e0..5740c9f 100644
--- a/web/WEB-INF/formats.jsp
+++ b/web/WEB-INF/formats.jsp
@@ -85,7 +85,7 @@ are recent:
 <li><tt><font color="gray">@type bridge-server-descriptor
 1.0</font></tt></li>
 <li><tt><font color="gray">@type bridge-extra-info 1.0</font></tt></li>
-<li><tt><font color="gray">@type torperf 1.0</font></tt></li>
+<li><tt>@type torperf 1.0</tt></li>
 <li><tt><font color="gray">@type bridge-pool-assignment
 1.0</font></tt></li>
 <li><tt><font color="gray">@type gettor 1.0</font></tt></li>
@@ -698,73 +698,166 @@ Torperf can be
 <a href="https://metrics.torproject.org/tools.html">downloaded</a>
 from the metrics
 website.
+A Torperf results file contains a single line per Torperf run with
+<tt>key=value</tt> pairs.
+Such a result line is sufficient to learn about 1) the Tor and Torperf
+configuration, 2) measurement results, and 3) additional information that
+might help explain the results.
+Known keys are explained below.
 </p>
+<ul>
+<li>Configuration
+<ul>
+<li><tt>SOURCE:</tt> Configured name of the data source; required.</li>
+<li><tt>FILESIZE:</tt> Configured file size in bytes; required.</li>
+<li>Other meta data describing the Tor or Torperf configuration, e.g.,
+GUARD for a custom guard choice; optional.</li>
+</ul>
+<li>Measurement results
+<ul>
+<li><tt>START:</tt> Time when the connection process starts;
+required.</li>
+<li><tt>SOCKET:</tt> Time when the socket was created; required.</li>
+<li><tt>CONNECT:</tt> Time when the socket was connected; required.</li>
+<li><tt>NEGOTIATE:</tt> Time when SOCKS 5 authentication methods have been
+negotiated; required.</li>
+<li><tt>REQUEST:</tt> Time when the SOCKS request was sent; required.</li>
+<li><tt>RESPONSE:</tt> Time when the SOCKS response was received;
+required.</li>
+<li><tt>DATAREQUEST:</tt> Time when the HTTP request was written;
+required.</li>
+<li><tt>DATARESPONSE:</tt> Time when the first response was received;
+required.</li>
+<li><tt>DATACOMPLETE:</tt> Time when the payload was complete;
+required.</li>
+<li><tt>WRITEBYTES:</tt> Total number of bytes written; required.</li>
+<li><tt>READBYTES:</tt> Total number of bytes read; required.</li>
+<li><tt>DIDTIMEOUT:</tt> 1 if the request timed out, 0 otherwise;
+optional.</li>
+<li><tt>DATAPERCx:</tt> Time when x% of expected bytes were read for
+x = { 10, 20, 30, 40, 50, 60, 70, 80, 90 }; optional.</li>
+<li>Other measurement results, e.g., START_RENDCIRC, GOT_INTROCIRC, etc.
+for hidden-service measurements; optional.</li>
+</ul>
+<li>Additional information
+<ul>
+<li><tt>LAUNCH:</tt> Time when the circuit was launched; optional.</li>
+<li><tt>USED_AT:</tt> Time when this circuit was used; optional.</li>
+<li><tt>PATH:</tt> List of relays in the circuit, separated by commas;
+optional.</li>
+<li><tt>BUILDTIMES:</tt> List of times when circuit hops were built,
+separated by commas; optional.</li>
+<li><tt>TIMEOUT:</tt> Circuit build timeout that the Tor client used when
+building this circuit; optional.</li>
+<li><tt>QUANTILE:</tt> Circuit build time quantile that the Tor client
+uses to determine its circuit-build timeout; optional.</li>
+<li><tt>CIRC_ID:</tt> Circuit identifier of the circuit used for this
+measurement; optional.</li>
+<li><tt>USED_BY:</tt> Stream identifier of the stream used for this
+measurement; optional.</li>
+<li>Other fields containing additional information; optional.</li>
+</ul>
+</ul>
 
+<blockquote>
 <p>
-Torperf can produce two output files: <tt>.data</tt> and <tt>.extradata</tt>.
-The <tt>.data</tt> file contains timestamps for nine substeps and the byte
-summaries for downloading a test file via Tor.
-The document below shows an example output of a Torperf run.
-The timestamps in the upper part of this output are seconds and
-microseconds since 1970-01-01 00:00:00.000000.
+<i>Torperf <tt>.tpf</tt> output lines for a single request to download a
+50 KiB file (reformatted):</i>
+</p>
+
+<p>
+<tt>BUILDTIMES=1.16901898384,1.86555600166,2.13295292854</tt><br>
+<tt>CIRC_ID=9878</tt><br>
+<tt>CONNECT=1338357901.42</tt><br>
+<tt>DATACOMPLETE=1338357902.91</tt><br>
+<tt>DATAPERC10=1338357902.48</tt><br>
+<tt>DATAPERC20=1338357902.48</tt><br>
+<tt>DATAPERC30=1338357902.61</tt><br>
+<tt>DATAPERC40=1338357902.64</tt><br>
+<tt>DATAPERC50=1338357902.65</tt><br>
+<tt>DATAPERC60=1338357902.74</tt><br>
+<tt>DATAPERC70=1338357902.74</tt><br>
+<tt>DATAPERC80=1338357902.75</tt><br>
+<tt>DATAPERC90=1338357902.79</tt><br>
+<tt>DATAREQUEST=1338357901.83</tt><br>
+<tt>DATARESPONSE=1338357902.25</tt><br>
+<tt>DIDTIMEOUT=0</tt><br>
+<tt>FILESIZE=51200</tt><br>
+<tt>LAUNCH=1338357661.74</tt><br>
+<tt>NEGOTIATE=1338357901.42</tt><br>
+<tt>PATH=$980D326017CEF4CBBF4089FBABE767DC83D059AF,$03545609092A24C71CCAD2F4523F5CCC6714F159,$CAC3CF7154AE9C656C4096DC38B4EFA145905654</tt><br>
+<tt>QUANTILE=0.800000</tt><br>
+<tt>READBYTES=51442</tt><br>
+<tt>REQUEST=1338357901.42</tt><br>
+<tt>RESPONSE=1338357901.83</tt><br>
+<tt>SOCKET=1338357901.42</tt><br>
+<tt>SOURCE=torperf</tt><br>
+<tt>START=1338357901.42</tt><br>
+<tt>TIMEOUT=5049</tt><br>
+<tt>USED_AT=1338357902.91</tt><br>
+<tt>USED_BY=18869</tt><br>
+<tt>WRITEBYTES=75</tt><br>
 </p>
+</blockquote>
+<br>
 
 <p>
+Torperf can produce two output files: <tt>.data</tt> and
+<tt>.extradata</tt>.
+The <tt>.data</tt> file contains timestamps for request substeps and the
+byte summaries for downloading a test file via Tor.
+The document below shows an example output of a Torperf run.
+The timestamps are seconds and microseconds since 1970-01-01
+00:00:00.000000.
 Torperf can be configured to write <tt>.extradata</tt> files by attaching
 a Tor controller and writing certain controller events to disk.
-The content of a <tt>.extradata</tt> line is shown in the lower part of
-the document below.
-The first column indicates if this circuit was actually used to fetch
-the data (<tt>ok</tt>) or if Tor chose a different circuit because this
-circuit was problematic (<tt>error</tt>).
-For every <tt>error</tt> entry there should be a following <tt>ok</tt> entry,
-unless the network of the Torperf instance is dead or the resource is
-unavailable.
-The circuit build completion time in the <tt>.extradata</tt> line is the
-time between Torperf sent a SOCKS request and received a SOCKS response in
-the <tt>.data</tt> file.
-The three or more hops of the circuit are listed by relay fingerprint and
-nickname.
-An <tt>=</tt> sign between the two means that a relay has the <tt>Named</tt>
-flag, whereas the <tt>~</tt> sign means it doesn't.
+The format of a <tt>.extradata</tt> line is similar to the combined format
+as specified above, except that it can only contain "Additional
+information" keywords.
 </p>
 
 <blockquote>
 <p>
-<i>Torperf output lines for a single request to download a 50 KiB
-file (reformatted and annotated with comments):</i>
+<i>Torperf <tt>.data</tt> and <tt>.extradata</tt> output lines for a
+single request to download a 50 KiB file (reformatted and annotated with
+comments):</i>
 </p>
 
 <p>
 <tt># Timestamps and byte summaries contained in .data files:</tt><br>
-<tt>1293543301 762678   # Connection process started</tt><br>
-<tt>1293543301 762704   # After socket is created</tt><br>
-<tt>1293543301 763074   # After socket is connected</tt><br>
-<tt>1293543301 763190   # After authentication methods are negotiated (SOCKS 5 only)</tt><br>
-<tt>1293543301 763816   # After SOCKS request is sent</tt><br>
-<tt>1293543302 901783   # After SOCKS response is received</tt><br>
-<tt>1293543302 901818   # After HTTP request is written</tt><br>
-<tt>1293543304 445732   # After first response is received</tt><br>
-<tt>1293543305 456664   # After payload is complete</tt><br>
+<tt>1338357901 422336   # Connection process started</tt><br>
+<tt>1338357901 422346   # After socket is created</tt><br>
+<tt>1338357901 422521   # After socket is connected</tt><br>
+<tt>1338357901 422604   # After authentication methods are negotiated (SOCKS 5 only)</tt><br>
+<tt>1338357901 423550   # After SOCKS request is sent</tt><br>
+<tt>1338357901 839639   # After SOCKS response is received</tt><br>
+<tt>1338357901 839849   # After HTTP request is written</tt><br>
+<tt>1338357902 258157   # After first response is received</tt><br>
+<tt>1338357902 914263   # After payload is complete</tt><br>
 <tt>75                  # Written bytes</tt><br>
 <tt>51442               # Read bytes</tt><br>
 <tt>0                   # Timeout (optional field)</tt><br>
-<tt>1293543304 546828   # After 10% of expected bytes are read (optional field)</tt><br>
-<tt>1293543304 647924   # After 20% of expected bytes are read (optional field)</tt><br>
-<tt>1293543304 749012   # After 30% of expected bytes are read (optional field)</tt><br>
-<tt>1293543304 850109   # After 40% of expected bytes are read (optional field)</tt><br>
-<tt>1293543304 951201   # After 50% of expected bytes are read (optional field)</tt><br>
-<tt>1293543305 052293   # After 60% of expected bytes are read (optional field)</tt><br>
-<tt>1293543305 153381   # After 70% of expected bytes are read (optional field)</tt><br>
-<tt>1293543305 254482   # After 80% of expected bytes are read (optional field)</tt><br>
-<tt>1293543305 355579   # After 90% of expected bytes are read (optional field)</tt><br>
-<tt></tt><br>
+<tt>1338357902 481591   # After 10% of expected bytes are read (optional field)</tt><br>
+<tt>1338357902 482719   # After 20% of expected bytes are read (optional field)</tt><br>
+<tt>1338357902 613169   # After 30% of expected bytes are read (optional field)</tt><br>
+<tt>1338357902 647108   # After 40% of expected bytes are read (optional field)</tt><br>
+<tt>1338357902 651764   # After 50% of expected bytes are read (optional field)</tt><br>
+<tt>1338357902 743705   # After 60% of expected bytes are read (optional field)</tt><br>
+<tt>1338357902 743876   # After 70% of expected bytes are read (optional field)</tt><br>
+<tt>1338357902 757475   # After 80% of expected bytes are read (optional field)</tt><br>
+<tt>1338357902 795100   # After 90% of expected bytes are read (optional field)</tt><br>
+</p>
+
+<p>
 <tt># Path information contained in .extradata files:</tt><br>
-<tt>ok                  # Status code</tt><br>
-<tt>1293543302          # Circuit build completion time</tt><br>
-<tt>$2F265B37920BDFE474BF795739978EEFA4427510=fejk4        # 1st hop</tt><br>
-<tt>$66CA87E164F1CFCE8C3BB5C095217A28578B8BAF=blutmagie3   # 2nd hop</tt><br>
-<tt>$76997E6557828E8E57F70FDFBD93FB3AA470C620~Amunet8      # 3rd hop</tt><br>
+<tt>CIRC_ID=9878</tt><br>
+<tt>LAUNCH=1338357661.74</tt><br>
+<tt>PATH=$980D326017CEF4CBBF4089FBABE767DC83D059AF,$03545609092A24C71CCAD2F4523F5CCC6714F159,$CAC3CF7154AE9C656C4096DC38B4EFA145905654</tt><br>
+<tt>BUILDTIMES=1.16901898384,1.86555600166,2.13295292854</tt><br>
+<tt>USED_AT=1338357902.91</tt><br>
+<tt>USED_BY=18869</tt><br>
+<tt>TIMEOUT=5049</tt><br>
+<tt>QUANTILE=0.800000</tt><br>
 </p>
 </blockquote>
 <hr>



More information about the tor-commits mailing list