tor-commits

tor-commits@lists.torproject.org

July 2016

21 participants
1272 discussions

[onionoo/master] Implements #19613: changed filestructure, added coverage task,
by karsten＠torproject.org 25 Jul '16

25 Jul '16

commit 254824395abea80787ef06af32ae72d0f288c2d7 Author: iwakeh <iwakeh(a)torproject.org> Date: Thu Jul 21 15:13:16 2016 +0200 Implements #19613: changed filestructure, added coverage task, updated some dependencies to current debian stable. --- build.xml | 160 ++- etc/context.xml.template | 3 - etc/jetty.xml | 45 - etc/logback.xml | 82 -- etc/web.xml.template | 78 -- src/main/resources/context.xml | 3 + src/main/resources/jetty.xml | 45 + src/main/resources/logback.xml | 82 ++ src/main/resources/web.xml | 78 ++ src/main/resources/web/css/style.css | 36 + src/main/resources/web/favicon.ico | Bin 0 -> 1150 bytes src/main/resources/web/index.html | 141 ++ src/main/resources/web/protocol.html | 2338 ++++++++++++++++++++++++++++++++++ src/main/resources/web/robots.txt | 8 + web/css/style.css | 36 - web/favicon.ico | Bin 1150 -> 0 bytes web/index.html | 141 -- web/protocol.html | 2338 ---------------------------------- web/robots.txt | 8 - 19 files changed, 2840 insertions(+), 2782 deletions(-) diff --git a/build.xml b/build.xml index 467c991..7cfd870 100644 --- a/build.xml +++ b/build.xml @@ -5,21 +5,19 @@ value="${onionoo.protocol.version}.0"/> <property name="descriptorversion" value="1.3.0"/> <property name="javasources" value="src/main/java"/> - <property name="tests" value="src/test/java"/> - <property name="generated" value="generated/"/> - <property name="classes" value="classes"/> + <property name="testsources" value="src/test/java"/> + <property name="generated" value="generated"/> + <property name="classes" value="${generated}/classes"/> + <property name="testclasses" value="${generated}/testclasses/"/> + <property name="resources" value="src/main/resources/"/> <property name="testresources" value="src/test/resources/"/> - <property name="dist" value="dist"/> + <property name="testresult" value="${generated}/test-results"/> + <property name="coverageresult" value="${generated}/coverage-report/"/> + <property name="instrument" value="${generated}/instrument/"/> + <property name="cobertura.ser.file" value="${basedir}/cobertura.ser" /> + <property name="dist" value="${generated}/dist"/> <property name="docs" value="${generated}/javadoc/"/> - <property name="libs" value="lib"/> - <property name="config" value="etc"/> - <property name="webxmlfile" value="${config}/web.xml"/> - <property name="contextxmltemplate" - value="${config}/context.xml.template"/> - <property name="contextxml" value="${config}/context.xml"/> - <property name="webxmltemplate" - value="${config}/web.xml.template"/> - <property name="webxml" value="${config}/web.xml"/> + <property name="libs" value="${basedir}/lib"/> <property name="warfile" value="${dist}/onionoo-${release.version}.war"/> <property name="onionoo.jarfile" @@ -28,13 +26,13 @@ <patternset id="runtime" > <include name="descriptor-${descriptorversion}.jar"/> - <include name="commons-codec-1.6.jar"/> - <include name="commons-compress-1.4.1.jar"/> - <include name="commons-lang3-3.1.jar"/> - <include name="gson-2.1.jar"/> - <include name="logback-classic-1.0.4.jar"/> - <include name="logback-core-1.0.4.jar"/> - <include name="slf4j-api-1.6.5.jar"/> + <include name="commons-codec-1.9.jar"/> + <include name="commons-compress-1.9.jar"/> + <include name="commons-lang3-3.3.2.jar"/> + <include name="gson-2.2.4.jar"/> + <include name="logback-classic-1.1.2.jar"/> + <include name="logback-core-1.1.2.jar"/> + <include name="slf4j-api-1.7.7.jar"/> </patternset> <patternset id="web" > @@ -52,7 +50,7 @@ </patternset> <patternset id="test" > - <include name="junit4-4.10.jar"/> + <include name="junit4-4.11.jar"/> <include name="hamcrest-core-1.3.jar"/> </patternset> @@ -62,13 +60,14 @@ <patternset refid="runtime" /> <patternset refid="web" /> </fileset> - <fileset dir="${config}"> + <fileset dir="${resources}"> <include name="logback.xml,jetty.xml"/> </fileset> </path> - <path id="testclasspath"> + <path id="test.classpath"> <path refid="classpath" /> + <pathelement path="${testclasses}"/> <fileset dir="${libs}"> <patternset refid="runtime" /> <patternset refid="test" /> @@ -82,20 +81,42 @@ </fileset> </path> + <path id="cobertura.classpath"> + <fileset dir="${libs}"> + <include name="descriptor-${descriptorversion}.jar"/> + <include name="cobertura-2.1.1.jar" /> + <include name="slf4j-api-1.7.7.jar" /> + <include name="commons-lang3-3.3.2.jar" /> + <include name="asm4-5.0.3.jar" /> + <include name="asm4-util-5.0.3.jar" /> + <include name="asm4-tree-5.0.3.jar" /> + <include name="asm4-commons-5.0.3.jar" /> + <include name="asm4-analysis-5.0.3.jar" /> + <include name="oro-2.0.8.jar" /> + <include name="logback-core-1.1.2.jar" /> + <include name="logback-classic-1.1.2.jar" /> + </fileset> + </path> + + <path id="cobertura.test.classpath"> + <path location="${instrument}" /> + <path refid="test.classpath" /> + <path refid="cobertura.classpath" /> + </path> + <target name="init"> - <copy file="${contextxmltemplate}" tofile="${contextxml}"/> - <copy file="${webxmltemplate}" tofile="${webxml}"/> <mkdir dir="${classes}"/> + <mkdir dir="${testclasses}"/> + <mkdir dir="${testresult}"/> <mkdir dir="${docs}"/> <mkdir dir="${dist}"/> - <mkdir dir="${generated}"/> </target> <target name="clean" > - <delete includeEmptyDirs="true"> - <fileset dir="${classes}" defaultexcludes="false" includes="**" /> - <fileset dir="${dist}" defaultexcludes="false" includes="**" /> + <delete includeEmptyDirs="true" quiet="true" > + <fileset dir="${generated}" defaultexcludes="false" includes="**" /> </delete> + <delete file="${cobertura.ser.file}" quiet="true"/> </target> <target name="compile" @@ -113,6 +134,20 @@ </javac> </target> + <target name="compile-tests" depends="compile"> + <javac destdir="${testclasses}" + srcdir="${testsources}" + source="${source-and-target-java-version}" + target="${source-and-target-java-version}" + debug="true" debuglevel="lines,source" + deprecation="true" + optimize="false" + failonerror="true" + includeantruntime="false"> + <classpath refid="test.classpath"/> + </javac> + </target> + <target name="docs" depends="init"> <javadoc destdir="${docs}" footer="&copy; 2016 The Tor Project" @@ -124,23 +159,12 @@ </javadoc> </target> - <target name="test" depends="compile"> - <javac destdir="${classes}" - srcdir="${tests}" - source="${source-and-target-java-version}" - target="${source-and-target-java-version}" - debug="true" - deprecation="true" - optimize="false" - failonerror="true" - includeantruntime="false"> - <classpath refid="testclasspath" /> - </javac> - <junit fork="true" haltonfailure="true" printsummary="off"> - <classpath refid="testclasspath" /> + <target name="test" depends="compile,compile-tests"> + <junit fork="true" haltonfailure="true" printsummary="on"> + <classpath refid="test.classpath" /> <formatter type="plain" usefile="false"/> <batchtest> - <fileset dir="${classes}" + <fileset dir="${testclasses}" includes="**/*Test.class"/> </batchtest> </junit> @@ -165,8 +189,8 @@ <target name="war" depends="compile"> <war destfile="${warfile}" - webxml="${webxmlfile}"> - <fileset dir="web"/> + webxml="${resources}/web.xml"> + <fileset dir="${resources}/web"/> <restrict> <not> <name name="META-INF/*" /> @@ -183,14 +207,14 @@ <fileset dir="${classes}" includes="**/*" excludes="**/Test*.class"/> - <fileset dir="${config}" includes="jetty.xml" /> - <zipfileset dir="${config}" + <fileset dir="${resources}" includes="jetty.xml" /> + <zipfileset dir="${resources}" prefix="" includes="logback.xml"/> - <zipfileset dir="${config}" + <zipfileset dir="${resources}" prefix="WEB-INF/classes" includes="logback.xml"/> - <metainf dir="${config}" + <metainf dir="${resources}" includes="context.xml"/> <manifest> <attribute name="Created-By" value="The Tor Project" /> @@ -222,7 +246,7 @@ </restrict> <fileset dir="${classes}" excludes="org/torproject/onionoo/server/" /> - <fileset dir="${config}" includes="logback.xml" /> + <fileset dir="${resources}" includes="logback.xml" /> <exclude name="**/Test*.class"/> <manifest> <attribute name="Created-By" value="The Tor Project" /> @@ -239,5 +263,39 @@ <target name="dist" depends="test, war, jar"/> + <taskdef classpathref="cobertura.classpath" resource="tasks.properties" /> + <target name="coverage" depends="compile,compile-tests"> + <copy todir="${instrument}" > + <fileset dir="${classes}" > + <include name="**/*"/> + </fileset> + </copy> + <cobertura-instrument ignoreTrivial="true"> + <fileset dir="${instrument}"> + <include name="**/**/*.class" /> + </fileset> + </cobertura-instrument> + <junit fork="true" haltonfailure="false" printsummary="on"> + <sysproperty key="net.sourceforge.cobertura.datafile" + file="${cobertura.ser.file}" /> + <classpath refid="cobertura.test.classpath" /> + <formatter type="xml" /> + <batchtest toDir="${testresult}" > + <fileset dir="${testclasses}" includes="**/*Test.class" /> + </batchtest> + </junit> + <cobertura-report format="html" destdir="${coverageresult}" > + <fileset dir="${javasources}"> + <include name="**/*.java" /> + </fileset> + </cobertura-report> + <cobertura-check branchrate="0" linerate="0" totallinerate="34" totalbranchrate="30" > + <regex pattern="org.torproject.onionoo.server" branchrate="67" linerate="77"/> + <regex pattern="org.torproject.onionoo.docs" branchrate="19" linerate="29"/> + <regex pattern="org.torproject.onionoo.updater" branchrate="17" linerate="18"/> + <regex pattern="org.torproject.onionoo.writer" branchrate="22" linerate="20"/> + </cobertura-check> + </target> + </project> diff --git a/etc/context.xml.template b/etc/context.xml.template deleted file mode 100644 index 09e5bac..0000000 --- a/etc/context.xml.template +++ /dev/null @@ -1,3 +0,0 @@ -<Context cookies="false"> -</Context> - diff --git a/etc/jetty.xml b/etc/jetty.xml deleted file mode 100644 index c172902..0000000 --- a/etc/jetty.xml +++ /dev/null @@ -1,45 +0,0 @@ -<?xml version="1.0"?> -<!DOCTYPE Configure PUBLIC "-//Jetty//Configure//EN" - "http://www.eclipse.org/jetty/configure.dtd"> - -<Configure id="server" class="org.eclipse.jetty.server.Server" > - <Set name="dumpAfterStart">false</Set> - - <Set name="threadPool"> - <New class="org.eclipse.jetty.util.thread.QueuedThreadPool"> - <Set name="minThreads">25</Set> - <Set name="maxThreads">250</Set> - <Set name="detailedDump">false</Set> - </New> - </Set> - - <New id="webAppContext" class="org.eclipse.jetty.webapp.WebAppContext"> - <Set name="logUrlOnStart">true</Set> - <Set name="war"> - <Call class="java.lang.System" name="getProperty"> - <Arg>java.class.path</Arg> - </Call> - </Set> - </New> - - <Call name="addConnector"> - <Arg> - <New class="org.eclipse.jetty.server.nio.SelectChannelConnector"> - <Set name="port">8080</Set> - </New> - </Arg> - </Call> - - <Set name="handler"> -  - <New class="org.eclipse.jetty.server.handler.ContextHandlerCollection"> - <Call name="addHandler"> - <Arg> - <Ref id="webAppContext"/> - </Arg> - </Call> - </New> - </Set> - -</Configure> - diff --git a/etc/logback.xml b/etc/logback.xml deleted file mode 100644 index 19ae9f7..0000000 --- a/etc/logback.xml +++ /dev/null @@ -1,82 +0,0 @@ -<configuration debug="false"> - -  - <property name="logfile-base" value="${LOGBASE}/onionoo-" /> - -  - <property name="fileall-logname" value="${logfile-base}all" /> - <property name="fileerr-logname" value="${logfile-base}err" /> - <property name="filestatistics-logname" value="${logfile-base}statistics" /> - -  - <property name="utc-date-pattern" value="%date{ISO8601, UTC}" /> - -  - <appender name="FILEALL" class="ch.qos.logback.core.rolling.RollingFileAppender"> - <file>${fileall-logname}.log</file> - <encoder> - <pattern>${utc-date-pattern} %level %logger{20}:%line %msg%n</pattern> - </encoder> - <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> -  - <FileNamePattern>${fileall-logname}.%d{yyyy-MM-dd}.%i.log</FileNamePattern> - <maxHistory>10</maxHistory> - <timeBasedFileNamingAndTriggeringPolicy - class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP"> -  - <maxFileSize>1MB</maxFileSize> - </timeBasedFileNamingAndTriggeringPolicy> - </rollingPolicy> - </appender> - - <appender name="FILEERR" class="ch.qos.logback.core.FileAppender"> - <file>${fileerr-logname}.log</file> - <encoder> - <pattern>${utc-date-pattern} %level %logger{20}:%line %msg%n</pattern> - </encoder> - -  - <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> - <level>ERROR</level> - </filter> - </appender> - - <appender name="FILESTATISTICS" class="ch.qos.logback.core.FileAppender"> - <file>${filestatistics-logname}.log</file> - <encoder> - <pattern>${utc-date-pattern} %msg%n</pattern> - </encoder> - -  - <filter class="ch.qos.logback.classic.filter.LevelFilter"> - <level>INFO</level> - <onMatch>ACCEPT</onMatch> - <onMismatch>DENY</onMismatch> - </filter> - </appender> - -  - <logger name="org.torproject" > - <appender-ref ref="FILEERR" /> - </logger> - - <logger name="org.eclipse" level="INFO" /> - - <logger name="org.torproject.onionoo.cron.Main" > - <appender-ref ref="FILESTATISTICS" /> - </logger> - - <logger name="org.torproject.onionoo.server.PerformanceMetrics" > - <appender-ref ref="FILESTATISTICS" /> - </logger> - - <logger name="statistics" > - <appender-ref ref="FILESTATISTICS" /> - </logger> - - <root level="ALL"> - <appender-ref ref="FILEALL" /> - </root> - -</configuration> - diff --git a/etc/web.xml.template b/etc/web.xml.template deleted file mode 100644 index cefdfae..0000000 --- a/etc/web.xml.template +++ /dev/null @@ -1,78 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> - -<web-app version="2.4" - xmlns="http://java.sun.com/xml/ns/j2ee" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://java.sun.com/xml/ns/j2ee - http://java.sun.com/xml/ns/j2ee/web-app_2_4.xsd" > - - <servlet> - <servlet-name>Resource</servlet-name> - <servlet-class> - org.torproject.onionoo.server.ResourceServlet - </servlet-class> - <init-param> - <param-name>maintenance</param-name> - <param-value>0</param-value> - </init-param> - </servlet> - <servlet-mapping> - <servlet-name>Resource</servlet-name> - <url-pattern>/summary</url-pattern> - </servlet-mapping> - <servlet-mapping> - <servlet-name>Resource</servlet-name> - <url-pattern>/details</url-pattern> - </servlet-mapping> - <servlet-mapping> - <servlet-name>Resource</servlet-name> - <url-pattern>/bandwidth</url-pattern> - </servlet-mapping> - <servlet-mapping> - <servlet-name>Resource</servlet-name> - <url-pattern>/weights</url-pattern> - </servlet-mapping> - <servlet-mapping> - <servlet-name>Resource</servlet-name> - <url-pattern>/clients</url-pattern> - </servlet-mapping> - <servlet-mapping> - <servlet-name>Resource</servlet-name> - <url-pattern>/uptime</url-pattern> - </servlet-mapping> - - <context-param> - <param-name>outDir</param-name> - <param-value>/srv/onionoo.torproject.org/onionoo/out/</param-value> - </context-param> - - <listener> - <listener-class> - org.torproject.onionoo.server.NodeIndexer - </listener-class> - </listener> - - <filter> - <filter-name>GzipFilter</filter-name> - <filter-class>org.eclipse.jetty.servlets.GzipFilter</filter-class> - <init-param> - <param-name>mimeTypes</param-name> - <param-value>text/html,text/xml,text/plain,application/json</param-value> - </init-param> - <init-param> - <param-name>excludedAgents</param-name> - <param-value>gozilla,traviata</param-value> - </init-param> - <init-param> - <param-name>minGzipSize</param-name> - <param-value>2048</param-value> - </init-param> - </filter> - - <filter-mapping> - <filter-name>GzipFilter</filter-name> - <url-pattern>/*</url-pattern> - </filter-mapping> - -</web-app> - diff --git a/src/main/resources/context.xml b/src/main/resources/context.xml new file mode 100644 index 0000000..09e5bac --- /dev/null +++ b/src/main/resources/context.xml @@ -0,0 +1,3 @@ +<Context cookies="false"> +</Context> + diff --git a/src/main/resources/jetty.xml b/src/main/resources/jetty.xml new file mode 100644 index 0000000..c172902 --- /dev/null +++ b/src/main/resources/jetty.xml @@ -0,0 +1,45 @@ +<?xml version="1.0"?> +<!DOCTYPE Configure PUBLIC "-//Jetty//Configure//EN" + "http://www.eclipse.org/jetty/configure.dtd"> + +<Configure id="server" class="org.eclipse.jetty.server.Server" > + <Set name="dumpAfterStart">false</Set> + + <Set name="threadPool"> + <New class="org.eclipse.jetty.util.thread.QueuedThreadPool"> + <Set name="minThreads">25</Set> + <Set name="maxThreads">250</Set> + <Set name="detailedDump">false</Set> + </New> + </Set> + + <New id="webAppContext" class="org.eclipse.jetty.webapp.WebAppContext"> + <Set name="logUrlOnStart">true</Set> + <Set name="war"> + <Call class="java.lang.System" name="getProperty"> + <Arg>java.class.path</Arg> + </Call> + </Set> + </New> + + <Call name="addConnector"> + <Arg> + <New class="org.eclipse.jetty.server.nio.SelectChannelConnector"> + <Set name="port">8080</Set> + </New> + </Arg> + </Call> + + <Set name="handler"> +  + <New class="org.eclipse.jetty.server.handler.ContextHandlerCollection"> + <Call name="addHandler"> + <Arg> + <Ref id="webAppContext"/> + </Arg> + </Call> + </New> + </Set> + +</Configure> + diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml new file mode 100644 index 0000000..19ae9f7 --- /dev/null +++ b/src/main/resources/logback.xml @@ -0,0 +1,82 @@ +<configuration debug="false"> + +  + <property name="logfile-base" value="${LOGBASE}/onionoo-" /> + +  + <property name="fileall-logname" value="${logfile-base}all" /> + <property name="fileerr-logname" value="${logfile-base}err" /> + <property name="filestatistics-logname" value="${logfile-base}statistics" /> + +  + <property name="utc-date-pattern" value="%date{ISO8601, UTC}" /> + +  + <appender name="FILEALL" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${fileall-logname}.log</file> + <encoder> + <pattern>${utc-date-pattern} %level %logger{20}:%line %msg%n</pattern> + </encoder> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> +  + <FileNamePattern>${fileall-logname}.%d{yyyy-MM-dd}.%i.log</FileNamePattern> + <maxHistory>10</maxHistory> + <timeBasedFileNamingAndTriggeringPolicy + class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP"> +  + <maxFileSize>1MB</maxFileSize> + </timeBasedFileNamingAndTriggeringPolicy> + </rollingPolicy> + </appender> + + <appender name="FILEERR" class="ch.qos.logback.core.FileAppender"> + <file>${fileerr-logname}.log</file> + <encoder> + <pattern>${utc-date-pattern} %level %logger{20}:%line %msg%n</pattern> + </encoder> + +  + <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> + <level>ERROR</level> + </filter> + </appender> + + <appender name="FILESTATISTICS" class="ch.qos.logback.core.FileAppender"> + <file>${filestatistics-logname}.log</file> + <encoder> + <pattern>${utc-date-pattern} %msg%n</pattern> + </encoder> + +  + <filter class="ch.qos.logback.classic.filter.LevelFilter"> + <level>INFO</level> + <onMatch>ACCEPT</onMatch> + <onMismatch>DENY</onMismatch> + </filter> + </appender> + +  + <logger name="org.torproject" > + <appender-ref ref="FILEERR" /> + </logger> + + <logger name="org.eclipse" level="INFO" /> + + <logger name="org.torproject.onionoo.cron.Main" > + <appender-ref ref="FILESTATISTICS" /> + </logger> + + <logger name="org.torproject.onionoo.server.PerformanceMetrics" > + <appender-ref ref="FILESTATISTICS" /> + </logger> + + <logger name="statistics" > + <appender-ref ref="FILESTATISTICS" /> + </logger> + + <root level="ALL"> + <appender-ref ref="FILEALL" /> + </root> + +</configuration> + diff --git a/src/main/resources/web.xml b/src/main/resources/web.xml new file mode 100644 index 0000000..cefdfae --- /dev/null +++ b/src/main/resources/web.xml @@ -0,0 +1,78 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<web-app version="2.4" + xmlns="http://java.sun.com/xml/ns/j2ee" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://java.sun.com/xml/ns/j2ee + http://java.sun.com/xml/ns/j2ee/web-app_2_4.xsd" > + + <servlet> + <servlet-name>Resource</servlet-name> + <servlet-class> + org.torproject.onionoo.server.ResourceServlet + </servlet-class> + <init-param> + <param-name>maintenance</param-name> + <param-value>0</param-value> + </init-param> + </servlet> + <servlet-mapping> + <servlet-name>Resource</servlet-name> + <url-pattern>/summary</url-pattern> + </servlet-mapping> + <servlet-mapping> + <servlet-name>Resource</servlet-name> + <url-pattern>/details</url-pattern> + </servlet-mapping> + <servlet-mapping> + <servlet-name>Resource</servlet-name> + <url-pattern>/bandwidth</url-pattern> + </servlet-mapping> + <servlet-mapping> + <servlet-name>Resource</servlet-name> + <url-pattern>/weights</url-pattern> + </servlet-mapping> + <servlet-mapping> + <servlet-name>Resource</servlet-name> + <url-pattern>/clients</url-pattern> + </servlet-mapping> + <servlet-mapping> + <servlet-name>Resource</servlet-name> + <url-pattern>/uptime</url-pattern> + </servlet-mapping> + + <context-param> + <param-name>outDir</param-name> + <param-value>/srv/onionoo.torproject.org/onionoo/out/</param-value> + </context-param> + + <listener> + <listener-class> + org.torproject.onionoo.server.NodeIndexer + </listener-class> + </listener> + + <filter> + <filter-name>GzipFilter</filter-name> + <filter-class>org.eclipse.jetty.servlets.GzipFilter</filter-class> + <init-param> + <param-name>mimeTypes</param-name> + <param-value>text/html,text/xml,text/plain,application/json</param-value> + </init-param> + <init-param> + <param-name>excludedAgents</param-name> + <param-value>gozilla,traviata</param-value> + </init-param> + <init-param> + <param-name>minGzipSize</param-name> + <param-value>2048</param-value> + </init-param> + </filter> + + <filter-mapping> + <filter-name>GzipFilter</filter-name> + <url-pattern>/*</url-pattern> + </filter-mapping> + +</web-app> + diff --git a/src/main/resources/web/css/style.css b/src/main/resources/web/css/style.css new file mode 100644 index 0000000..e3caa4a --- /dev/null +++ b/src/main/resources/web/css/style.css @@ -0,0 +1,36 @@ +body { font-family: "Open Sans","lucida grande","Segoe UI",arial,verdana, + "lucida sans unicode",tahoma,sans-serif; background: #fafafa; + font-size: 13px; line-height: 22px; color: #222; } +h3 { color: #7D4698; position: relative } +a { color: #7D4698; text-decoration: none; font-weight: bold; } +p { margin: 0; padding: 10px; } +a[name] { padding: 0; margin: 0; } +.box { max-width: 850px; width: 100%; margin: 0 auto 30px auto; + padding-bottom: 30px; background: white; border: 1px solid #eee; } +.box > * { margin-left: 30px; margin-right: 30px; } +.box h3 a { visibility: hidden; } +.box:hover h3 a { visibility: visible; } +.api-request { border-bottom: 1px solid #eee; position: relative } +.request-url, .request-type, .request-response { padding: 8px 10px; + vertical-align: middle } +.request-type { color: #57145F; display: inline-block; } +.request-url { color: #333; font-size: 18px; } +.request-response { position: absolute; color: #666; right: 0; } +h3 .request-response { padding: 0 !important; } +.api-urls>li:last-child { border-bottom: 0; } +.required-true, .required-false, .typeof { display: inline-block; + vertical-align: middle; padding: 5px 10px; } +.required-true { color: #1d7508; } +.required-false { color: #aaa; } +.properties { margin-top: 10px; margin-bottom: 10px; + border: 1px solid #eee; } +.properties li { padding: 5px 0; } +.properties li ul { border: 1px solid #eee; margin: 10px 10px 10px 40px; + background: white; } +.properties .properties { margin-left: 10px; } +.properties li:nth-child(even) { background: #fafafa; } +.properties p { padding: 10px 15px; } +.properties b { padding: 5px 10px; display: inline-block; + vertical-align: middle; } +.api-urls{ margin-top: 30px; margin-bottom: 30px; } + diff --git a/src/main/resources/web/favicon.ico b/src/main/resources/web/favicon.ico new file mode 100644 index 0000000..48060b1 Binary files /dev/null and b/src/main/resources/web/favicon.ico differ diff --git a/src/main/resources/web/index.html b/src/main/resources/web/index.html new file mode 100644 index 0000000..c7095ec --- /dev/null +++ b/src/main/resources/web/index.html @@ -0,0 +1,141 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> +<html> +<head> +<title>Onionoo — a Tor network status protocol</title> +<link href="css/style.css" type="text/css" rel="stylesheet"> +<meta http-equiv="content-type" content="text/html; charset=ISO-8859-1"> +<link href="favicon.ico" type="image/x-icon" rel="shortcut icon"> +</head> +<body> + +<div class="box"> + +<h1><a href="index.html">Onionoo</a> —</h1> +<h2>a Tor network status <a href="protocol.html">protocol</a></h2> + +Onionoo is a web-based protocol to learn about currently running +Tor relays and bridges. Onionoo itself was not designed as a service +for human beings—at least not directly. Onionoo provides the +data for other applications and websites which in turn present Tor +network status information to humans. The following Onionoo clients +are currently available: +<ul> +<li><a href="https://atlas.torproject.org/">Atlas</a> is a web +application to discover Tor relays. It provides useful +information on how relays are configured along with graphics about +their past.</li> +<li><a href="https://compass.torproject.org/">Compass</a> +is a Python script and website that extracts consensus weight +information of currently running relays and aggregates weights of +relays running in the same country or same autonomous system.</li> +<li><a href="http://tor2web.org/">Tor2web</a> is a web proxy to Tor +Hidden Services. It uses Onionoo to get the list of currently running +Tor Exits to detect if the client is a Tor user and if so redirect +them to the .onion address.</li> +<li>The <a href="https://nos-oignons.net/Services/index.en.html">Nos +oignons</a> website uses Onionoo to visualize bandwidth histories of +their relays.</li> +<li>The <a href="https://metrics.torproject.org/bubbles.html">metrics +website</a> visualizes diversity of the Tor network using bubble +graphs.</li> +<li><a href="https://github.com/woeisme/torchart">torchart</a> is a PHP +WordPress plugin which draws 3 day traffic charts using pChart currently +used for displaying metric charts on +<a href="http://icetor.is/wordpress/metrics/">icetor.is</a>.</li> +<li><a href="https://github.com/kloesing/challenger">challenger</a> +aggregates graph data from relays participating in +<a href="https://www.eff.org/torchallenge/">EFF's 2014 Tor Challenge</a> +to provide statistics like total bytes transferred.</li> +<li><a href="https://play.google.com/store/apps/details?id=com.networksaremadeofstring.a…">AnOnionooid</a> +is an Android app that helps find and explore Tor relays and bridges.</li> +<li><a href="https://oniontip.com/">OnionTip</a> uses Onionoo's data to +distribute donations to relays that have a bitcoin address in their +contact line.</li> +<li><a href="https://savannah.nongnu.org/projects/koninoo/">koninoo</a> is +a simple Java command line interface for querying Onionoo data.</li> +<li>The <a href="https://duckduckgo.com/">DuckDuckGo</a> search engine +displays Tor node details when being asked for "tor node" followed by a +search term.</li> +<li><a href="https://onionview.com/">OnionView</a> is a simple web service +which uses Tor relay data to plot the location of active Tor nodes onto an +interactive map of the world.</li> +</ul> + +The following library facilitates development of Onionoo clients: +<ul> +<li><a href="https://github.com/duk3luk3/onion-py">OnionPy</a> is a +comprehensive pure-Python (2.7+) Onionoo wrapper with caching +support.</li> +</ul> + +<h2>Developing Onionoo applications</h2> + +The project pages of the Onionoo clients listed above have further +information for contacting the authors and contributing ideas or code. +The authors will be happy to hear your thoughts! + +You don't find your favorite Onionoo client above? Want to +implement your own and tell us to add it to the list? The Onionoo +clients above are backed by a web-based +<a href="protocol.html">protocol</a>, which +facilitates developing new applications displaying Tor status +information. Here are a few ideas for new Onionoo clients: +<ul> +<li>Tor controller extension: Extend +<a href="https://www.torproject.org/projects/vidalia.html.en">Vidalia</a> and/or +<a href="https://www.torproject.org/projects/arm.html.en">arm</a> to look up details for the bridge +that the user is running and display what pool the bridge is contained +in.</li> +<li>Social network site plugin: Add a plugin to the social network +site of your choice to show your friends what Tor relays and bridges +you're running and how that helps users around the world.</li> +<li>Desktop tray icon: Write a tray icon for your favorite desktop +environment that tells you when your relay or bridge is down and that +displays some basic usage statistics.</li> +<li>E-mail notification service: Improve our e-mail notification +service <a href="https://weather.torproject.org/">Weather</a> by +implementing its own relay search or extending it to report when a +bridge drops off the network.</li> +<li>Command-line tool: Implement a command-line tool that quickly +searches a relay or bridge and prints out some status information to +help debug problems.</li> +<li>(Insert your idea here.)</li> +</ul> + +Want to help with developing the Onionoo server that provides +Tor status data, or want to run your own Onionoo server instance? The +Onionoo server is written in Java with a tiny portion of Java +Servlets. Instructions for setting up the Onionoo server to fetch the +required data from the Tor servers is described in the INSTALL file in +the sources. For more details see the +<a href="https://gitweb.torproject.org/onionoo.git">source code</a> and +<a href="https://trac.torproject.org/projects/tor/query?status=!closed&component=Oni…">issue +tracker</a>. + +<h2>Related projects</h2> + +TorStatus is the name of a nowadays +<a href="https://svn.torproject.org/svn/torstatus/trunk/">unmaintained</a> +website that displays Tor relay information similar to +<a href="http://atlas.torproject.org/">Atlas</a>. There are still a +few <a href="http://torstatus.blutmagie.de/">TorStatus</a> +<a href="https://torstatus.rueckgr.at/">websites</a> +<a href="http://tns.hermetix.org/">running</a>. + +There's another project from summer 2011 called TorStatus which is +a <a href="https://gitweb.torproject.org/torstatus.git">rewrite</a> of +the original TorStatus in Python/Django. Unfortunately, it's also +unmaintained. + +Finally, there's the +<a href="https://consensus-health.torproject.org/">consensus-health +page</a> which has the primary purpose of indicating problems with +creating a network status consensus. As a side-effect this page lists +all currently running relays and how the directory authorities voted +on them. + +</div>  + +</body> +</html> + diff --git a/src/main/resources/web/protocol.html b/src/main/resources/web/protocol.html new file mode 100644 index 0000000..c6cc534 --- /dev/null +++ b/src/main/resources/web/protocol.html @@ -0,0 +1,2338 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> +<html> +<head> +<title>Onionoo — protocol overview</title> +<link href="css/style.css" type="text/css" rel="stylesheet"> +<meta http-equiv="content-type" content="text/html; charset=ISO-8859-1"> +<link href="favicon.ico" type="image/x-icon" rel="shortcut icon"> +</head> +<body> + +<div class="box"> + +<h1><a href="index.html">Onionoo</a> —</h1> +<h2>protocol overview</h2> + + <h4>Table of contents</h4> + <ul> + <li><a href="#general">General</a></li> + <li><a href="#methods">Methods</a></li> + <li><a href="#summary">Summary documents</a></li> + <li><a href="#details">Details documents</a></li> + <li><a href="#history">History objects</a></li> + <li><a href="#bandwidth">Bandwidth documents</a></li> + <li><a href="#weights">Weights documents</a></li> + <li><a href="#clients">Clients documents</a></li> + <li><a href="#uptime">Uptime documents</a></li> + <li><a href="#examples">Example usage</a></li> + </ul> + +</div> +<div class="box"> + +<a name="general"></a> +<h3>General <a href="#general">#</a></h3> + + +The Onionoo service is designed as a RESTful web service. +Onionoo clients send HTTP GET requests to the Onionoo server which +responds with JSON-formatted replies. +Onionoo clients and their developers should follow a few rules: + + +<h4>Compression</h4> + +Clients should include an "Accept-Encoding: +gzip" header in their requests and handle gzip-compressed +responses. +Only requests starting at a certain size will be compressed by the +server. + + +<h4>Caching</h4> +Clients should make use of the "Last-Modified" +header of responses and include that timestamp in a +"If-Modified-Since" header of subsequent requests. + + +<h4>Response codes</h4> + +Clients should handle response codes by +distinguishing between client and server errors, and if there's a problem, +informing their users about the kind of problem. +The following response codes are used: + + +<ul class="properties"> + +<li> +200 OK + +The request was processed successfully. + +</li> + +<li> +304 Not Modified + +Server data has not changed since the +"If-Modified-Since" header included in the request. + +</li> + +<li> +400 Bad Request + +The request for a known resource could not be processed because of bad +syntax. +This is most likely a client problem. + +</li> + +<li> +404 Not Available + +The request could not be processed because the requested resource could +not be found. +This is most likely a client problem. + +</li> + +<li> +500 Internal Server Error + +There is an unspecific problem with +the server which the service operator may not yet be aware of. +Please check if there's already a bug report for this problem, and if not, +file one. + +</li> + +<li> +503 Service Unavailable + +The server is temporarily down for +maintenance, or there is a temporary problem with the server that the +service operator is already aware of. +Only file a bug report if this problem persists. + +</li> + +</ul> + +<h4>Protocol versions</h4> + +There are plenty of reasons why we may have +to change the protocol described here. +Clients should be able to handle all valid JSON responses, ignoring +unknown fields and not breaking horribly in case of missing fields. +Protocol changes will be announced here by writing deprecated API parts in +red and new parts in +blue. +Deprecated parts will be removed one month after their announcement. +If you want to be informed of upcoming protocol changes, subscribe to the +<a href="https://lists.torproject.org/cgi-bin/mailman/listinfo/onionoo-announce">onionoo-announce</a> +mailing list. + + +All responses contain a "version" string that +indicates whether clients can parse the document or not. +Version strings consist of a major and a minor version number. +The major version number is raised when previously required fields are +dropped or turned into optional fields, when request parameters or +response documents are removed, or when there are structural changes. +The minor version number is raised when new fields, request parameters, or +response documents are added or optional fields are dropped. +If clients support the same major version given in a response but only a +lower minor version, they should be able to parse the document but may not +understand all fields. +If clients support a lower major version, they should not attempt to parse +a document, because there may be backward-incompatible changes. + +Responses may also contain the optional +"next_major_version_scheduled" field that announces when +the next major version is scheduled to be deployed. +Clients should inform their users that they should upgrade to the next +major protocol version before the provided date. + +The following versions have been used in the past six months or are +scheduled to be deployed in the next months: + +<ul> +<li>1.0: First assigned version number on August 31, +2014.</li> +<li>1.1: Added optional "next_major_version_scheduled" +field on September 16, 2014.</li> +<li>1.2: Added qualified search terms to "search" +parameter on October 17, 2014.</li> +<li>2.0: Extended search parameter to base64-encoded +fingerprints on November 15, 2014.</li> +<li>2.1: Removed optional "advertised_bandwidth_fraction" +field from details documents and optional "advertised_bandwidth" and +"advertised_bandwidth_fraction" fields from weights documents on November +16, 2014.</li> +<li>2.2: Removed optional "pool_assignment" field and +added "transports" field to bridge details documents on December 8, +2014.</li> +<li>2.3: Added optional "flags" field to uptime +documents on March 22, 2015.</li> +<li>2.4: Added optional "effective_family" field to +details documents on July 3, 2015.</li> +<li>2.5: Added optional "measured" field to details +documents on August 13, 2015.</li> +<li>2.6: Added optional "alleged_family" and +"indirect_family" fields and deprecated optional "family" field in details +documents on August 25, 2015.</li> +<li>3.0: Extended search parameter to match any 4 hex +characters of a space-separated fingerprint on November 15, 2015.</li> +<li>3.1: Removed optional "family" field on January 18, +2016.</li> +</ul> + +</div>  + +<div class="box"> + +<a name="methods"></a> +<h3>Methods <a href="#methods">#</a></h3> + + +The following methods each return a single document containing zero or +more objects of relays and/or bridges. +By default, all relays and bridges are included that have been running in +the past week. + + +<ul class="api-urls"> + +<li class="api-request"> +GET +https://onionoo.torproject.org/summary +returns a <a href="#summary">summary +document</a> +</li> + +<li class="api-request"> +GET +https://onionoo.torproject.org/details +returns a <a href="#details">details +document</a> +</li> + +<li class="api-request"> +GET +https://onionoo.torproject.org/bandwidth +returns a <a href="#bandwidth">bandwidth +document</a> +</li> + +<li class="api-request"> +GET +https://onionoo.torproject.org/weights +returns a <a href="#weights">weights +document</a> +</li> + +<li class="api-request"> +GET +https://onionoo.torproject.org/clients +returns a <a href="#clients">clients +document</a> +</li> + +<li class="api-request"> +GET +https://onionoo.torproject.org/uptime +returns a <a href="#uptime">uptime +document</a> +</li> + +</ul> + +<h4>Parameters</h4> + +Each of the methods can be parameterized to select only a subset of relay +and/or bridge documents that are currently running or that have been +running in the past week. +(The fingerprint parameter is special here, because it +allows selecting a specific relay or bridge, regardless of whether it has +been running in the past week.) +If multiple parameters are specified, they are combined using a logical +AND operation, meaning that only the intersection of relays and bridges +matching all parameters is returned. +If the same parameter is specified more than once, only the first +parameter value is considered. + + +<ul class="properties"> + +<li> +type + +Return only relay (parameter value +relay) or only bridge documents (parameter value +bridge). +Parameter values are case-insensitive. + +</li> + +<li> +running + +Return only running (parameter value +true) or only non-running relays and/or bridges +(paramter value +false). +Parameter values are case-insensitive. + +</li> + +<li> +search + +Return only (1) relays with the parameter value matching (part of a) +nickname, (possibly $-prefixed) beginning of a hex-encoded fingerprint, +any 4 hex characters of a space-separated fingerprint, beginning of a +base64-encoded fingerprint without trailing equal signs, or beginning of +an IP address, (2) bridges with (part of a) nickname or (possibly +$-prefixed) beginning of a hashed hex-encoded fingerprint, and (3) relays +and/or bridges matching a given qualified search term. +Searches by relay IP address include all known addresses used for onion +routing and for exiting to the Internet. +Searches for beginnings of IP addresses are performed on textual +representations of canonical IP address forms, so that searches using CIDR +notation or non-canonical forms will return empty results. +Searches are case-insensitive, except for base64-encoded fingerprints. +If multiple search terms are given, separated by spaces, the intersection +of all relays and bridges matching all search terms will be returned. +Complete hex-encoded fingerprints should always be hashed using SHA-1, +regardless of searching for a relay or a bridge, in order to not +accidentally leak non-hashed bridge fingerprints in the URL. +Qualified search terms have the form "key:value" (without double quotes) +with "key" being one of the parameters listed here except for "search", +"fingerprint", "order", "limit", "offset", and "fields", and "value" being +the string that will internally be passed to that parameter. + +</li> + +<li> +lookup + +Return only the relay with the parameter +value matching the fingerprint or the bridge with the parameter value +matching the hashed fingerprint. +Fingerprints should always be hashed using SHA-1, regardless of looking up +a relay or a bridge, in order to not accidentally leak non-hashed bridge +fingerprints in the URL. +Lookups only work for full fingerprints or hashed fingerprints consisting +of 40 hex characters. +Lookups are case-insensitive. + +</li> + +<li> +fingerprint + +Return only the relay with the parameter value matching the fingerprint +or the bridge with the parameter value matching the hashed fingerprint. +Fingerprints must consist of 40 hex characters, case does not matter. +This parameter is quite similar to the lookup parameter +with two exceptions: +(1) the provided relay fingerprint or hashed bridge fingerprint must +not be hashed (again) using SHA-1; +(2) the response will contain any matching relay or bridge regardless of +whether they have been running in the past week. + +</li> + +<li> +country + +Return only relays which are located in the +given country as identified by a two-letter country code. +Filtering by country code is case-insensitive. + +</li> + +<li> +as + +Return only relays which are located in the +given autonomous system (AS) as identified by the AS number (with or +without preceding "AS" part). +Filtering by AS number is case-insensitive. + +</li> + +<li> +flag + +Return only relays which have the +given relay flag assigned by the directory authorities. +Note that if the flag parameter is specified more than once, only the +first parameter value will be considered. +Filtering by flag is case-insensitive. + +</li> + +<li> +first_seen_days + +Return only relays or bridges which +have first been seen during the given range of days ago. +A parameter value "x-y" with x <= y returns relays or bridges that have +first been seen at least x and at most y days ago. +Accepted short forms are "x", "x-", and "-y" which are interpreted as +"x-x", "x-infinity", and "0-y". + +</li> + +<li> +last_seen_days + +Return only relays or bridges which +have last been seen during the given range of days ago. +A parameter value "x-y" with x <= y returns relays or bridges that have +last been seen at least x and at most y days ago. +Accepted short forms are "x", "x-", and "-y" which are interpreted as +"x-x", "x-infinity", and "0-y". +Note that relays and bridges that haven't been running in the past week +are not included in results, so that setting x to 8 or higher will lead to +an empty result set. + +</li> + +<li> +contact + +Return only relays with the parameter value +matching (part of) the contact line. +If the parameter value contains spaces, only relays are returned which +contain all space-separated parts in their contact line. +Only printable ASCII characters are permitted in the parameter value, +some of which need to be percent-encoded (# as %23, % as %25, & as +%26, + as %2B, and / as %2F). +Comparisons are case-insensitive. + +</li> + +<li> +family + +Return only the relay whose fingerprint matches the parameter value and +all relays that this relay has listed in its family by fingerprint and +that in turn have listed this relay in their family by fingerprint. +If relays have listed other relays in their family by nickname, those +family relationships will not be considered, regardless of whether they +have the Named flag or not. +The provided relay fingerprint must consist of 40 hex characters where +case does not matter, and it must not be hashed using SHA-1. +Bridges are not contained in the result, regardless of whether they define +a family. + +</li> + +</ul> + + +Response documents can be reduced in size by requesting only a subset +of contained fields. + + +<ul class="properties"> + +<li> +fields + +Comma-separated list of fields that will be +included in the result. +So far, only top-level fields in relay or bridge objects of details +documents can be specified, e.g., +nickname,hashed_fingerprint. +If the fields parameter is provided, all other fields which are not +contained in the provided list will be removed from the result. +Field names are case-insensitive. + +</li> + +</ul> + + +Relay and/or bridge documents in the response can be ordered and +limited by providing further parameters. +If the same parameter is specified more than once, only the first +parameter value is considered. + + +<ul class="properties"> + +<li> +order + +Re-order results by a comma-separated list +of fields in ascending or descending order. +Results are first ordered by the first list element, then by the second, +and so on. +Possible fields for ordering are: consensus_weight. +Field names are case-insensitive. +Ascending order is the default; descending order is selected by prepending +fields with a minus sign (-). +Relays or bridges which don't have any value for a field to be ordered by +are always appended to the end, regardless or sorting order. +The ordering is defined independent of the requested document type and +does not require the ordering field to be contained in the document. +If no order parameter is given, ordering of results is +undefined. + +</li> + +<li> +offset + +Skip the given number of relays and/or +bridges. +Relays are skipped first, then bridges. +Non-positive offset values are treated as zero and don't +change the +result. + +</li> + +<li> +limit + +Limit result to the given number of +relays and/or bridges. +Relays are kept first, then bridges. +Non-positive limit values are treated as zero and lead +to an empty +result. +When used together with offset, the offsetting step +precedes the +limiting step. + +</li> + +</ul> + +</div>  + +<div class="box"> +<a name="summary"></a> +<h3>Summary documents <a href="#summary">#</a> + +<a href="summary?limit=4">example request</a> + +</h3> + +Summary documents contain short summaries of relays with nicknames, +fingerprints, IP addresses, and running information as well as bridges +with hashed fingerprints and running information. +Summary documents contain the following fields: + +<ul class="properties"> + +<li> +version +<code class="typeof">string</code> +required + +Onionoo protocol version string. + +</li> + +<li> +next_major_version_scheduled +<code class="typeof">string</code> +optional + +UTC date (YYYY-MM-DD) when the next major protocol version is scheduled to +be deployed. +Omitted if no major protocol changes are planned. + +</li> + +<li> +relays_published +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when the last known relay network +status consensus started being valid. +Indicates how recent the relay summaries in this document are. + +</li> + +<li> +relays +<code class="typeof">array of objects</code> +required + +Array of relay summary objects as specified below. + +</li> + +<li> +bridges_published +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when +the last known bridge network status was published. +Indicates how recent the bridge summaries in this document are. + +</li> + +<li> +bridges +<code class="typeof">array of objects</code> +required + +Array of bridge summary objects as specified below. + +</li> + +</ul> + +<h4>Relay summary objects</h4> + + +Relay summary objects contain the following key-value pairs: + + +<ul class="properties"> + +<li> +n +<code class="typeof">string</code> +optional + +Relay nickname consisting of 1–19 alphanumerical characters. +Omitted if the relay nickname is "Unnamed". + +</li> + +<li> +f +<code class="typeof">string</code> +required + +Relay fingerprint consisting of 40 upper-case hexadecimal characters. + +</li> + +<li> +a +<code class="typeof">array of strings</code> +required + +Array of IPv4 or IPv6 addresses where the relay accepts +onion-routing connections or which the relay used to exit to the Internet +in the past 24 hours. +The first address is the primary onion-routing address that the relay used +to register in the network, subsequent addresses are in arbitrary order. +IPv6 hex characters are all lower-case. + +</li> + +<li> +r +<code class="typeof">boolean</code> +required + +Boolean field saying whether this relay was listed as +running in the last relay network status consensus. + +</li> + +</ul> + +<h4>Bridge summary objects</h4> + + +Bridge summary objects contain the following key-value pairs: + + +<ul class="properties"> + +<li> +n +<code class="typeof">string</code> +optional + +Bridge nickname consisting of 1–19 alphanumerical characters. +Omitted if the bridge nickname is "Unnamed". + +</li> + +<li> +h +<code class="typeof">string</code> +required + +SHA-1 hash of the bridge fingerprint consisting of 40 +upper-case hexadecimal characters. + +</li> + +<li> +r +<code class="typeof">boolean</code> +required + +Boolean field saying whether this bridge was listed as +running in the last bridge network status. + +</li> + +</ul> + +</div>  + +<div class="box"> +<a name="details"></a> +<h3>Details documents <a href="#details">#</a> + +<a href="details?limit=4">example request</a> + +</h3> + + +Details documents are based on network statuses published by the Tor +directories, server descriptors published by relays and bridges, and data +published by Tor network services TorDNSEL and BridgeDB. +Details documents use the most recently published data from these sources, +which may lead to contradictions between fields based on different sources +in rare edge cases. +Details documents contain the following fields: + + +<ul class="properties"> + +<li> +version +<code class="typeof">string</code> +required + +Onionoo protocol version string. + +</li> + +<li> +next_major_version_scheduled +<code class="typeof">string</code> +optional + +UTC date (YYYY-MM-DD) when the next major protocol version is scheduled to +be deployed. +Omitted if no major protocol changes are planned. + +</li> + +<li> +relays_published +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when +the last known relay network status consensus started being valid. +Indicates how recent the relay details in this document are. + +</li> + +<li> +relays +<code class="typeof">array of objects</code> +required + +Array of relay details objects as specified below. + +</li> + +<li> +bridges_published +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when +the last known bridge network status was published. +Indicates how recent the bridge details in this document are. + +</li> + +<li> +bridges +<code class="typeof">array of objects</code> +required + +Array of bridge details objects as specified below. + + +</li> + +</ul> + +<h4>Relay details objects</h4> + + +Relay details objects contain the following key-value pairs: + + +<ul class="properties"> + +<li> +nickname +<code class="typeof">string</code> +optional + +Relay nickname consisting of 1–19 alphanumerical characters. +Omitted if the relay nickname is "Unnamed". + +</li> + +<li> +fingerprint +<code class="typeof">string</code> +required + +Relay fingerprint consisting of 40 upper-case +hexadecimal characters. + +</li> + +<li> +or_addresses +<code class="typeof">array of strings</code> +required + +Array of IPv4 or IPv6 addresses and TCP ports +or port lists where the relay accepts onion-routing connections. +The first address is the primary onion-routing address that the relay used +to register in the network, subsequent addresses are in arbitrary order. +IPv6 hex characters are all lower-case. + +</li> + +<li> +exit_addresses +<code class="typeof">array of strings</code> +optional + +Array of IPv4 or IPv6 addresses that the +relay used to exit to the Internet in the past 24 hours. +IPv6 hex characters are all lower-case. +Only those addresses are listed that are different from onion-routing +addresses. +Omitted if array is empty. + +</li> + +<li> +dir_address +<code class="typeof">string</code> +optional + +IPv4 address and TCP port where the relay +accepts directory connections. +Omitted if the relay does not accept directory connections. + +</li> + +<li> +last_seen +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when this +relay was last seen in a network status consensus. + +</li> + +<li> +last_changed_address_or_port +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD +hh:mm:ss) when this relay last stopped announcing an IPv4 or IPv6 address +or TCP port where it previously accepted onion-routing or directory +connections. +This timestamp can serve as indicator whether this relay would be a +suitable fallback directory. + +</li> + +<li> +first_seen +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when this +relay was first seen in a network status consensus. + +</li> + +<li> +running +<code class="typeof">boolean</code> +required + +Boolean field saying whether this relay was listed as +running in the last relay network status consensus. + +</li> + +<li> +hibernating +<code class="typeof">boolean</code> +optional + +Boolean field saying whether this relay indicated that it is hibernating +in its last known server descriptor. +This information may be helpful to decide whether a relay that is not +running anymore has reached its accounting limit and has not dropped out +of the network for another, unknown reason. +Omitted if either the relay is not hibernating, or if no information is +available about the hiberation status of the relay. + +</li> + +<li> +flags +<code class="typeof">array of strings</code> +optional + +Array of relay flags that the directory authorities +assigned to this relay. +Omitted if empty. + +</li> + +<li> +country +<code class="typeof">string</code> +optional + +Two-letter lower-case country code as found in a +GeoIP database by resolving the relay's first onion-routing IP address. +Omitted if the relay IP address could not be found in the GeoIP +database. + +</li> + +<li> +country_name +<code class="typeof">string</code> +optional + +Country name as found in a GeoIP database by +resolving the relay's first onion-routing IP address. +Omitted if the relay IP address could not be found in the GeoIP +database, or if the GeoIP database did not contain a country name. + +</li> + +<li> +region_name +<code class="typeof">string</code> +optional + +Region name as found in a GeoIP database by +resolving the relay's first onion-routing IP address. +Omitted if the relay IP address could not be found in the GeoIP +database, or if the GeoIP database did not contain a region name. + +</li> + +<li> +city_name +<code class="typeof">string</code> +optional + +City name as found in a +GeoIP database by resolving the relay's first onion-routing IP address. +Omitted if the relay IP address could not be found in the GeoIP +database, or if the GeoIP database did not contain a city name. + +</li> + +<li> +latitude +<code class="typeof">number</code> +optional + +Latitude as found in a GeoIP database by resolving +the relay's first onion-routing IP address. +Omitted if the relay IP address could not be found in the GeoIP +database. + +</li> + +<li> +longitude +<code class="typeof">number</code> +optional + +Longitude as found in a GeoIP database by +resolving the relay's first onion-routing IP address. +Omitted if the relay IP address could not be found in the GeoIP +database. + +</li> + +<li> +as_number +<code class="typeof">string</code> +optional + +AS number as found in an AS database by +resolving the relay's first onion-routing IP address. +AS number strings start with "AS", followed directly by the AS number. +Omitted if the relay IP address could not be found in the AS +database. + +</li> + +<li> +as_name +<code class="typeof">string</code> +optional + +AS name as found in an AS database by resolving the +relay's first onion-routing IP address. +Omitted if the relay IP address could not be found in the AS +database. + +</li> + +<li> +consensus_weight +<code class="typeof">number</code> +required + +Weight assigned to this relay by the +directory authorities that clients use in their path selection algorithm. +The unit is arbitrary; currently it's kilobytes per second, but that might +change in the future. + +</li> + +<li> +host_name +<code class="typeof">string</code> +optional + +Host name as found in a reverse DNS lookup of the +relay IP address. +This field is updated at most once in 12 hours, unless the relay IP +address changes. +Omitted if the relay IP address was not looked up or if no lookup request +was successful yet. + +</li> + +<li> +last_restarted +<code class="typeof">string</code> +optional + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when the +relay was last (re-)started. +Missing if router descriptor containing this information cannot be +found. + +</li> + +<li> +bandwidth_rate +<code class="typeof">number</code> +optional + +Average bandwidth +in bytes per second that this relay is willing to sustain over long +periods. +Missing if router descriptor containing this information cannot be +found. + +</li> + +<li> +bandwidth_burst +<code class="typeof">number</code> +optional + +Bandwidth in bytes +per second that this relay is willing to sustain in very short intervals. +Missing if router descriptor containing this information cannot be +found. + +</li> + +<li> +observed_bandwidth +<code class="typeof">number</code> +optional + +Bandwidth +estimate in bytes per second of the capacity this relay can handle. +The relay remembers the maximum bandwidth sustained output over any ten +second period in the past day, and another sustained input. +The "observed_bandwidth" value is the lesser of these two numbers. +Missing if router descriptor containing this information cannot be +found. + +</li> + +<li> +advertised_bandwidth +<code class="typeof">number</code> +optional + +Bandwidth in bytes per second that this +relay is willing and capable to provide. +This bandwidth value is the minimum of bandwidth_rate, +bandwidth_burst, and observed_bandwidth. +Missing if router descriptor containing this information cannot be +found. + +</li> + +<li> +exit_policy +<code class="typeof">array of strings</code> +optional + +Array of exit-policy lines. +Missing if router descriptor containing this information cannot be +found. +May contradict the "exit_policy_summary" field in a rare +edge case: this happens when the relay changes its exit policy after the +directory authorities summarized the previous exit policy. + +</li> + +<li> +exit_policy_summary +<code class="typeof">object</code> +optional + +Summary +version of the relay's exit policy containing a dictionary with either an +"accept" or a "reject" element. +If there is an "accept" ("reject") element, the relay accepts (rejects) +all TCP ports or port ranges in the given list for most IP addresses and +rejects (accepts) all other ports. +May contradict the "exit_policy" field in a rare edge +case: this happens when the relay changes its exit policy after the +directory authorities summarized the previous exit policy. + +</li> + +<li> +exit_policy_v6_summary +<code class="typeof">object</code> +optional + +Summary version of the relay's IPv6 exit policy containing a dictionary +with either an "accept" or a "reject" element. +If there is an "accept" ("reject") element, the relay accepts (rejects) +all TCP ports or port ranges in the given list for most IP addresses and +rejects (accepts) all other ports. +Missing if the relay rejects all connections to IPv6 addresses. +May contradict the "exit_policy_summary" field in a rare +edge case: this happens when the relay changes its exit policy after the +directory authorities summarized the previous exit policy. + +</li> + +<li> +contact +<code class="typeof">string</code> +optional + +Contact address of the relay operator. +Omitted if empty or if descriptor containing this information cannot be +found. + +</li> + +<li> +platform +<code class="typeof">string</code> +optional + +Platform string containing operating system and Tor +version details. +Omitted if empty or if descriptor containing this information cannot be +found. + +</li> + +<li> +recommended_version +<code class="typeof">boolean</code> +optional + +Boolean field saying whether the Tor software version of this relay is +recommended by the directory authorities or not. +Omitted if either the directory authorities did not recommend versions, or +the relay did not report which version it runs. + +</li> + +<li> +family +<code class="typeof">array of strings</code> +optional + +Array of fingerprints or nicknames of relays that this relay considers to +be part of its family. +There are no cross-checks whether the listed relays exist or consider this +relay part of their family, so that the effective family of this relay may +be smaller. +Omitted if empty or if descriptor containing this information cannot be +found. +Deprecated on August 25, 2015, removed on January 18, +2016. + +</li> + +<li> +effective_family +<code class="typeof">array of strings</code> +optional + +Array of $-prefixed fingerprints of relays that are in an effective, +mutual family relationship with this relay. +These relays are part of this relay's family and they consider this relay +to be part of their family. +Omitted if empty or if descriptor containing this information cannot be +found. + +</li> + +<li> +alleged_family +<code class="typeof">array of strings</code> +optional + +Array of $-prefixed fingerprints of relays that are not in an effective, +mutual family relationship with this relay. +These relays are part of this relay's family but they don't consider this +relay to be part of their family. +Omitted if empty or if descriptor containing this information cannot be +found. + +</li> + +<li> +indirect_family +<code class="typeof">array of strings</code> +optional + +Array of $-prefixed fingerprints of relays that are not in an effective, +mutual family relationship with this relay but that can be reached by +following effective, mutual family relationships starting at this relay. +Omitted if empty or if descriptor containing this information cannot be +found. + +</li> + +<li> +consensus_weight_fraction +<code class="typeof">number</code> +optional + +Fraction of this relay's consensus weight compared to the sum of all +consensus weights in the network. +This fraction is a very rough approximation of the probability of this +relay to be selected by clients. +Omitted if the relay is not running. + +</li> + +<li> +guard_probability +<code class="typeof">number</code> +optional + +Probability of this relay to be selected for the guard position. +This probability is calculated based on consensus weights, relay flags, +and bandwidth weights in the consensus. +Path selection depends on more factors, so that this probability can only +be an approximation. +Omitted if the relay is not running, or the consensus does not contain +bandwidth weights. + +</li> + +<li> +middle_probability +<code class="typeof">number</code> +optional + +Probability of this relay to be selected for the middle position. +This probability is calculated based on consensus weights, relay flags, +and bandwidth weights in the consensus. +Path selection depends on more factors, so that this probability can only +be an approximation. +Omitted if the relay is not running, or the consensus does not contain +bandwidth weights. + +</li> + +<li> +exit_probability +<code class="typeof">number</code> +optional + +Probability of this relay to be selected for the exit position. +This probability is calculated based on consensus weights, relay flags, +and bandwidth weights in the consensus. +Path selection depends on more factors, so that this probability can only +be an approximation. +Omitted if the relay is not running, or the consensus does not contain +bandwidth weights. + +</li> + +<li> +measured +<code class="typeof">boolean</code> +optional + +Boolean field saying whether the consensus weight of this relay is based +on a threshold of 3 or more measurements by Tor bandwidth authorities. +Omitted if the network status consensus containing this relay does not +contain measurement information. + +</li> + +</ul> + +<h4>Bridge details objects</h4> + + +Bridge details objects contain the following key-value pairs: + + +<ul class="properties"> + +<li> +nickname +<code class="typeof">string</code> +optional + +Bridge nickname consisting of 1–19 +alphanumerical characters. +Omitted if the bridge nickname is "Unnamed". + +</li> + +<li> +hashed_fingerprint +<code class="typeof">string</code> +required + +SHA-1 hash of the bridge fingerprint +consisting of 40 upper-case hexadecimal characters. + +</li> + +<li> +or_addresses +<code class="typeof">array of strings</code> +required + +Array of sanitized IPv4 or IPv6 addresses and +TCP ports or port lists where the bridge accepts onion-routing +connections. +The first address is the primary onion-routing address that the bridge +used to register in the network, subsequent addresses are in arbitrary +order. +IPv6 hex characters are all lower-case. +Sanitized IP addresses are always in 10/8 or +[fd9f:2e19:3bcf/48] IP networks and are only useful to +learn which +IP version the bridge uses and to detect whether the bridge changed its +address. +Sanitized IP addresses always change on the 1st of every month at 00:00:00 +UTC, regardless of the bridge actually changing its IP address. +TCP ports are not sanitized. + +</li> + +<li> +last_seen +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when this +bridge was last seen in a bridge network status. + +</li> + +<li> +first_seen +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when this +bridge was first seen in a bridge network status. + +</li> + +<li> +running +<code class="typeof">boolean</code> +required + +Boolean field saying whether this bridge was listed +as running in the last bridge network status. + +</li> + +<li> +flags +<code class="typeof">array of strings</code> +optional + +Array of relay flags that the bridge authority +assigned to this bridge. +Omitted if empty. + +</li> + +<li> +last_restarted +<code class="typeof">string</code> +optional + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when the +bridge was last (re-)started. +Missing if router descriptor containing this information cannot be +found. + +</li> + +<li> +advertised_bandwidth +<code class="typeof">number</code> +optional + +Bandwidth in bytes per second that this +bridge is willing and capable to provide. +This bandwidth value is the minimum of bandwidth_rate, +bandwidth_burst, and observed_bandwidth. +Missing if router descriptor containing this information cannot be +found. + +</li> + +<li> +platform +<code class="typeof">string</code> +optional + +Platform string containing operating system and Tor +version details. +Omitted if not provided by the bridge or if descriptor containing this +information cannot be found. + +</li> + +<li> +transports +<code class="typeof">array of strings</code> +optional + +Array of (pluggable) transport names supported by this bridge. + +</li> + +</ul> + +</div>  + +<div class="box"> +<a name="history"></a> +<h3>History objects <a href="#history">#</a></h3> + + +History objects are no documents by themselves, but are contained in +subsequent documents. + + +<h4>Graph history objects</h4> + + +Graph history objects contain the following fields: + + +<ul class="properties"> + +<li> +first +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) of the first data point, or more +specifically the interval midpoint of the first interval. + +</li> + +<li> +last +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) of the last data point, or more +specifically the interval midpoint of the last interval. + +</li> + +<li> +interval +<code class="typeof">number</code> +required + +Time interval between two data points in seconds. + +</li> + +<li> +factor +<code class="typeof">number</code> +required + +Factor by which subsequent data values need to be multiplied to obtain +original values. +The idea is to reduce document size while still providing sufficient +detail for very different data scales. + +</li> + +<li> +count +<code class="typeof">number</code> +optional + +Number of provided data points, included mostly for debugging purposes. +Can also be derived from the number of elements in the subsequent array. + +</li> + +<li> +values +<code class="typeof">array of numbers</code> +required + +Array of normalized values between 0 and 999. +May contain null values. +Contains at least two subsequent non-null values to enable drawing of line +graphs. + +</li> + +</ul> + +</div>  + +<div class="box"> +<a name="bandwidth"></a> +<h3>Bandwidth documents <a href="#bandwidth">#</a> + +<a href="bandwidth?limit=4">example request</a> + +</h3> + + +Bandwidth documents contain aggregate statistics of a relay's or +bridge's consumed bandwidth for different time intervals. +Bandwidth documents are only updated when a relay or bridge publishes a +new server descriptor, which may take up to 18 hours during normal +operation. +Bandwidth documents contain the following fields: + + +<ul class="properties"> + +<li> +version +<code class="typeof">string</code> +required + +Onionoo protocol version string. + +</li> + +<li> +next_major_version_scheduled +<code class="typeof">string</code> +optional + +UTC date (YYYY-MM-DD) when the next major protocol version is scheduled to +be deployed. +Omitted if no major protocol changes are planned. + +</li> + +<li> +relays_published +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when +the last known relay network status consensus started being valid. +Indicates how recent the relay bandwidth documents in this document are. + +</li> + +<li> +relays +<code class="typeof">array of objects</code> +required + +Array of relay bandwidth objects as specified below. + +</li> + +<li> +bridges_published +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when +the last known bridge network status was published. +Indicates how recent the bridge bandwidth documents in this document are. + +</li> + +<li> +bridges +<code class="typeof">array of objects</code> +required + +Array of bridge bandwidth objects as specified below. + +</li> + +</ul> + +<h4>Relay bandwidth objects</h4> + + +Relay bandwidth objects contain the following key-value pairs: + + +<ul class="properties"> + +<li> +fingerprint +<code class="typeof">string</code> +required + +Relay fingerprint consisting of 40 upper-case +hexadecimal characters. + +</li> + +<li> +write_history +<code class="typeof">object</code> +optional + +Object containing graph history objects with written bytes for different +time periods. +Keys are string representation of the time period covered by the graph +history object. +Keys are fixed strings "3_days", +"1_week", "1_month", +"3_months", "1_year", and +"5_years". +Keys refer to the last known bandwidth history of a relay, not to the time +when the bandwidth document was published. +A graph history object is only contained if the time period it covers is +not already contained in another graph history object with shorter time +period and higher data resolution. +Similarly, a graph history object is excluded if the relay did not provide +bandwidth histories on the required level of detail. +The unit is bytes per second. +Contained graph history objects may contain null values if the relay did +not provide any bandwidth data or only data for less than 20% of a given +time period. + +</li> + +<li> +read_history +<code class="typeof">object</code> +optional + +Object containing graph history objects with read bytes for different time +periods. +The specification of graph history objects is similar to those in the +write_history field. + +</li> + +</ul> + +<h4>Bridge bandwidth objects</h4> + + +Bridge bandwidth objects contain the following key-value pairs: + + +<ul class="properties"> + +<li> +fingerprint +<code class="typeof">string</code> +required + +SHA-1 hash of the bridge fingerprint consisting +of 40 upper-case hexadecimal characters. + +</li> + +<li> +write_history +<code class="typeof">object</code> +optional + +Object containing graph history objects with written bytes for different +time periods. +The specification of graph history objects is similar to those in the +write_history field of relays. + +</li> + +<li> +read_history +<code class="typeof">object</code> +optional + +Object containing graph history objects with read bytes for different time +periods. +The specification of graph history objects is similar to those in the +write_history field of relays. + +</li> + +</ul> + +</div>  + +<div class="box"> +<a name="weights"></a> +<h3>Weights documents <a href="#weights">#</a> + +<a href="weights?limit=4">example request</a> + +</h3> + + +Weights documents contain aggregate statistics of a relay's probability +to be selected by clients for building paths. +Weights documents contain different time intervals and are available for +relays only. +Weights documents contain the following fields: + + +<ul class="properties"> + +<li> +version +<code class="typeof">string</code> +required + +Onionoo protocol version string. + +</li> + +<li> +next_major_version_scheduled +<code class="typeof">string</code> +optional + +UTC date (YYYY-MM-DD) when the next major protocol version is scheduled to +be deployed. +Omitted if no major protocol changes are planned. + +</li> + +<li> +relays_published +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when +the last known relay network status consensus started being valid. +Indicates how recent the relay weights documents in this document are. + +</li> + +<li> +relays +<code class="typeof">array of objects</code> +required + +Array of relay weights objects as specified below. + +</li> + +<li> +bridges_published +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when +the last known bridge network status was published. +Only included for compatibility reasons with the other document types. + +</li> + +<li> +bridges +<code class="typeof">array of objects</code> +required + +Empty array of objects that would represent bridge weights documents. +Only included for compatibility reasons with the other document types. + +</li> + +</ul> + +<h4>Relay weights objects</h4> + + +Relay weights objects contain the following key-value pairs: + + +<ul class="properties"> + +<li> +fingerprint +<code class="typeof">string</code> +required + +Relay fingerprint consisting of 40 upper-case +hexadecimal characters. + +</li> + +<li> +consensus_weight_fraction +<code class="typeof">object</code> +optional + +History object containing the +fraction of this relay's consensus weight compared to the sum of all +consensus weights in the network. +This fraction is a very rough approximation of the probability of this +relay to be selected by clients. +Keys are string representation of the time period covered by the graph +history object. +Keys are fixed strings "1_week", +"1_month", "3_months", +"1_year", and "5_years". +Keys refer to the last known weights history of a relay, not to the time +when the weights document was published. +A graph history object is only contained if the time period it covers is +not already contained in another graph history object with shorter time +period and higher data resolution. +The unit is path-selection probability. +Contained graph history objects may contain null values if the relay was +running less than 20% of a given time period. + +</li> + +<li> +guard_probability +<code class="typeof">object</code> +optional + +History object containing the probability +of this relay to be selected for the guard position. +This probability is calculated based on consensus weights, relay flags, +and bandwidth weights in the consensus. +Path selection depends on more factors, so that this probability can only +be an approximation. +The specification of this history object is similar to that in the +consensus_weight_fraction field above. + +</li> + +<li> +middle_probability +<code class="typeof">object</code> +optional + +History object containing the probability +of this relay to be selected for the middle position. +This probability is calculated based on consensus weights, relay flags, +and bandwidth weights in the consensus. +Path selection depends on more factors, so that this probability can only +be an approximation. +The specification of this history object is similar to that in the +consensus_weight_fraction field above. + +</li> + +<li> +exit_probability +<code class="typeof">object</code> +optional + +History object containing the probability +of this relay to be selected for the exit position. +This probability is calculated based on consensus weights, relay flags, +and bandwidth weights in the consensus. +Path selection depends on more factors, so that this probability can only +be an approximation. +The specification of this history object is similar to that in the +consensus_weight_fraction field above. + +</li> + +<li> +consensus_weight +<code class="typeof">object</code> +optional + +History object containing the absolute consensus weight of this relay. +The specification of this history object is similar to that in the +consensus_weight_fraction field above. + +</li> + +</ul> + +</div>  + +<div class="box"> +<a name="clients"></a> +<h3>Clients documents <a href="#clients">#</a> + +<a href="clients?limit=4">example request</a> + +</h3> + + +Clients documents contain estimates of the average number of clients +connecting to a bridge every day. +There are no clients documents available for relays, just for bridges. +Clients documents contain different time intervals and are available for +bridges only. +Clients documents contain the following fields: + + +<ul class="properties"> + +<li> +version +<code class="typeof">string</code> +required + +Onionoo protocol version string. + +</li> + +<li> +next_major_version_scheduled +<code class="typeof">string</code> +optional + +UTC date (YYYY-MM-DD) when the next major protocol version is scheduled to +be deployed. +Omitted if no major protocol changes are planned. + +</li> + +<li> +relays_published +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when +the last known relay network status consensus started being valid. +Only included for compatibility reasons with the other document types. + +</li> + +<li> +relays +<code class="typeof">array of objects</code> +required + +Empty array of objects that would represent relay clients documents. +Only included for compatibility reasons with the other document types. + +</li> + +<li> +bridges_published +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when +the last known bridge network status was published. +Indicates how recent the bridge clients documents in this document are. + +</li> + +<li> +bridges +<code class="typeof">array of objects</code> +required + +Array of bridge clients objects as specified below. + + +</li> + +</ul> + +<h4>Bridge clients objects</h4> + + +Bridge clients objects contain the following key-value pairs: + + +<ul class="properties"> + +<li> +fingerprint +<code class="typeof">string</code> +required + +SHA-1 hash of the bridge fingerprint consisting +of 40 upper-case hexadecimal characters. + +</li> + +<li> +average_clients +<code class="typeof">object</code> +optional + +Object containing graph history objects with the average number of clients +connecting to this bridge. +Keys are string representation of the time period covered by the graph +history object. +Keys are fixed strings "1_week", +"1_month", "3_months", +"1_year", and "5_years". +Keys refer to the last known clients history of a bridge, not to the time +when the clients document was published. +A graph history object is only contained if the time period it covers +is not already contained in another clients graph object with shorter +time period and higher data resolution. +The unit is number of clients. +Contained graph history objects may contain null values if the bridge did +not report client statistics for at least 50% of a given time period. +Each graph history object contains the following additional key-value +pairs: + + +<ul class="properties"> + +<li> +countries +<code class="typeof">object</code> +optional + +Object containing fractions of clients by country in the considered time +period. +Keys are two-letter lower-case country codes as found in a GeoIP database. +Values are numbers between 0 and 1 standing for the fraction of clients by +country. +A country is only included if at least 1% of clients came from this +country. +Omitted if the bridge did not report client statistics by country. +BETA: This field breaks compatibility +with the history objects contained in other documents pretty badly. +It might be removed in the future without notice. + +</li> + +<li> +transports +<code class="typeof">object</code> +optional + +Object containing fractions of clients by transport in the considered time +period. +Keys are transport names, or "<OR>" for the default +onion-routing transport protocol. +Values are numbers between 0 and 1 standing for the fraction of clients by +transport. +Omitted if the bridge did not report client statistics by transport. +BETA: This field breaks compatibility +with the history objects contained in other documents pretty badly. +It might be removed in the future without notice. + +</li> + +<li> +versions +<code class="typeof">object</code> +optional + +Object containing fractions of clients by IP version in the considered +time period. +Keys are either "v4" for IPv4 or "v6" +for IPv6. +Values are numbers between 0 and 1 standing for the fraction of clients by +version. +Omitted if the bridge did not report client statistics by IP version. +BETA: This field breaks compatibility +with the history objects contained in other documents pretty badly. +It might be removed in the future without notice. + +</li> + +</ul> + +</li> + +</ul> + +</div>  + +<div class="box"> +<a name="uptime"></a> +<h3>Uptime documents <a href="#uptime">#</a> + +<a href="uptime?limit=4">example request</a> + +</h3> + + +Uptime documents contain fractional uptimes of relays and bridges. +Uptime documents contain different time intervals and are available for +relays and bridges. +Uptime documents contain the following fields: + + +<ul class="properties"> + +<li> +version +<code class="typeof">string</code> +required + +Onionoo protocol version string. + +</li> + +<li> +next_major_version_scheduled +<code class="typeof">string</code> +optional + +UTC date (YYYY-MM-DD) when the next major protocol version is scheduled to +be deployed. +Omitted if no major protocol changes are planned. + +</li> + +<li> +relays_published +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when +the last known relay network status consensus started being valid. +Indicates how recent the relay uptime documents in this document are. + +</li> + +<li> +relays +<code class="typeof">array of objects</code> +required + +Array of relay uptime objects as specified below. + +</li> + +<li> +bridges_published +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when +the last known bridge network status was published. +Indicates how recent the bridge uptime documents in this document are. + +</li> + +<li> +bridges +<code class="typeof">array of objects</code> +required + +Array of bridge uptime objects as specified below. + +</li> + +</ul> + +<h4>Relay uptime objects</h4> + + +Relay uptime objects contain the following key-value pairs: + + +<ul class="properties"> + +<li> +fingerprint +<code class="typeof">string</code> +required + +Relay fingerprint consisting of 40 upper-case +hexadecimal characters. + +</li> + +<li> +uptime +<code class="typeof">object</code> +optional + +Object containing graph history objects with the fractional uptime of this +relay. +Keys are string representation of the time period covered by the graph +history object. +Keys are fixed strings "1_week", +"1_month", "3_months", +"1_year", and "5_years". +Keys refer to the last known uptime history of a relay, not to the time +when the uptime document was published. +A graph history object is only contained if the time period it covers is +not already contained in another graph history object with shorter time +period and higher data resolution. +The unit is fractional uptime from 0 to 1. +Contained graph history objects may contain null values if less than 20% +of network statuses have been processed for a given time period. + +</li> + +<li> +flags +<code class="typeof">object</code> +optional + +Object containing fractional times of this relay having relay flags +assigned. +Keys are flag names like "Running" or +"Exit", values are objects similar to the +uptime field above, again with keys like +"1_week" etc. +If a relay never had a given relay flag assigned, no object is included +for that flag. + +</li> + +</ul> + +<h4>Bridge uptime objects</h4> + + +Bridge uptime objects contain the following key-value pairs: + + +<ul class="properties"> + +<li> +fingerprint +<code class="typeof">string</code> +required + +SHA-1 hash of the bridge fingerprint consisting +of 40 upper-case hexadecimal characters. + +</li> + +<li> +uptime +<code class="typeof">object</code> +optional + +Object containing uptime history objects for different time periods. +The specification of uptime history objects is similar to those in the +uptime field of relays. + +</li> + +</ul> + +</div>  + +<div class="box"> +<a name="examples"></a> +<h3>Example usage <a href="#examples">#</a> +</h3> + + +The following examples illustrate how to build requests for some trivial +and some more complex use cases. +While Onionoo is designed mainly for developers and not end users, there +may be cases when it's easier to quickly write a specific query Onionoo +rather than to find an Onionoo client that provides the desired +information. + + +<pre>https://onionoo.torproject.org/summary?limit=4</pre> + + +This first query returns the first four summary documents that Onionoo can +find. +The <code>limit</code> parameter should always be used while developing +new queries to avoid downloading huge responses. + + +<pre>https://onionoo.torproject.org/summary?limit=4&search=moria</pre> + + +The second query restricts results to relays and bridges containing the +string "moria" in one of a few searched fields. + + +<pre>https://onionoo.torproject.org/details?limit=4&search=moria</pre> + + +The third query switches from the short summary documents to the longer +details documents containing, well, more details. + + +<pre>https://onionoo.torproject.org/details?limit=4&search=moria&fields=nickname</pre> + + +The fourth query adds the <code>fields</code> parameter which removes all +fields except the specified ones from the result. +This parameter is only implemented for details documents. + + +<pre>https://onionoo.torproject.org/details?limit=4&search=moria&fields=nickname…</pre> + + +The fifth query sorts results by relay consensus weight from largest to +smallest. + + + +Obviously, this query can be made even more complex by adding more +parameters, and in some cases this is necessary and useful. +Please refer to the protocol specification for details. + + +</div>  + +</body> +</html> + diff --git a/src/main/resources/web/robots.txt b/src/main/resources/web/robots.txt new file mode 100644 index 0000000..9b19f8f --- /dev/null +++ b/src/main/resources/web/robots.txt @@ -0,0 +1,8 @@ +User-agent: * +Disallow: /summary +Disallow: /details +Disallow: /bandwidth +Disallow: /weights +Disallow: /clients +Disallow: /uptime + diff --git a/web/css/style.css b/web/css/style.css deleted file mode 100644 index e3caa4a..0000000 --- a/web/css/style.css +++ /dev/null @@ -1,36 +0,0 @@ -body { font-family: "Open Sans","lucida grande","Segoe UI",arial,verdana, - "lucida sans unicode",tahoma,sans-serif; background: #fafafa; - font-size: 13px; line-height: 22px; color: #222; } -h3 { color: #7D4698; position: relative } -a { color: #7D4698; text-decoration: none; font-weight: bold; } -p { margin: 0; padding: 10px; } -a[name] { padding: 0; margin: 0; } -.box { max-width: 850px; width: 100%; margin: 0 auto 30px auto; - padding-bottom: 30px; background: white; border: 1px solid #eee; } -.box > * { margin-left: 30px; margin-right: 30px; } -.box h3 a { visibility: hidden; } -.box:hover h3 a { visibility: visible; } -.api-request { border-bottom: 1px solid #eee; position: relative } -.request-url, .request-type, .request-response { padding: 8px 10px; - vertical-align: middle } -.request-type { color: #57145F; display: inline-block; } -.request-url { color: #333; font-size: 18px; } -.request-response { position: absolute; color: #666; right: 0; } -h3 .request-response { padding: 0 !important; } -.api-urls>li:last-child { border-bottom: 0; } -.required-true, .required-false, .typeof { display: inline-block; - vertical-align: middle; padding: 5px 10px; } -.required-true { color: #1d7508; } -.required-false { color: #aaa; } -.properties { margin-top: 10px; margin-bottom: 10px; - border: 1px solid #eee; } -.properties li { padding: 5px 0; } -.properties li ul { border: 1px solid #eee; margin: 10px 10px 10px 40px; - background: white; } -.properties .properties { margin-left: 10px; } -.properties li:nth-child(even) { background: #fafafa; } -.properties p { padding: 10px 15px; } -.properties b { padding: 5px 10px; display: inline-block; - vertical-align: middle; } -.api-urls{ margin-top: 30px; margin-bottom: 30px; } - diff --git a/web/favicon.ico b/web/favicon.ico deleted file mode 100644 index 48060b1..0000000 Binary files a/web/favicon.ico and /dev/null differ diff --git a/web/index.html b/web/index.html deleted file mode 100644 index c7095ec..0000000 --- a/web/index.html +++ /dev/null @@ -1,141 +0,0 @@ -<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> -<html> -<head> -<title>Onionoo — a Tor network status protocol</title> -<link href="css/style.css" type="text/css" rel="stylesheet"> -<meta http-equiv="content-type" content="text/html; charset=ISO-8859-1"> -<link href="favicon.ico" type="image/x-icon" rel="shortcut icon"> -</head> -<body> - -<div class="box"> - -<h1><a href="index.html">Onionoo</a> —</h1> -<h2>a Tor network status <a href="protocol.html">protocol</a></h2> - -Onionoo is a web-based protocol to learn about currently running -Tor relays and bridges. Onionoo itself was not designed as a service -for human beings—at least not directly. Onionoo provides the -data for other applications and websites which in turn present Tor -network status information to humans. The following Onionoo clients -are currently available: -<ul> -<li><a href="https://atlas.torproject.org/">Atlas</a> is a web -application to discover Tor relays. It provides useful -information on how relays are configured along with graphics about -their past.</li> -<li><a href="https://compass.torproject.org/">Compass</a> -is a Python script and website that extracts consensus weight -information of currently running relays and aggregates weights of -relays running in the same country or same autonomous system.</li> -<li><a href="http://tor2web.org/">Tor2web</a> is a web proxy to Tor -Hidden Services. It uses Onionoo to get the list of currently running -Tor Exits to detect if the client is a Tor user and if so redirect -them to the .onion address.</li> -<li>The <a href="https://nos-oignons.net/Services/index.en.html">Nos -oignons</a> website uses Onionoo to visualize bandwidth histories of -their relays.</li> -<li>The <a href="https://metrics.torproject.org/bubbles.html">metrics -website</a> visualizes diversity of the Tor network using bubble -graphs.</li> -<li><a href="https://github.com/woeisme/torchart">torchart</a> is a PHP -WordPress plugin which draws 3 day traffic charts using pChart currently -used for displaying metric charts on -<a href="http://icetor.is/wordpress/metrics/">icetor.is</a>.</li> -<li><a href="https://github.com/kloesing/challenger">challenger</a> -aggregates graph data from relays participating in -<a href="https://www.eff.org/torchallenge/">EFF's 2014 Tor Challenge</a> -to provide statistics like total bytes transferred.</li> -<li><a href="https://play.google.com/store/apps/details?id=com.networksaremadeofstring.a…">AnOnionooid</a> -is an Android app that helps find and explore Tor relays and bridges.</li> -<li><a href="https://oniontip.com/">OnionTip</a> uses Onionoo's data to -distribute donations to relays that have a bitcoin address in their -contact line.</li> -<li><a href="https://savannah.nongnu.org/projects/koninoo/">koninoo</a> is -a simple Java command line interface for querying Onionoo data.</li> -<li>The <a href="https://duckduckgo.com/">DuckDuckGo</a> search engine -displays Tor node details when being asked for "tor node" followed by a -search term.</li> -<li><a href="https://onionview.com/">OnionView</a> is a simple web service -which uses Tor relay data to plot the location of active Tor nodes onto an -interactive map of the world.</li> -</ul> - -The following library facilitates development of Onionoo clients: -<ul> -<li><a href="https://github.com/duk3luk3/onion-py">OnionPy</a> is a -comprehensive pure-Python (2.7+) Onionoo wrapper with caching -support.</li> -</ul> - -<h2>Developing Onionoo applications</h2> - -The project pages of the Onionoo clients listed above have further -information for contacting the authors and contributing ideas or code. -The authors will be happy to hear your thoughts! - -You don't find your favorite Onionoo client above? Want to -implement your own and tell us to add it to the list? The Onionoo -clients above are backed by a web-based -<a href="protocol.html">protocol</a>, which -facilitates developing new applications displaying Tor status -information. Here are a few ideas for new Onionoo clients: -<ul> -<li>Tor controller extension: Extend -<a href="https://www.torproject.org/projects/vidalia.html.en">Vidalia</a> and/or -<a href="https://www.torproject.org/projects/arm.html.en">arm</a> to look up details for the bridge -that the user is running and display what pool the bridge is contained -in.</li> -<li>Social network site plugin: Add a plugin to the social network -site of your choice to show your friends what Tor relays and bridges -you're running and how that helps users around the world.</li> -<li>Desktop tray icon: Write a tray icon for your favorite desktop -environment that tells you when your relay or bridge is down and that -displays some basic usage statistics.</li> -<li>E-mail notification service: Improve our e-mail notification -service <a href="https://weather.torproject.org/">Weather</a> by -implementing its own relay search or extending it to report when a -bridge drops off the network.</li> -<li>Command-line tool: Implement a command-line tool that quickly -searches a relay or bridge and prints out some status information to -help debug problems.</li> -<li>(Insert your idea here.)</li> -</ul> - -Want to help with developing the Onionoo server that provides -Tor status data, or want to run your own Onionoo server instance? The -Onionoo server is written in Java with a tiny portion of Java -Servlets. Instructions for setting up the Onionoo server to fetch the -required data from the Tor servers is described in the INSTALL file in -the sources. For more details see the -<a href="https://gitweb.torproject.org/onionoo.git">source code</a> and -<a href="https://trac.torproject.org/projects/tor/query?status=!closed&component=Oni…">issue -tracker</a>. - -<h2>Related projects</h2> - -TorStatus is the name of a nowadays -<a href="https://svn.torproject.org/svn/torstatus/trunk/">unmaintained</a> -website that displays Tor relay information similar to -<a href="http://atlas.torproject.org/">Atlas</a>. There are still a -few <a href="http://torstatus.blutmagie.de/">TorStatus</a> -<a href="https://torstatus.rueckgr.at/">websites</a> -<a href="http://tns.hermetix.org/">running</a>. - -There's another project from summer 2011 called TorStatus which is -a <a href="https://gitweb.torproject.org/torstatus.git">rewrite</a> of -the original TorStatus in Python/Django. Unfortunately, it's also -unmaintained. - -Finally, there's the -<a href="https://consensus-health.torproject.org/">consensus-health -page</a> which has the primary purpose of indicating problems with -creating a network status consensus. As a side-effect this page lists -all currently running relays and how the directory authorities voted -on them. - -</div>  - -</body> -</html> - diff --git a/web/protocol.html b/web/protocol.html deleted file mode 100644 index c6cc534..0000000 --- a/web/protocol.html +++ /dev/null @@ -1,2338 +0,0 @@ -<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> -<html> -<head> -<title>Onionoo — protocol overview</title> -<link href="css/style.css" type="text/css" rel="stylesheet"> -<meta http-equiv="content-type" content="text/html; charset=ISO-8859-1"> -<link href="favicon.ico" type="image/x-icon" rel="shortcut icon"> -</head> -<body> - -<div class="box"> - -<h1><a href="index.html">Onionoo</a> —</h1> -<h2>protocol overview</h2> - - <h4>Table of contents</h4> - <ul> - <li><a href="#general">General</a></li> - <li><a href="#methods">Methods</a></li> - <li><a href="#summary">Summary documents</a></li> - <li><a href="#details">Details documents</a></li> - <li><a href="#history">History objects</a></li> - <li><a href="#bandwidth">Bandwidth documents</a></li> - <li><a href="#weights">Weights documents</a></li> - <li><a href="#clients">Clients documents</a></li> - <li><a href="#uptime">Uptime documents</a></li> - <li><a href="#examples">Example usage</a></li> - </ul> - -</div> -<div class="box"> - -<a name="general"></a> -<h3>General <a href="#general">#</a></h3> - - -The Onionoo service is designed as a RESTful web service. -Onionoo clients send HTTP GET requests to the Onionoo server which -responds with JSON-formatted replies. -Onionoo clients and their developers should follow a few rules: - - -<h4>Compression</h4> - -Clients should include an "Accept-Encoding: -gzip" header in their requests and handle gzip-compressed -responses. -Only requests starting at a certain size will be compressed by the -server. - - -<h4>Caching</h4> -Clients should make use of the "Last-Modified" -header of responses and include that timestamp in a -"If-Modified-Since" header of subsequent requests. - - -<h4>Response codes</h4> - -Clients should handle response codes by -distinguishing between client and server errors, and if there's a problem, -informing their users about the kind of problem. -The following response codes are used: - - -<ul class="properties"> - -<li> -200 OK - -The request was processed successfully. - -</li> - -<li> -304 Not Modified - -Server data has not changed since the -"If-Modified-Since" header included in the request. - -</li> - -<li> -400 Bad Request - -The request for a known resource could not be processed because of bad -syntax. -This is most likely a client problem. - -</li> - -<li> -404 Not Available - -The request could not be processed because the requested resource could -not be found. -This is most likely a client problem. - -</li> - -<li> -500 Internal Server Error - -There is an unspecific problem with -the server which the service operator may not yet be aware of. -Please check if there's already a bug report for this problem, and if not, -file one. - -</li> - -<li> -503 Service Unavailable - -The server is temporarily down for -maintenance, or there is a temporary problem with the server that the -service operator is already aware of. -Only file a bug report if this problem persists. - -</li> - -</ul> - -<h4>Protocol versions</h4> - -There are plenty of reasons why we may have -to change the protocol described here. -Clients should be able to handle all valid JSON responses, ignoring -unknown fields and not breaking horribly in case of missing fields. -Protocol changes will be announced here by writing deprecated API parts in -red and new parts in -blue. -Deprecated parts will be removed one month after their announcement. -If you want to be informed of upcoming protocol changes, subscribe to the -<a href="https://lists.torproject.org/cgi-bin/mailman/listinfo/onionoo-announce">onionoo-announce</a> -mailing list. - - -All responses contain a "version" string that -indicates whether clients can parse the document or not. -Version strings consist of a major and a minor version number. -The major version number is raised when previously required fields are -dropped or turned into optional fields, when request parameters or -response documents are removed, or when there are structural changes. -The minor version number is raised when new fields, request parameters, or -response documents are added or optional fields are dropped. -If clients support the same major version given in a response but only a -lower minor version, they should be able to parse the document but may not -understand all fields. -If clients support a lower major version, they should not attempt to parse -a document, because there may be backward-incompatible changes. - -Responses may also contain the optional -"next_major_version_scheduled" field that announces when -the next major version is scheduled to be deployed. -Clients should inform their users that they should upgrade to the next -major protocol version before the provided date. - -The following versions have been used in the past six months or are -scheduled to be deployed in the next months: - -<ul> -<li>1.0: First assigned version number on August 31, -2014.</li> -<li>1.1: Added optional "next_major_version_scheduled" -field on September 16, 2014.</li> -<li>1.2: Added qualified search terms to "search" -parameter on October 17, 2014.</li> -<li>2.0: Extended search parameter to base64-encoded -fingerprints on November 15, 2014.</li> -<li>2.1: Removed optional "advertised_bandwidth_fraction" -field from details documents and optional "advertised_bandwidth" and -"advertised_bandwidth_fraction" fields from weights documents on November -16, 2014.</li> -<li>2.2: Removed optional "pool_assignment" field and -added "transports" field to bridge details documents on December 8, -2014.</li> -<li>2.3: Added optional "flags" field to uptime -documents on March 22, 2015.</li> -<li>2.4: Added optional "effective_family" field to -details documents on July 3, 2015.</li> -<li>2.5: Added optional "measured" field to details -documents on August 13, 2015.</li> -<li>2.6: Added optional "alleged_family" and -"indirect_family" fields and deprecated optional "family" field in details -documents on August 25, 2015.</li> -<li>3.0: Extended search parameter to match any 4 hex -characters of a space-separated fingerprint on November 15, 2015.</li> -<li>3.1: Removed optional "family" field on January 18, -2016.</li> -</ul> - -</div>  - -<div class="box"> - -<a name="methods"></a> -<h3>Methods <a href="#methods">#</a></h3> - - -The following methods each return a single document containing zero or -more objects of relays and/or bridges. -By default, all relays and bridges are included that have been running in -the past week. - - -<ul class="api-urls"> - -<li class="api-request"> -GET -https://onionoo.torproject.org/summary -returns a <a href="#summary">summary -document</a> -</li> - -<li class="api-request"> -GET -https://onionoo.torproject.org/details -returns a <a href="#details">details -document</a> -</li> - -<li class="api-request"> -GET -https://onionoo.torproject.org/bandwidth -returns a <a href="#bandwidth">bandwidth -document</a> -</li> - -<li class="api-request"> -GET -https://onionoo.torproject.org/weights -returns a <a href="#weights">weights -document</a> -</li> - -<li class="api-request"> -GET -https://onionoo.torproject.org/clients -returns a <a href="#clients">clients -document</a> -</li> - -<li class="api-request"> -GET -https://onionoo.torproject.org/uptime -returns a <a href="#uptime">uptime -document</a> -</li> - -</ul> - -<h4>Parameters</h4> - -Each of the methods can be parameterized to select only a subset of relay -and/or bridge documents that are currently running or that have been -running in the past week. -(The fingerprint parameter is special here, because it -allows selecting a specific relay or bridge, regardless of whether it has -been running in the past week.) -If multiple parameters are specified, they are combined using a logical -AND operation, meaning that only the intersection of relays and bridges -matching all parameters is returned. -If the same parameter is specified more than once, only the first -parameter value is considered. - - -<ul class="properties"> - -<li> -type - -Return only relay (parameter value -relay) or only bridge documents (parameter value -bridge). -Parameter values are case-insensitive. - -</li> - -<li> -running - -Return only running (parameter value -true) or only non-running relays and/or bridges -(paramter value -false). -Parameter values are case-insensitive. - -</li> - -<li> -search - -Return only (1) relays with the parameter value matching (part of a) -nickname, (possibly $-prefixed) beginning of a hex-encoded fingerprint, -any 4 hex characters of a space-separated fingerprint, beginning of a -base64-encoded fingerprint without trailing equal signs, or beginning of -an IP address, (2) bridges with (part of a) nickname or (possibly -$-prefixed) beginning of a hashed hex-encoded fingerprint, and (3) relays -and/or bridges matching a given qualified search term. -Searches by relay IP address include all known addresses used for onion -routing and for exiting to the Internet. -Searches for beginnings of IP addresses are performed on textual -representations of canonical IP address forms, so that searches using CIDR -notation or non-canonical forms will return empty results. -Searches are case-insensitive, except for base64-encoded fingerprints. -If multiple search terms are given, separated by spaces, the intersection -of all relays and bridges matching all search terms will be returned. -Complete hex-encoded fingerprints should always be hashed using SHA-1, -regardless of searching for a relay or a bridge, in order to not -accidentally leak non-hashed bridge fingerprints in the URL. -Qualified search terms have the form "key:value" (without double quotes) -with "key" being one of the parameters listed here except for "search", -"fingerprint", "order", "limit", "offset", and "fields", and "value" being -the string that will internally be passed to that parameter. - -</li> - -<li> -lookup - -Return only the relay with the parameter -value matching the fingerprint or the bridge with the parameter value -matching the hashed fingerprint. -Fingerprints should always be hashed using SHA-1, regardless of looking up -a relay or a bridge, in order to not accidentally leak non-hashed bridge -fingerprints in the URL. -Lookups only work for full fingerprints or hashed fingerprints consisting -of 40 hex characters. -Lookups are case-insensitive. - -</li> - -<li> -fingerprint - -Return only the relay with the parameter value matching the fingerprint -or the bridge with the parameter value matching the hashed fingerprint. -Fingerprints must consist of 40 hex characters, case does not matter. -This parameter is quite similar to the lookup parameter -with two exceptions: -(1) the provided relay fingerprint or hashed bridge fingerprint must -not be hashed (again) using SHA-1; -(2) the response will contain any matching relay or bridge regardless of -whether they have been running in the past week. - -</li> - -<li> -country - -Return only relays which are located in the -given country as identified by a two-letter country code. -Filtering by country code is case-insensitive. - -</li> - -<li> -as - -Return only relays which are located in the -given autonomous system (AS) as identified by the AS number (with or -without preceding "AS" part). -Filtering by AS number is case-insensitive. - -</li> - -<li> -flag - -Return only relays which have the -given relay flag assigned by the directory authorities. -Note that if the flag parameter is specified more than once, only the -first parameter value will be considered. -Filtering by flag is case-insensitive. - -</li> - -<li> -first_seen_days - -Return only relays or bridges which -have first been seen during the given range of days ago. -A parameter value "x-y" with x <= y returns relays or bridges that have -first been seen at least x and at most y days ago. -Accepted short forms are "x", "x-", and "-y" which are interpreted as -"x-x", "x-infinity", and "0-y". - -</li> - -<li> -last_seen_days - -Return only relays or bridges which -have last been seen during the given range of days ago. -A parameter value "x-y" with x <= y returns relays or bridges that have -last been seen at least x and at most y days ago. -Accepted short forms are "x", "x-", and "-y" which are interpreted as -"x-x", "x-infinity", and "0-y". -Note that relays and bridges that haven't been running in the past week -are not included in results, so that setting x to 8 or higher will lead to -an empty result set. - -</li> - -<li> -contact - -Return only relays with the parameter value -matching (part of) the contact line. -If the parameter value contains spaces, only relays are returned which -contain all space-separated parts in their contact line. -Only printable ASCII characters are permitted in the parameter value, -some of which need to be percent-encoded (# as %23, % as %25, & as -%26, + as %2B, and / as %2F). -Comparisons are case-insensitive. - -</li> - -<li> -family - -Return only the relay whose fingerprint matches the parameter value and -all relays that this relay has listed in its family by fingerprint and -that in turn have listed this relay in their family by fingerprint. -If relays have listed other relays in their family by nickname, those -family relationships will not be considered, regardless of whether they -have the Named flag or not. -The provided relay fingerprint must consist of 40 hex characters where -case does not matter, and it must not be hashed using SHA-1. -Bridges are not contained in the result, regardless of whether they define -a family. - -</li> - -</ul> - - -Response documents can be reduced in size by requesting only a subset -of contained fields. - - -<ul class="properties"> - -<li> -fields - -Comma-separated list of fields that will be -included in the result. -So far, only top-level fields in relay or bridge objects of details -documents can be specified, e.g., -nickname,hashed_fingerprint. -If the fields parameter is provided, all other fields which are not -contained in the provided list will be removed from the result. -Field names are case-insensitive. - -</li> - -</ul> - - -Relay and/or bridge documents in the response can be ordered and -limited by providing further parameters. -If the same parameter is specified more than once, only the first -parameter value is considered. - - -<ul class="properties"> - -<li> -order - -Re-order results by a comma-separated list -of fields in ascending or descending order. -Results are first ordered by the first list element, then by the second, -and so on. -Possible fields for ordering are: consensus_weight. -Field names are case-insensitive. -Ascending order is the default; descending order is selected by prepending -fields with a minus sign (-). -Relays or bridges which don't have any value for a field to be ordered by -are always appended to the end, regardless or sorting order. -The ordering is defined independent of the requested document type and -does not require the ordering field to be contained in the document. -If no order parameter is given, ordering of results is -undefined. - -</li> - -<li> -offset - -Skip the given number of relays and/or -bridges. -Relays are skipped first, then bridges. -Non-positive offset values are treated as zero and don't -change the -result. - -</li> - -<li> -limit - -Limit result to the given number of -relays and/or bridges. -Relays are kept first, then bridges. -Non-positive limit values are treated as zero and lead -to an empty -result. -When used together with offset, the offsetting step -precedes the -limiting step. - -</li> - -</ul> - -</div>  - -<div class="box"> -<a name="summary"></a> -<h3>Summary documents <a href="#summary">#</a> - -<a href="summary?limit=4">example request</a> - -</h3> - -Summary documents contain short summaries of relays with nicknames, -fingerprints, IP addresses, and running information as well as bridges -with hashed fingerprints and running information. -Summary documents contain the following fields: - -<ul class="properties"> - -<li> -version -<code class="typeof">string</code> -required - -Onionoo protocol version string. - -</li> - -<li> -next_major_version_scheduled -<code class="typeof">string</code> -optional - -UTC date (YYYY-MM-DD) when the next major protocol version is scheduled to -be deployed. -Omitted if no major protocol changes are planned. - -</li> - -<li> -relays_published -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when the last known relay network -status consensus started being valid. -Indicates how recent the relay summaries in this document are. - -</li> - -<li> -relays -<code class="typeof">array of objects</code> -required - -Array of relay summary objects as specified below. - -</li> - -<li> -bridges_published -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when -the last known bridge network status was published. -Indicates how recent the bridge summaries in this document are. - -</li> - -<li> -bridges -<code class="typeof">array of objects</code> -required - -Array of bridge summary objects as specified below. - -</li> - -</ul> - -<h4>Relay summary objects</h4> - - -Relay summary objects contain the following key-value pairs: - - -<ul class="properties"> - -<li> -n -<code class="typeof">string</code> -optional - -Relay nickname consisting of 1–19 alphanumerical characters. -Omitted if the relay nickname is "Unnamed". - -</li> - -<li> -f -<code class="typeof">string</code> -required - -Relay fingerprint consisting of 40 upper-case hexadecimal characters. - -</li> - -<li> -a -<code class="typeof">array of strings</code> -required - -Array of IPv4 or IPv6 addresses where the relay accepts -onion-routing connections or which the relay used to exit to the Internet -in the past 24 hours. -The first address is the primary onion-routing address that the relay used -to register in the network, subsequent addresses are in arbitrary order. -IPv6 hex characters are all lower-case. - -</li> - -<li> -r -<code class="typeof">boolean</code> -required - -Boolean field saying whether this relay was listed as -running in the last relay network status consensus. - -</li> - -</ul> - -<h4>Bridge summary objects</h4> - - -Bridge summary objects contain the following key-value pairs: - - -<ul class="properties"> - -<li> -n -<code class="typeof">string</code> -optional - -Bridge nickname consisting of 1–19 alphanumerical characters. -Omitted if the bridge nickname is "Unnamed". - -</li> - -<li> -h -<code class="typeof">string</code> -required - -SHA-1 hash of the bridge fingerprint consisting of 40 -upper-case hexadecimal characters. - -</li> - -<li> -r -<code class="typeof">boolean</code> -required - -Boolean field saying whether this bridge was listed as -running in the last bridge network status. - -</li> - -</ul> - -</div>  - -<div class="box"> -<a name="details"></a> -<h3>Details documents <a href="#details">#</a> - -<a href="details?limit=4">example request</a> - -</h3> - - -Details documents are based on network statuses published by the Tor -directories, server descriptors published by relays and bridges, and data -published by Tor network services TorDNSEL and BridgeDB. -Details documents use the most recently published data from these sources, -which may lead to contradictions between fields based on different sources -in rare edge cases. -Details documents contain the following fields: - - -<ul class="properties"> - -<li> -version -<code class="typeof">string</code> -required - -Onionoo protocol version string. - -</li> - -<li> -next_major_version_scheduled -<code class="typeof">string</code> -optional - -UTC date (YYYY-MM-DD) when the next major protocol version is scheduled to -be deployed. -Omitted if no major protocol changes are planned. - -</li> - -<li> -relays_published -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when -the last known relay network status consensus started being valid. -Indicates how recent the relay details in this document are. - -</li> - -<li> -relays -<code class="typeof">array of objects</code> -required - -Array of relay details objects as specified below. - -</li> - -<li> -bridges_published -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when -the last known bridge network status was published. -Indicates how recent the bridge details in this document are. - -</li> - -<li> -bridges -<code class="typeof">array of objects</code> -required - -Array of bridge details objects as specified below. - - -</li> - -</ul> - -<h4>Relay details objects</h4> - - -Relay details objects contain the following key-value pairs: - - -<ul class="properties"> - -<li> -nickname -<code class="typeof">string</code> -optional - -Relay nickname consisting of 1–19 alphanumerical characters. -Omitted if the relay nickname is "Unnamed". - -</li> - -<li> -fingerprint -<code class="typeof">string</code> -required - -Relay fingerprint consisting of 40 upper-case -hexadecimal characters. - -</li> - -<li> -or_addresses -<code class="typeof">array of strings</code> -required - -Array of IPv4 or IPv6 addresses and TCP ports -or port lists where the relay accepts onion-routing connections. -The first address is the primary onion-routing address that the relay used -to register in the network, subsequent addresses are in arbitrary order. -IPv6 hex characters are all lower-case. - -</li> - -<li> -exit_addresses -<code class="typeof">array of strings</code> -optional - -Array of IPv4 or IPv6 addresses that the -relay used to exit to the Internet in the past 24 hours. -IPv6 hex characters are all lower-case. -Only those addresses are listed that are different from onion-routing -addresses. -Omitted if array is empty. - -</li> - -<li> -dir_address -<code class="typeof">string</code> -optional - -IPv4 address and TCP port where the relay -accepts directory connections. -Omitted if the relay does not accept directory connections. - -</li> - -<li> -last_seen -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when this -relay was last seen in a network status consensus. - -</li> - -<li> -last_changed_address_or_port -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD -hh:mm:ss) when this relay last stopped announcing an IPv4 or IPv6 address -or TCP port where it previously accepted onion-routing or directory -connections. -This timestamp can serve as indicator whether this relay would be a -suitable fallback directory. - -</li> - -<li> -first_seen -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when this -relay was first seen in a network status consensus. - -</li> - -<li> -running -<code class="typeof">boolean</code> -required - -Boolean field saying whether this relay was listed as -running in the last relay network status consensus. - -</li> - -<li> -hibernating -<code class="typeof">boolean</code> -optional - -Boolean field saying whether this relay indicated that it is hibernating -in its last known server descriptor. -This information may be helpful to decide whether a relay that is not -running anymore has reached its accounting limit and has not dropped out -of the network for another, unknown reason. -Omitted if either the relay is not hibernating, or if no information is -available about the hiberation status of the relay. - -</li> - -<li> -flags -<code class="typeof">array of strings</code> -optional - -Array of relay flags that the directory authorities -assigned to this relay. -Omitted if empty. - -</li> - -<li> -country -<code class="typeof">string</code> -optional - -Two-letter lower-case country code as found in a -GeoIP database by resolving the relay's first onion-routing IP address. -Omitted if the relay IP address could not be found in the GeoIP -database. - -</li> - -<li> -country_name -<code class="typeof">string</code> -optional - -Country name as found in a GeoIP database by -resolving the relay's first onion-routing IP address. -Omitted if the relay IP address could not be found in the GeoIP -database, or if the GeoIP database did not contain a country name. - -</li> - -<li> -region_name -<code class="typeof">string</code> -optional - -Region name as found in a GeoIP database by -resolving the relay's first onion-routing IP address. -Omitted if the relay IP address could not be found in the GeoIP -database, or if the GeoIP database did not contain a region name. - -</li> - -<li> -city_name -<code class="typeof">string</code> -optional - -City name as found in a -GeoIP database by resolving the relay's first onion-routing IP address. -Omitted if the relay IP address could not be found in the GeoIP -database, or if the GeoIP database did not contain a city name. - -</li> - -<li> -latitude -<code class="typeof">number</code> -optional - -Latitude as found in a GeoIP database by resolving -the relay's first onion-routing IP address. -Omitted if the relay IP address could not be found in the GeoIP -database. - -</li> - -<li> -longitude -<code class="typeof">number</code> -optional - -Longitude as found in a GeoIP database by -resolving the relay's first onion-routing IP address. -Omitted if the relay IP address could not be found in the GeoIP -database. - -</li> - -<li> -as_number -<code class="typeof">string</code> -optional - -AS number as found in an AS database by -resolving the relay's first onion-routing IP address. -AS number strings start with "AS", followed directly by the AS number. -Omitted if the relay IP address could not be found in the AS -database. - -</li> - -<li> -as_name -<code class="typeof">string</code> -optional - -AS name as found in an AS database by resolving the -relay's first onion-routing IP address. -Omitted if the relay IP address could not be found in the AS -database. - -</li> - -<li> -consensus_weight -<code class="typeof">number</code> -required - -Weight assigned to this relay by the -directory authorities that clients use in their path selection algorithm. -The unit is arbitrary; currently it's kilobytes per second, but that might -change in the future. - -</li> - -<li> -host_name -<code class="typeof">string</code> -optional - -Host name as found in a reverse DNS lookup of the -relay IP address. -This field is updated at most once in 12 hours, unless the relay IP -address changes. -Omitted if the relay IP address was not looked up or if no lookup request -was successful yet. - -</li> - -<li> -last_restarted -<code class="typeof">string</code> -optional - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when the -relay was last (re-)started. -Missing if router descriptor containing this information cannot be -found. - -</li> - -<li> -bandwidth_rate -<code class="typeof">number</code> -optional - -Average bandwidth -in bytes per second that this relay is willing to sustain over long -periods. -Missing if router descriptor containing this information cannot be -found. - -</li> - -<li> -bandwidth_burst -<code class="typeof">number</code> -optional - -Bandwidth in bytes -per second that this relay is willing to sustain in very short intervals. -Missing if router descriptor containing this information cannot be -found. - -</li> - -<li> -observed_bandwidth -<code class="typeof">number</code> -optional - -Bandwidth -estimate in bytes per second of the capacity this relay can handle. -The relay remembers the maximum bandwidth sustained output over any ten -second period in the past day, and another sustained input. -The "observed_bandwidth" value is the lesser of these two numbers. -Missing if router descriptor containing this information cannot be -found. - -</li> - -<li> -advertised_bandwidth -<code class="typeof">number</code> -optional - -Bandwidth in bytes per second that this -relay is willing and capable to provide. -This bandwidth value is the minimum of bandwidth_rate, -bandwidth_burst, and observed_bandwidth. -Missing if router descriptor containing this information cannot be -found. - -</li> - -<li> -exit_policy -<code class="typeof">array of strings</code> -optional - -Array of exit-policy lines. -Missing if router descriptor containing this information cannot be -found. -May contradict the "exit_policy_summary" field in a rare -edge case: this happens when the relay changes its exit policy after the -directory authorities summarized the previous exit policy. - -</li> - -<li> -exit_policy_summary -<code class="typeof">object</code> -optional - -Summary -version of the relay's exit policy containing a dictionary with either an -"accept" or a "reject" element. -If there is an "accept" ("reject") element, the relay accepts (rejects) -all TCP ports or port ranges in the given list for most IP addresses and -rejects (accepts) all other ports. -May contradict the "exit_policy" field in a rare edge -case: this happens when the relay changes its exit policy after the -directory authorities summarized the previous exit policy. - -</li> - -<li> -exit_policy_v6_summary -<code class="typeof">object</code> -optional - -Summary version of the relay's IPv6 exit policy containing a dictionary -with either an "accept" or a "reject" element. -If there is an "accept" ("reject") element, the relay accepts (rejects) -all TCP ports or port ranges in the given list for most IP addresses and -rejects (accepts) all other ports. -Missing if the relay rejects all connections to IPv6 addresses. -May contradict the "exit_policy_summary" field in a rare -edge case: this happens when the relay changes its exit policy after the -directory authorities summarized the previous exit policy. - -</li> - -<li> -contact -<code class="typeof">string</code> -optional - -Contact address of the relay operator. -Omitted if empty or if descriptor containing this information cannot be -found. - -</li> - -<li> -platform -<code class="typeof">string</code> -optional - -Platform string containing operating system and Tor -version details. -Omitted if empty or if descriptor containing this information cannot be -found. - -</li> - -<li> -recommended_version -<code class="typeof">boolean</code> -optional - -Boolean field saying whether the Tor software version of this relay is -recommended by the directory authorities or not. -Omitted if either the directory authorities did not recommend versions, or -the relay did not report which version it runs. - -</li> - -<li> -family -<code class="typeof">array of strings</code> -optional - -Array of fingerprints or nicknames of relays that this relay considers to -be part of its family. -There are no cross-checks whether the listed relays exist or consider this -relay part of their family, so that the effective family of this relay may -be smaller. -Omitted if empty or if descriptor containing this information cannot be -found. -Deprecated on August 25, 2015, removed on January 18, -2016. - -</li> - -<li> -effective_family -<code class="typeof">array of strings</code> -optional - -Array of $-prefixed fingerprints of relays that are in an effective, -mutual family relationship with this relay. -These relays are part of this relay's family and they consider this relay -to be part of their family. -Omitted if empty or if descriptor containing this information cannot be -found. - -</li> - -<li> -alleged_family -<code class="typeof">array of strings</code> -optional - -Array of $-prefixed fingerprints of relays that are not in an effective, -mutual family relationship with this relay. -These relays are part of this relay's family but they don't consider this -relay to be part of their family. -Omitted if empty or if descriptor containing this information cannot be -found. - -</li> - -<li> -indirect_family -<code class="typeof">array of strings</code> -optional - -Array of $-prefixed fingerprints of relays that are not in an effective, -mutual family relationship with this relay but that can be reached by -following effective, mutual family relationships starting at this relay. -Omitted if empty or if descriptor containing this information cannot be -found. - -</li> - -<li> -consensus_weight_fraction -<code class="typeof">number</code> -optional - -Fraction of this relay's consensus weight compared to the sum of all -consensus weights in the network. -This fraction is a very rough approximation of the probability of this -relay to be selected by clients. -Omitted if the relay is not running. - -</li> - -<li> -guard_probability -<code class="typeof">number</code> -optional - -Probability of this relay to be selected for the guard position. -This probability is calculated based on consensus weights, relay flags, -and bandwidth weights in the consensus. -Path selection depends on more factors, so that this probability can only -be an approximation. -Omitted if the relay is not running, or the consensus does not contain -bandwidth weights. - -</li> - -<li> -middle_probability -<code class="typeof">number</code> -optional - -Probability of this relay to be selected for the middle position. -This probability is calculated based on consensus weights, relay flags, -and bandwidth weights in the consensus. -Path selection depends on more factors, so that this probability can only -be an approximation. -Omitted if the relay is not running, or the consensus does not contain -bandwidth weights. - -</li> - -<li> -exit_probability -<code class="typeof">number</code> -optional - -Probability of this relay to be selected for the exit position. -This probability is calculated based on consensus weights, relay flags, -and bandwidth weights in the consensus. -Path selection depends on more factors, so that this probability can only -be an approximation. -Omitted if the relay is not running, or the consensus does not contain -bandwidth weights. - -</li> - -<li> -measured -<code class="typeof">boolean</code> -optional - -Boolean field saying whether the consensus weight of this relay is based -on a threshold of 3 or more measurements by Tor bandwidth authorities. -Omitted if the network status consensus containing this relay does not -contain measurement information. - -</li> - -</ul> - -<h4>Bridge details objects</h4> - - -Bridge details objects contain the following key-value pairs: - - -<ul class="properties"> - -<li> -nickname -<code class="typeof">string</code> -optional - -Bridge nickname consisting of 1–19 -alphanumerical characters. -Omitted if the bridge nickname is "Unnamed". - -</li> - -<li> -hashed_fingerprint -<code class="typeof">string</code> -required - -SHA-1 hash of the bridge fingerprint -consisting of 40 upper-case hexadecimal characters. - -</li> - -<li> -or_addresses -<code class="typeof">array of strings</code> -required - -Array of sanitized IPv4 or IPv6 addresses and -TCP ports or port lists where the bridge accepts onion-routing -connections. -The first address is the primary onion-routing address that the bridge -used to register in the network, subsequent addresses are in arbitrary -order. -IPv6 hex characters are all lower-case. -Sanitized IP addresses are always in 10/8 or -[fd9f:2e19:3bcf/48] IP networks and are only useful to -learn which -IP version the bridge uses and to detect whether the bridge changed its -address. -Sanitized IP addresses always change on the 1st of every month at 00:00:00 -UTC, regardless of the bridge actually changing its IP address. -TCP ports are not sanitized. - -</li> - -<li> -last_seen -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when this -bridge was last seen in a bridge network status. - -</li> - -<li> -first_seen -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when this -bridge was first seen in a bridge network status. - -</li> - -<li> -running -<code class="typeof">boolean</code> -required - -Boolean field saying whether this bridge was listed -as running in the last bridge network status. - -</li> - -<li> -flags -<code class="typeof">array of strings</code> -optional - -Array of relay flags that the bridge authority -assigned to this bridge. -Omitted if empty. - -</li> - -<li> -last_restarted -<code class="typeof">string</code> -optional - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when the -bridge was last (re-)started. -Missing if router descriptor containing this information cannot be -found. - -</li> - -<li> -advertised_bandwidth -<code class="typeof">number</code> -optional - -Bandwidth in bytes per second that this -bridge is willing and capable to provide. -This bandwidth value is the minimum of bandwidth_rate, -bandwidth_burst, and observed_bandwidth. -Missing if router descriptor containing this information cannot be -found. - -</li> - -<li> -platform -<code class="typeof">string</code> -optional - -Platform string containing operating system and Tor -version details. -Omitted if not provided by the bridge or if descriptor containing this -information cannot be found. - -</li> - -<li> -transports -<code class="typeof">array of strings</code> -optional - -Array of (pluggable) transport names supported by this bridge. - -</li> - -</ul> - -</div>  - -<div class="box"> -<a name="history"></a> -<h3>History objects <a href="#history">#</a></h3> - - -History objects are no documents by themselves, but are contained in -subsequent documents. - - -<h4>Graph history objects</h4> - - -Graph history objects contain the following fields: - - -<ul class="properties"> - -<li> -first -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) of the first data point, or more -specifically the interval midpoint of the first interval. - -</li> - -<li> -last -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) of the last data point, or more -specifically the interval midpoint of the last interval. - -</li> - -<li> -interval -<code class="typeof">number</code> -required - -Time interval between two data points in seconds. - -</li> - -<li> -factor -<code class="typeof">number</code> -required - -Factor by which subsequent data values need to be multiplied to obtain -original values. -The idea is to reduce document size while still providing sufficient -detail for very different data scales. - -</li> - -<li> -count -<code class="typeof">number</code> -optional - -Number of provided data points, included mostly for debugging purposes. -Can also be derived from the number of elements in the subsequent array. - -</li> - -<li> -values -<code class="typeof">array of numbers</code> -required - -Array of normalized values between 0 and 999. -May contain null values. -Contains at least two subsequent non-null values to enable drawing of line -graphs. - -</li> - -</ul> - -</div>  - -<div class="box"> -<a name="bandwidth"></a> -<h3>Bandwidth documents <a href="#bandwidth">#</a> - -<a href="bandwidth?limit=4">example request</a> - -</h3> - - -Bandwidth documents contain aggregate statistics of a relay's or -bridge's consumed bandwidth for different time intervals. -Bandwidth documents are only updated when a relay or bridge publishes a -new server descriptor, which may take up to 18 hours during normal -operation. -Bandwidth documents contain the following fields: - - -<ul class="properties"> - -<li> -version -<code class="typeof">string</code> -required - -Onionoo protocol version string. - -</li> - -<li> -next_major_version_scheduled -<code class="typeof">string</code> -optional - -UTC date (YYYY-MM-DD) when the next major protocol version is scheduled to -be deployed. -Omitted if no major protocol changes are planned. - -</li> - -<li> -relays_published -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when -the last known relay network status consensus started being valid. -Indicates how recent the relay bandwidth documents in this document are. - -</li> - -<li> -relays -<code class="typeof">array of objects</code> -required - -Array of relay bandwidth objects as specified below. - -</li> - -<li> -bridges_published -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when -the last known bridge network status was published. -Indicates how recent the bridge bandwidth documents in this document are. - -</li> - -<li> -bridges -<code class="typeof">array of objects</code> -required - -Array of bridge bandwidth objects as specified below. - -</li> - -</ul> - -<h4>Relay bandwidth objects</h4> - - -Relay bandwidth objects contain the following key-value pairs: - - -<ul class="properties"> - -<li> -fingerprint -<code class="typeof">string</code> -required - -Relay fingerprint consisting of 40 upper-case -hexadecimal characters. - -</li> - -<li> -write_history -<code class="typeof">object</code> -optional - -Object containing graph history objects with written bytes for different -time periods. -Keys are string representation of the time period covered by the graph -history object. -Keys are fixed strings "3_days", -"1_week", "1_month", -"3_months", "1_year", and -"5_years". -Keys refer to the last known bandwidth history of a relay, not to the time -when the bandwidth document was published. -A graph history object is only contained if the time period it covers is -not already contained in another graph history object with shorter time -period and higher data resolution. -Similarly, a graph history object is excluded if the relay did not provide -bandwidth histories on the required level of detail. -The unit is bytes per second. -Contained graph history objects may contain null values if the relay did -not provide any bandwidth data or only data for less than 20% of a given -time period. - -</li> - -<li> -read_history -<code class="typeof">object</code> -optional - -Object containing graph history objects with read bytes for different time -periods. -The specification of graph history objects is similar to those in the -write_history field. - -</li> - -</ul> - -<h4>Bridge bandwidth objects</h4> - - -Bridge bandwidth objects contain the following key-value pairs: - - -<ul class="properties"> - -<li> -fingerprint -<code class="typeof">string</code> -required - -SHA-1 hash of the bridge fingerprint consisting -of 40 upper-case hexadecimal characters. - -</li> - -<li> -write_history -<code class="typeof">object</code> -optional - -Object containing graph history objects with written bytes for different -time periods. -The specification of graph history objects is similar to those in the -write_history field of relays. - -</li> - -<li> -read_history -<code class="typeof">object</code> -optional - -Object containing graph history objects with read bytes for different time -periods. -The specification of graph history objects is similar to those in the -write_history field of relays. - -</li> - -</ul> - -</div>  - -<div class="box"> -<a name="weights"></a> -<h3>Weights documents <a href="#weights">#</a> - -<a href="weights?limit=4">example request</a> - -</h3> - - -Weights documents contain aggregate statistics of a relay's probability -to be selected by clients for building paths. -Weights documents contain different time intervals and are available for -relays only. -Weights documents contain the following fields: - - -<ul class="properties"> - -<li> -version -<code class="typeof">string</code> -required - -Onionoo protocol version string. - -</li> - -<li> -next_major_version_scheduled -<code class="typeof">string</code> -optional - -UTC date (YYYY-MM-DD) when the next major protocol version is scheduled to -be deployed. -Omitted if no major protocol changes are planned. - -</li> - -<li> -relays_published -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when -the last known relay network status consensus started being valid. -Indicates how recent the relay weights documents in this document are. - -</li> - -<li> -relays -<code class="typeof">array of objects</code> -required - -Array of relay weights objects as specified below. - -</li> - -<li> -bridges_published -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when -the last known bridge network status was published. -Only included for compatibility reasons with the other document types. - -</li> - -<li> -bridges -<code class="typeof">array of objects</code> -required - -Empty array of objects that would represent bridge weights documents. -Only included for compatibility reasons with the other document types. - -</li> - -</ul> - -<h4>Relay weights objects</h4> - - -Relay weights objects contain the following key-value pairs: - - -<ul class="properties"> - -<li> -fingerprint -<code class="typeof">string</code> -required - -Relay fingerprint consisting of 40 upper-case -hexadecimal characters. - -</li> - -<li> -consensus_weight_fraction -<code class="typeof">object</code> -optional - -History object containing the -fraction of this relay's consensus weight compared to the sum of all -consensus weights in the network. -This fraction is a very rough approximation of the probability of this -relay to be selected by clients. -Keys are string representation of the time period covered by the graph -history object. -Keys are fixed strings "1_week", -"1_month", "3_months", -"1_year", and "5_years". -Keys refer to the last known weights history of a relay, not to the time -when the weights document was published. -A graph history object is only contained if the time period it covers is -not already contained in another graph history object with shorter time -period and higher data resolution. -The unit is path-selection probability. -Contained graph history objects may contain null values if the relay was -running less than 20% of a given time period. - -</li> - -<li> -guard_probability -<code class="typeof">object</code> -optional - -History object containing the probability -of this relay to be selected for the guard position. -This probability is calculated based on consensus weights, relay flags, -and bandwidth weights in the consensus. -Path selection depends on more factors, so that this probability can only -be an approximation. -The specification of this history object is similar to that in the -consensus_weight_fraction field above. - -</li> - -<li> -middle_probability -<code class="typeof">object</code> -optional - -History object containing the probability -of this relay to be selected for the middle position. -This probability is calculated based on consensus weights, relay flags, -and bandwidth weights in the consensus. -Path selection depends on more factors, so that this probability can only -be an approximation. -The specification of this history object is similar to that in the -consensus_weight_fraction field above. - -</li> - -<li> -exit_probability -<code class="typeof">object</code> -optional - -History object containing the probability -of this relay to be selected for the exit position. -This probability is calculated based on consensus weights, relay flags, -and bandwidth weights in the consensus. -Path selection depends on more factors, so that this probability can only -be an approximation. -The specification of this history object is similar to that in the -consensus_weight_fraction field above. - -</li> - -<li> -consensus_weight -<code class="typeof">object</code> -optional - -History object containing the absolute consensus weight of this relay. -The specification of this history object is similar to that in the -consensus_weight_fraction field above. - -</li> - -</ul> - -</div>  - -<div class="box"> -<a name="clients"></a> -<h3>Clients documents <a href="#clients">#</a> - -<a href="clients?limit=4">example request</a> - -</h3> - - -Clients documents contain estimates of the average number of clients -connecting to a bridge every day. -There are no clients documents available for relays, just for bridges. -Clients documents contain different time intervals and are available for -bridges only. -Clients documents contain the following fields: - - -<ul class="properties"> - -<li> -version -<code class="typeof">string</code> -required - -Onionoo protocol version string. - -</li> - -<li> -next_major_version_scheduled -<code class="typeof">string</code> -optional - -UTC date (YYYY-MM-DD) when the next major protocol version is scheduled to -be deployed. -Omitted if no major protocol changes are planned. - -</li> - -<li> -relays_published -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when -the last known relay network status consensus started being valid. -Only included for compatibility reasons with the other document types. - -</li> - -<li> -relays -<code class="typeof">array of objects</code> -required - -Empty array of objects that would represent relay clients documents. -Only included for compatibility reasons with the other document types. - -</li> - -<li> -bridges_published -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when -the last known bridge network status was published. -Indicates how recent the bridge clients documents in this document are. - -</li> - -<li> -bridges -<code class="typeof">array of objects</code> -required - -Array of bridge clients objects as specified below. - - -</li> - -</ul> - -<h4>Bridge clients objects</h4> - - -Bridge clients objects contain the following key-value pairs: - - -<ul class="properties"> - -<li> -fingerprint -<code class="typeof">string</code> -required - -SHA-1 hash of the bridge fingerprint consisting -of 40 upper-case hexadecimal characters. - -</li> - -<li> -average_clients -<code class="typeof">object</code> -optional - -Object containing graph history objects with the average number of clients -connecting to this bridge. -Keys are string representation of the time period covered by the graph -history object. -Keys are fixed strings "1_week", -"1_month", "3_months", -"1_year", and "5_years". -Keys refer to the last known clients history of a bridge, not to the time -when the clients document was published. -A graph history object is only contained if the time period it covers -is not already contained in another clients graph object with shorter -time period and higher data resolution. -The unit is number of clients. -Contained graph history objects may contain null values if the bridge did -not report client statistics for at least 50% of a given time period. -Each graph history object contains the following additional key-value -pairs: - - -<ul class="properties"> - -<li> -countries -<code class="typeof">object</code> -optional - -Object containing fractions of clients by country in the considered time -period. -Keys are two-letter lower-case country codes as found in a GeoIP database. -Values are numbers between 0 and 1 standing for the fraction of clients by -country. -A country is only included if at least 1% of clients came from this -country. -Omitted if the bridge did not report client statistics by country. -BETA: This field breaks compatibility -with the history objects contained in other documents pretty badly. -It might be removed in the future without notice. - -</li> - -<li> -transports -<code class="typeof">object</code> -optional - -Object containing fractions of clients by transport in the considered time -period. -Keys are transport names, or "<OR>" for the default -onion-routing transport protocol. -Values are numbers between 0 and 1 standing for the fraction of clients by -transport. -Omitted if the bridge did not report client statistics by transport. -BETA: This field breaks compatibility -with the history objects contained in other documents pretty badly. -It might be removed in the future without notice. - -</li> - -<li> -versions -<code class="typeof">object</code> -optional - -Object containing fractions of clients by IP version in the considered -time period. -Keys are either "v4" for IPv4 or "v6" -for IPv6. -Values are numbers between 0 and 1 standing for the fraction of clients by -version. -Omitted if the bridge did not report client statistics by IP version. -BETA: This field breaks compatibility -with the history objects contained in other documents pretty badly. -It might be removed in the future without notice. - -</li> - -</ul> - -</li> - -</ul> - -</div>  - -<div class="box"> -<a name="uptime"></a> -<h3>Uptime documents <a href="#uptime">#</a> - -<a href="uptime?limit=4">example request</a> - -</h3> - - -Uptime documents contain fractional uptimes of relays and bridges. -Uptime documents contain different time intervals and are available for -relays and bridges. -Uptime documents contain the following fields: - - -<ul class="properties"> - -<li> -version -<code class="typeof">string</code> -required - -Onionoo protocol version string. - -</li> - -<li> -next_major_version_scheduled -<code class="typeof">string</code> -optional - -UTC date (YYYY-MM-DD) when the next major protocol version is scheduled to -be deployed. -Omitted if no major protocol changes are planned. - -</li> - -<li> -relays_published -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when -the last known relay network status consensus started being valid. -Indicates how recent the relay uptime documents in this document are. - -</li> - -<li> -relays -<code class="typeof">array of objects</code> -required - -Array of relay uptime objects as specified below. - -</li> - -<li> -bridges_published -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when -the last known bridge network status was published. -Indicates how recent the bridge uptime documents in this document are. - -</li> - -<li> -bridges -<code class="typeof">array of objects</code> -required - -Array of bridge uptime objects as specified below. - -</li> - -</ul> - -<h4>Relay uptime objects</h4> - - -Relay uptime objects contain the following key-value pairs: - - -<ul class="properties"> - -<li> -fingerprint -<code class="typeof">string</code> -required - -Relay fingerprint consisting of 40 upper-case -hexadecimal characters. - -</li> - -<li> -uptime -<code class="typeof">object</code> -optional - -Object containing graph history objects with the fractional uptime of this -relay. -Keys are string representation of the time period covered by the graph -history object. -Keys are fixed strings "1_week", -"1_month", "3_months", -"1_year", and "5_years". -Keys refer to the last known uptime history of a relay, not to the time -when the uptime document was published. -A graph history object is only contained if the time period it covers is -not already contained in another graph history object with shorter time -period and higher data resolution. -The unit is fractional uptime from 0 to 1. -Contained graph history objects may contain null values if less than 20% -of network statuses have been processed for a given time period. - -</li> - -<li> -flags -<code class="typeof">object</code> -optional - -Object containing fractional times of this relay having relay flags -assigned. -Keys are flag names like "Running" or -"Exit", values are objects similar to the -uptime field above, again with keys like -"1_week" etc. -If a relay never had a given relay flag assigned, no object is included -for that flag. - -</li> - -</ul> - -<h4>Bridge uptime objects</h4> - - -Bridge uptime objects contain the following key-value pairs: - - -<ul class="properties"> - -<li> -fingerprint -<code class="typeof">string</code> -required - -SHA-1 hash of the bridge fingerprint consisting -of 40 upper-case hexadecimal characters. - -</li> - -<li> -uptime -<code class="typeof">object</code> -optional - -Object containing uptime history objects for different time periods. -The specification of uptime history objects is similar to those in the -uptime field of relays. - -</li> - -</ul> - -</div>  - -<div class="box"> -<a name="examples"></a> -<h3>Example usage <a href="#examples">#</a> -</h3> - - -The following examples illustrate how to build requests for some trivial -and some more complex use cases. -While Onionoo is designed mainly for developers and not end users, there -may be cases when it's easier to quickly write a specific query Onionoo -rather than to find an Onionoo client that provides the desired -information. - - -<pre>https://onionoo.torproject.org/summary?limit=4</pre> - - -This first query returns the first four summary documents that Onionoo can -find. -The <code>limit</code> parameter should always be used while developing -new queries to avoid downloading huge responses. - - -<pre>https://onionoo.torproject.org/summary?limit=4&search=moria</pre> - - -The second query restricts results to relays and bridges containing the -string "moria" in one of a few searched fields. - - -<pre>https://onionoo.torproject.org/details?limit=4&search=moria</pre> - - -The third query switches from the short summary documents to the longer -details documents containing, well, more details. - - -<pre>https://onionoo.torproject.org/details?limit=4&search=moria&fields=nickname</pre> - - -The fourth query adds the <code>fields</code> parameter which removes all -fields except the specified ones from the result. -This parameter is only implemented for details documents. - - -<pre>https://onionoo.torproject.org/details?limit=4&search=moria&fields=nickname…</pre> - - -The fifth query sorts results by relay consensus weight from largest to -smallest. - - - -Obviously, this query can be made even more complex by adding more -parameters, and in some cases this is necessary and useful. -Please refer to the protocol specification for details. - - -</div>  - -</body> -</html> - diff --git a/web/robots.txt b/web/robots.txt deleted file mode 100644 index 9b19f8f..0000000 --- a/web/robots.txt +++ /dev/null @@ -1,8 +0,0 @@ -User-agent: * -Disallow: /summary -Disallow: /details -Disallow: /bandwidth -Disallow: /weights -Disallow: /clients -Disallow: /uptime -

1 0

[onionoo/master] Resolve all remaining Javadoc-related checkstyle issues.
by karsten＠torproject.org 25 Jul '16

25 Jul '16

commit c51516bd5044b4b59c24c3cfe6bddc8d9882e018 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jul 19 10:11:02 2016 +0200 Resolve all remaining Javadoc-related checkstyle issues. Resolve very few of these warnings by suppressing them, in particular getters and setters that happen to contain more than 2 lines of code, because we should either document all or none of them. Implements more of #19613. --- build.xml | 13 ++++++++++++ .../java/org/torproject/onionoo/cron/Main.java | 3 +++ .../torproject/onionoo/docs/BandwidthStatus.java | 2 ++ .../torproject/onionoo/docs/ClientsHistory.java | 12 +++++++++++ .../org/torproject/onionoo/docs/ClientsStatus.java | 6 ++++++ .../torproject/onionoo/docs/DateTimeHelper.java | 6 ++++++ .../torproject/onionoo/docs/DetailsDocument.java | 8 ++++++++ .../org/torproject/onionoo/docs/DetailsStatus.java | 3 +++ .../org/torproject/onionoo/docs/DocumentStore.java | 13 ++++++++++++ .../onionoo/docs/DocumentStoreFactory.java | 5 +++++ .../org/torproject/onionoo/docs/NodeStatus.java | 23 ++++++++++++++++++++++ .../torproject/onionoo/docs/SummaryDocument.java | 13 ++++++++++++ .../org/torproject/onionoo/docs/UpdateStatus.java | 2 ++ .../org/torproject/onionoo/docs/UptimeHistory.java | 12 +++++++++++ .../org/torproject/onionoo/docs/UptimeStatus.java | 6 ++++++ .../org/torproject/onionoo/docs/WeightsStatus.java | 6 ++++++ .../org/torproject/onionoo/server/NodeIndexer.java | 12 +++++++++++ .../onionoo/server/NodeIndexerFactory.java | 5 +++++ .../onionoo/server/PerformanceMetrics.java | 6 ++++++ .../torproject/onionoo/server/RequestHandler.java | 4 ++++ .../torproject/onionoo/server/ResourceServlet.java | 6 ++++++ .../org/torproject/onionoo/server/ServerMain.java | 1 + .../onionoo/updater/BandwidthStatusUpdater.java | 6 ++++++ .../onionoo/updater/ClientsStatusUpdater.java | 6 ++++++ .../onionoo/updater/DescriptorSource.java | 10 ++++++++++ .../onionoo/updater/DescriptorSourceFactory.java | 5 +++++ .../torproject/onionoo/updater/LookupService.java | 5 +++++ .../onionoo/updater/NodeDetailsStatusUpdater.java | 6 ++++++ .../onionoo/updater/RdnsLookupRequest.java | 1 + .../onionoo/updater/RdnsLookupWorker.java | 1 + .../onionoo/updater/ReverseDomainNameResolver.java | 9 +++++++++ .../onionoo/updater/StatusUpdateRunner.java | 4 ++++ .../onionoo/updater/UptimeStatusUpdater.java | 6 ++++++ .../onionoo/updater/WeightsStatusUpdater.java | 6 ++++++ .../torproject/onionoo/util/FormattingUtils.java | 5 +++++ .../org/torproject/onionoo/util/TimeFactory.java | 5 +++++ .../onionoo/writer/BandwidthDocumentWriter.java | 2 ++ .../onionoo/writer/ClientsDocumentWriter.java | 2 ++ .../onionoo/writer/DetailsDocumentWriter.java | 2 ++ .../onionoo/writer/DocumentWriterRunner.java | 4 ++++ .../onionoo/writer/SummaryDocumentWriter.java | 2 ++ .../onionoo/writer/UptimeDocumentWriter.java | 2 ++ .../onionoo/writer/WeightsDocumentWriter.java | 2 ++ src/test/resources/metrics_checks.xml | 2 ++ 44 files changed, 260 insertions(+) diff --git a/build.xml b/build.xml index 1688207..f7f6bc0 100644 --- a/build.xml +++ b/build.xml @@ -10,6 +10,7 @@ <property name="classes" value="classes"/> <property name="testresources" value="src/test/resources/"/> <property name="dist" value="dist"/> + <property name="docs" value="${generated}/javadoc/"/> <property name="libs" value="lib"/> <property name="config" value="etc"/> <property name="webxmlfile" value="${config}/web.xml"/> @@ -85,6 +86,7 @@ <copy file="${contextxmltemplate}" tofile="${contextxml}"/> <copy file="${webxmltemplate}" tofile="${webxml}"/> <mkdir dir="${classes}"/> + <mkdir dir="${docs}"/> <mkdir dir="${dist}"/> <mkdir dir="${generated}"/> </target> @@ -111,6 +113,17 @@ </javac> </target> + <target name="docs" depends="init"> + <javadoc destdir="${docs}" + footer="&copy; 2016 The Tor Project" + doctitle="Onionoo Documentation" + use="true" + windowtitle="Onionoo Documentation"> + <classpath refid="classpath"/> + <fileset dir="${javasources}/" includes="**/*.java" /> + </javadoc> + </target> + <target name="test" depends="compile"> <javac destdir="${classes}" srcdir="${tests}" diff --git a/src/main/java/org/torproject/onionoo/cron/Main.java b/src/main/java/org/torproject/onionoo/cron/Main.java index 4a57f66..04b6955 100644 --- a/src/main/java/org/torproject/onionoo/cron/Main.java +++ b/src/main/java/org/torproject/onionoo/cron/Main.java @@ -28,6 +28,8 @@ public class Main implements Runnable { private Logger log = LoggerFactory.getLogger(Main.class); + /** Executes a single update run or partial update run, or initiates + * hourly executions, depending on the given command-line arguments. */ public static void main(String[] args) { Main main = new Main(); main.parseArgsOrExit(args); @@ -121,6 +123,7 @@ public class Main implements Runnable { TimeUnit.MINUTES); } + @Override public void run() { this.acquireLockOrExit(); this.initialize(); diff --git a/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java b/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java index 65d4cf5..ac7085f 100644 --- a/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java @@ -51,6 +51,7 @@ public class BandwidthStatus extends Document { return this.readHistory; } + @Override public void setFromDocumentString(String documentString) { try (Scanner s = new Scanner(documentString)) { while (s.hasNextLine()) { @@ -170,6 +171,7 @@ public class BandwidthStatus extends Document { } } + @Override public String toDocumentString() { StringBuilder sb = new StringBuilder(); for (long[] v : writeHistory.values()) { diff --git a/src/main/java/org/torproject/onionoo/docs/ClientsHistory.java b/src/main/java/org/torproject/onionoo/docs/ClientsHistory.java index 0f4c148..e99c3ef 100644 --- a/src/main/java/org/torproject/onionoo/docs/ClientsHistory.java +++ b/src/main/java/org/torproject/onionoo/docs/ClientsHistory.java @@ -51,6 +51,9 @@ public class ClientsHistory implements Comparable<ClientsHistory> { return this.responsesByVersion; } + /** Instantiates a new clients history object with given interval start + * and end, total responses, and responses by country, transport, and + * version. */ public ClientsHistory(long startMillis, long endMillis, double totalResponses, SortedMap<String, Double> responsesByCountry, @@ -64,6 +67,8 @@ public class ClientsHistory implements Comparable<ClientsHistory> { this.responsesByVersion = responsesByVersion; } + /** Instantiates a new clients history object from the given string that + * may have been produced by {@link #toString()}. */ public static ClientsHistory fromString( String responseHistoryString) { String[] parts = responseHistoryString.split(" ", 8); @@ -133,6 +138,7 @@ public class ClientsHistory implements Comparable<ClientsHistory> { return responses; } + @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append(DateTimeHelper.format(startMillis)); @@ -154,6 +160,9 @@ public class ClientsHistory implements Comparable<ClientsHistory> { } } + /** Adds responses from another clients history object to this one by + * summing up response numbers and extending interval start and/or + * end. */ public void addResponses(ClientsHistory other) { this.totalResponses += other.totalResponses; this.addResponsesByCategory(this.responsesByCountry, @@ -183,16 +192,19 @@ public class ClientsHistory implements Comparable<ClientsHistory> { } } + @Override public int compareTo(ClientsHistory other) { return this.startMillis < other.startMillis ? -1 : this.startMillis > other.startMillis ? 1 : 0; } + @Override public boolean equals(Object other) { return other instanceof ClientsHistory && this.startMillis == ((ClientsHistory) other).startMillis; } + @Override public int hashCode() { return (int) this.startMillis; } diff --git a/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java b/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java index 4ef998e..45466ae 100644 --- a/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java @@ -38,6 +38,7 @@ public class ClientsStatus extends Document { return this.history; } + @Override public void setFromDocumentString(String documentString) { try (Scanner s = new Scanner(documentString)) { while (s.hasNextLine()) { @@ -53,6 +54,8 @@ public class ClientsStatus extends Document { } } + /** Adds all given clients history objects that don't overlap with + * existing clients history objects. */ public void addToHistory(SortedSet<ClientsHistory> newIntervals) { for (ClientsHistory interval : newIntervals) { if ((this.history.headSet(interval).isEmpty() @@ -67,6 +70,8 @@ public class ClientsStatus extends Document { } } + /** Compresses the history of clients objects by merging adjacent + * intervals, depending on how far back in the past they lie. */ public void compressHistory() { SortedSet<ClientsHistory> uncompressedHistory = new TreeSet<ClientsHistory>(this.history); @@ -107,6 +112,7 @@ public class ClientsStatus extends Document { } } + @Override public String toDocumentString() { StringBuilder sb = new StringBuilder(); for (ClientsHistory interval : this.history) { diff --git a/src/main/java/org/torproject/onionoo/docs/DateTimeHelper.java b/src/main/java/org/torproject/onionoo/docs/DateTimeHelper.java index 4e422e9..1dc40c3 100644 --- a/src/main/java/org/torproject/onionoo/docs/DateTimeHelper.java +++ b/src/main/java/org/torproject/onionoo/docs/DateTimeHelper.java @@ -109,6 +109,9 @@ public class DateTimeHelper { return format(millis, ISO_DATETIME_FORMAT); } + /** Parses the given string using the given format and return the time + * in milliseconds since the epoch or {@link #NO_TIME_AVAILABLE} if the + * string cannot be parsed. */ public static long parse(String string, String format) { if (null == string) { log.warn("Date String was null."); @@ -122,6 +125,9 @@ public class DateTimeHelper { } } + /** Parses the given string using {@link #ISO_DATETIME_FORMAT} as format + * and return the time in milliseconds since the epoch or + * {@link #NO_TIME_AVAILABLE} if the string cannot be parsed. */ public static long parse(String string) { return parse(string, ISO_DATETIME_FORMAT); } diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java index 8ffb358..85c0154 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java @@ -377,6 +377,8 @@ public class DetailsDocument extends Document { private Float consensus_weight_fraction; + /** Sets the consensus weight fraction to the given value, but only if + * that value is neither null nor negative. */ public void setConsensusWeightFraction(Float consensusWeightFraction) { if (consensusWeightFraction == null || consensusWeightFraction >= 0.0) { @@ -390,6 +392,8 @@ public class DetailsDocument extends Document { private Float guard_probability; + /** Sets the guard probability to the given value, but only if that + * value is neither null nor negative. */ public void setGuardProbability(Float guardProbability) { if (guardProbability == null || guardProbability >= 0.0) { this.guard_probability = guardProbability; @@ -402,6 +406,8 @@ public class DetailsDocument extends Document { private Float middle_probability; + /** Sets the middle probability to the given value, but only if that + * value is neither null nor negative. */ public void setMiddleProbability(Float middleProbability) { if (middleProbability == null || middleProbability >= 0.0) { this.middle_probability = middleProbability; @@ -414,6 +420,8 @@ public class DetailsDocument extends Document { private Float exit_probability; + /** Sets the exit probability to the given value, but only if that + * value is neither null nor negative. */ public void setExitProbability(Float exitProbability) { if (exitProbability == null || exitProbability >= 0.0) { this.exit_probability = exitProbability; diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java index dcc6fcb..1a3c05d 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java @@ -250,6 +250,9 @@ public class DetailsStatus extends Document { this.or_addresses_and_ports; } + /** Returns all addresses used for the onion-routing protocol which + * includes the primary address and all additionally configured + * onion-routing addresses. */ public SortedSet<String> getOrAddresses() { SortedSet<String> orAddresses = new TreeSet<String>(); if (this.address != null) { diff --git a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java index 600115c..42c75aa 100644 --- a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java +++ b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java @@ -97,6 +97,8 @@ public class DocumentStore { return this.list(documentType, 0L); } + /** Returns all fingerprints of documents of the given type that have + * been updated after the given time in milliseconds since the epoch. */ public <T extends Document> SortedSet<String> list( Class<T> documentType, long updatedAfter) { if (documentType.equals(NodeStatus.class)) { @@ -258,6 +260,8 @@ public class DocumentStore { return this.store(document, null); } + /** Stores the given document using the given fingerprint as + * identifier. */ public <T extends Document> boolean store(T document, String fingerprint) { if (document instanceof NodeStatus) { @@ -372,6 +376,8 @@ public class DocumentStore { return this.retrieve(documentType, parse, null); } + /** Retrieves the document with given type and identified by the given + * fingerprint, and either parses it or returns it unparsed. */ public <T extends Document> T retrieve(Class<T> documentType, boolean parse, String fingerprint) { if (documentType.equals(NodeStatus.class)) { @@ -576,6 +582,8 @@ public class DocumentStore { return this.remove(documentType, null); } + /** Removes the document with given type and identified by the given + * fingerprint. */ public <T extends Document> boolean remove(Class<T> documentType, String fingerprint) { if (documentType.equals(NodeStatus.class)) { @@ -681,6 +689,8 @@ public class DocumentStore { return documentFile; } + /** Writes cached node statuses, cached summary documents, and then the + * update file to disk. */ public void flushDocumentCache() { /* Write cached node statuses to disk, and write update file * containing current time. It's important to write the update file @@ -698,6 +708,8 @@ public class DocumentStore { } } + /** Invalidates the document cache, so that it will be freshly populated + * during the next execution. */ public void invalidateDocumentCache() { this.cachedNodeStatuses = null; this.cachedSummaryDocuments = null; @@ -810,6 +822,7 @@ public class DocumentStore { this.store(updateStatus); } + /** Returns a string with statistics on document storage operations. */ public String getStatsString() { StringBuilder sb = new StringBuilder(); sb.append(" " + FormattingUtils.formatDecimalNumber(listOperations) diff --git a/src/main/java/org/torproject/onionoo/docs/DocumentStoreFactory.java b/src/main/java/org/torproject/onionoo/docs/DocumentStoreFactory.java index 3fa28fc..455a27f 100644 --- a/src/main/java/org/torproject/onionoo/docs/DocumentStoreFactory.java +++ b/src/main/java/org/torproject/onionoo/docs/DocumentStoreFactory.java @@ -7,10 +7,15 @@ public class DocumentStoreFactory { private static DocumentStore documentStoreInstance; + /** Sets a custom singleton document store instance that will be + * returned by {@link #getDocumentStore()} rather than creating an + * instance upon first invocation. */ public static void setDocumentStore(DocumentStore documentStore) { documentStoreInstance = documentStore; } + /** Returns the singleton document store instance that gets created upon + * first invocation of this method. */ public static DocumentStore getDocumentStore() { if (documentStoreInstance == null) { documentStoreInstance = new DocumentStore(); diff --git a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java index 630171b..afdd6c6 100644 --- a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java @@ -29,6 +29,9 @@ public class NodeStatus extends Document { private String contact; + /** Sets the contact to a lower-cased variant of the given string with + * all non-printable characters outside of ASCII code 32 (space) to 126 + * (dash) replaced with spaces. */ public void setContact(String contact) { if (contact == null) { this.contact = null; @@ -130,6 +133,9 @@ public class NodeStatus extends Document { : this.orAddressesAndPorts; } + /** Returns all addresses used for the onion-routing protocol which + * includes the primary address and all additionally configured + * onion-routing addresses. */ public SortedSet<String> getOrAddresses() { SortedSet<String> orAddresses = new TreeSet<String>(); if (this.address != null) { @@ -195,6 +201,7 @@ public class NodeStatus extends Document { private BitSet relayFlags; + @SuppressWarnings("checkstyle:javadocmethod") public void setRelayFlags(SortedSet<String> relayFlags) { BitSet newRelayFlags = new BitSet(relayFlagIndexes.size()); for (String relayFlag : relayFlags) { @@ -207,6 +214,7 @@ public class NodeStatus extends Document { this.relayFlags = newRelayFlags; } + @SuppressWarnings("checkstyle:javadocmethod") public SortedSet<String> getRelayFlags() { SortedSet<String> result = new TreeSet<String>(); if (this.relayFlags != null) { @@ -255,6 +263,9 @@ public class NodeStatus extends Document { return new TreeMap<Long, Set<String>>(this.lastAddresses); } + /** Adds addresses and ports together with the time in milliseconds + * since the epoch when they were last seen to the history of last seen + * addresses and ports. */ public void addLastAddresses(long lastSeenMillis, String address, int orPort, int dirPort, SortedSet<String> orAddressesAndPorts) { Set<String> addressesAndPorts = new HashSet<String>(); @@ -270,6 +281,8 @@ public class NodeStatus extends Document { } } + /** Returns the time in milliseconds since the epoch when addresses or + * ports were last changed. */ public long getLastChangedOrAddressOrPort() { long lastChangedAddressesMillis = -1L; if (this.lastAddresses != null) { @@ -369,6 +382,9 @@ public class NodeStatus extends Document { return stringArrayToSortedSet(this.extendedFamily); } + /** Returns the alleged family consisting of all relays in this relay's + * declared family that are not in a mutual family relationship with + * this relay. */ public SortedSet<String> getAllegedFamily() { SortedSet<String> allegedFamily = new TreeSet<String>( stringArrayToSortedSet(this.declaredFamily)); @@ -376,6 +392,9 @@ public class NodeStatus extends Document { return allegedFamily; } + /** Returns the indirect family consisting of all relays that can be + * reached via mutual family relationships except for those that can be + * reached directly via such a relationship. */ public SortedSet<String> getIndirectFamily() { SortedSet<String> indirectFamily = new TreeSet<String>( stringArrayToSortedSet(this.extendedFamily)); @@ -385,10 +404,13 @@ public class NodeStatus extends Document { /* Constructor and (de-)serialization methods: */ + /** Instantiates a new node status object from the given fingerprint. */ public NodeStatus(String fingerprint) { this.fingerprint = fingerprint; } + /** Instantiates a new node status object from the given string that may + * have been produced by {@link #toString()}. */ public static NodeStatus fromString(String documentString) { try { String[] parts = documentString.trim().split("\t"); @@ -529,6 +551,7 @@ public class NodeStatus extends Document { } } + @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append(this.isRelay ? "r" : "b"); diff --git a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java index cebcf5e..6a520c8 100644 --- a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java @@ -30,6 +30,9 @@ public class SummaryDocument extends Document { private String f; + /** Sets the fingerprint to the given 40 hex characters and clears + * SHA1-hashed and base64 fingerprints, so that they are re-computed at + * next request. */ public void setFingerprint(String fingerprint) { if (fingerprint != null) { Pattern fingerprintPattern = Pattern.compile("^[0-9a-fA-F]{40}$"); @@ -49,6 +52,8 @@ public class SummaryDocument extends Document { private transient String hashedFingerprint = null; + /** Returns the SHA1-hashed fingerprint, or <code>null</code> if no + * fingerprint is set. */ public String getHashedFingerprint() { if (this.hashedFingerprint == null && this.f != null) { try { @@ -63,6 +68,8 @@ public class SummaryDocument extends Document { private transient String base64Fingerprint = null; + /** Returns the base64-encoded fingerprint, or <code>null</code> if no + * fingerprint is set. */ public String getBase64Fingerprint() { if (this.base64Fingerprint == null && this.f != null) { try { @@ -77,6 +84,9 @@ public class SummaryDocument extends Document { private transient String[] fingerprintSortedHexBlocks = null; + /** Returns a sorted array containing blocks of 4 upper-case hex + * characters from the fingerprint, or <code>null</code> if no + * fingerprint is set. */ public String[] getFingerprintSortedHexBlocks() { if (this.fingerprintSortedHexBlocks == null && this.f != null) { String fingerprint = this.f.toUpperCase(); @@ -94,6 +104,7 @@ public class SummaryDocument extends Document { private String n; + @SuppressWarnings("checkstyle:javadocmethod") public void setNickname(String nickname) { if (nickname == null || nickname.equals("Unnamed")) { this.n = null; @@ -219,6 +230,7 @@ public class SummaryDocument extends Document { private String c; + @SuppressWarnings("checkstyle:javadocmethod") public void setContact(String contact) { if (contact != null && contact.length() == 0) { this.c = null; @@ -257,6 +269,7 @@ public class SummaryDocument extends Document { /* The familyFingerprints parameter can go away after September 8, 2015. * See above. */ + /** Instantiates a summary document with all given properties. */ public SummaryDocument(boolean isRelay, String nickname, String fingerprint, List<String> addresses, long lastSeenMillis, boolean running, SortedSet<String> relayFlags, long consensusWeight, diff --git a/src/main/java/org/torproject/onionoo/docs/UpdateStatus.java b/src/main/java/org/torproject/onionoo/docs/UpdateStatus.java index 07d5d20..4223337 100644 --- a/src/main/java/org/torproject/onionoo/docs/UpdateStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/UpdateStatus.java @@ -20,6 +20,7 @@ public class UpdateStatus extends Document { return this.updatedMillis; } + @Override public void setFromDocumentString(String documentString) { try { this.updatedMillis = Long.parseLong(documentString.trim()); @@ -30,6 +31,7 @@ public class UpdateStatus extends Document { } } + @Override public String toDocumentString() { return String.valueOf(this.updatedMillis); } diff --git a/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java b/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java index 6f1cb2e..3cd0a0e 100644 --- a/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java +++ b/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java @@ -38,6 +38,8 @@ public class UptimeHistory implements Comparable<UptimeHistory> { return this.flags; } + /** Instantiates a new uptime history object for a relay or bridge with + * the given interval start, uptime hours, and relay flags. */ UptimeHistory(boolean relay, long startMillis, int uptimeHours, SortedSet<String> flags) { this.relay = relay; @@ -46,6 +48,8 @@ public class UptimeHistory implements Comparable<UptimeHistory> { this.flags = flags; } + /** Instantiates a new uptime history object from the given string that + * may have been produced by {@link #toString()}. */ public static UptimeHistory fromString(String uptimeHistoryString) { String[] parts = uptimeHistoryString.split(" ", -1); if (parts.length < 3) { @@ -87,6 +91,7 @@ public class UptimeHistory implements Comparable<UptimeHistory> { return new UptimeHistory(relay, startMillis, uptimeHours, flags); } + @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append(this.relay ? (this.flags == null ? "r" : "R") : "b"); @@ -101,6 +106,10 @@ public class UptimeHistory implements Comparable<UptimeHistory> { return sb.toString(); } + /** Adds uptime hours from another uptime history object, which is + * assumed to either start right after this one or which ends right + * before it, and sets the interval start to the earlier interval + * start. */ public void addUptime(UptimeHistory other) { this.uptimeHours += other.uptimeHours; if (this.startMillis > other.startMillis) { @@ -108,6 +117,7 @@ public class UptimeHistory implements Comparable<UptimeHistory> { } } + @Override public int compareTo(UptimeHistory other) { if (this.relay && !other.relay) { return -1; @@ -118,12 +128,14 @@ public class UptimeHistory implements Comparable<UptimeHistory> { : this.startMillis > other.startMillis ? 1 : 0; } + @Override public boolean equals(Object other) { return other instanceof UptimeHistory && this.relay == ((UptimeHistory) other).relay && this.startMillis == ((UptimeHistory) other).startMillis; } + @Override public int hashCode() { return (int) this.startMillis + (this.relay ? 1 : 0); } diff --git a/src/main/java/org/torproject/onionoo/docs/UptimeStatus.java b/src/main/java/org/torproject/onionoo/docs/UptimeStatus.java index ce3a4d1..b725acc 100644 --- a/src/main/java/org/torproject/onionoo/docs/UptimeStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/UptimeStatus.java @@ -40,6 +40,7 @@ public class UptimeStatus extends Document { return this.bridgeHistory; } + @Override public void setFromDocumentString(String documentString) { try (Scanner s = new Scanner(documentString)) { while (s.hasNextLine()) { @@ -59,6 +60,8 @@ public class UptimeStatus extends Document { } } + /** Adds all given uptime history objects that don't overlap with + * existing uptime history objects. */ public void addToHistory(boolean relay, long startMillis, SortedSet<String> flags) { SortedSet<UptimeHistory> history = relay ? this.relayHistory @@ -105,6 +108,8 @@ public class UptimeStatus extends Document { this.isDirty = true; } + /** Compresses the history of uptime objects by merging adjacent + * intervals. */ public void compressHistory() { this.compressHistory(this.relayHistory); this.compressHistory(this.bridgeHistory); @@ -137,6 +142,7 @@ public class UptimeStatus extends Document { } } + @Override public String toDocumentString() { StringBuilder sb = new StringBuilder(); for (UptimeHistory interval : this.relayHistory) { diff --git a/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java b/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java index 47eaed9..efb7c25 100644 --- a/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java @@ -46,6 +46,7 @@ public class WeightsStatus extends Document { return this.history; } + @Override public void setFromDocumentString(String documentString) { try (Scanner s = new Scanner(documentString)) { while (s.hasNextLine()) { @@ -90,6 +91,8 @@ public class WeightsStatus extends Document { } } + /** Adds all given weights history objects that don't overlap with + * existing weights history objects. */ public void addToHistory(long validAfterMillis, long freshUntilMillis, double[] weights) { long[] interval = new long[] { validAfterMillis, freshUntilMillis }; @@ -104,6 +107,8 @@ public class WeightsStatus extends Document { } } + /** Compresses the history of weights objects by merging adjacent + * intervals, depending on how far back in the past they lie. */ public void compressHistory() { SortedMap<long[], double[]> uncompressedHistory = new TreeMap<long[], double[]>(this.history); @@ -176,6 +181,7 @@ public class WeightsStatus extends Document { } } + @Override public String toDocumentString() { StringBuilder sb = new StringBuilder(); for (Map.Entry<long[], double[]> e : history.entrySet()) { diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java index 30a6f0e..93b5af7 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java @@ -34,6 +34,7 @@ public class NodeIndexer implements ServletContextListener, Runnable { private static final Logger log = LoggerFactory.getLogger( NodeIndexer.class); + @Override public void contextInitialized(ServletContextEvent contextEvent) { ServletContext servletContext = contextEvent.getServletContext(); File outDir = new File(servletContext.getInitParameter("outDir")); @@ -50,6 +51,7 @@ public class NodeIndexer implements ServletContextListener, Runnable { this.startIndexing(); } + @Override public void contextDestroyed(ServletContextEvent contextEvent) { this.stopIndexing(); } @@ -60,6 +62,9 @@ public class NodeIndexer implements ServletContextListener, Runnable { private Thread nodeIndexerThread = null; + /** Returns the creation time of the last known node index in + * milliseconds since the epoch, or <code>-1</code> if no node index + * could be retrieved within <code>timeoutMillis</code> milliseconds. */ public synchronized long getLastIndexed(long timeoutMillis) { if (this.lastIndexed == -1L && this.nodeIndexerThread != null && timeoutMillis > 0L) { @@ -71,6 +76,8 @@ public class NodeIndexer implements ServletContextListener, Runnable { return this.lastIndexed; } + /** Returns the last known node index, or null if no node index could be + * retrieved within <code>timeoutMillis</code> milliseconds. */ public synchronized NodeIndex getLatestNodeIndex(long timeoutMillis) { if (this.latestNodeIndex == null && this.nodeIndexerThread != null && timeoutMillis > 0L) { @@ -82,6 +89,8 @@ public class NodeIndexer implements ServletContextListener, Runnable { return this.latestNodeIndex; } + /** Start reading the node index into memory periodically in a + * background thread. */ public synchronized void startIndexing() { if (this.nodeIndexerThread == null) { this.nodeIndexerThread = new Thread(this); @@ -94,6 +103,7 @@ public class NodeIndexer implements ServletContextListener, Runnable { private static final long ONE_DAY = 24L * 60L * ONE_MINUTE; + @Override public void run() { while (this.nodeIndexerThread != null) { this.indexNodeStatuses(); @@ -104,6 +114,8 @@ public class NodeIndexer implements ServletContextListener, Runnable { } } + /** Stop the background process that is periodically reading the node + * index. */ public synchronized void stopIndexing() { Thread indexerThread = this.nodeIndexerThread; this.nodeIndexerThread = null; diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexerFactory.java b/src/main/java/org/torproject/onionoo/server/NodeIndexerFactory.java index 1dfa859..b938c9e 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndexerFactory.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndexerFactory.java @@ -7,10 +7,15 @@ public class NodeIndexerFactory { private static NodeIndexer nodeIndexerInstance; + /** Sets a custom singleton node indexer instance that will be returned + * by {@link #getNodeIndexer()} rather than creating an instance upon + * first invocation. */ public static void setNodeIndexer(NodeIndexer nodeIndexer) { nodeIndexerInstance = nodeIndexer; } + /** Returns the singleton node indexer instance that gets created upon + * first invocation of this method. */ public static NodeIndexer getNodeIndexer() { if (nodeIndexerInstance == null) { nodeIndexerInstance = new NodeIndexer(); diff --git a/src/main/java/org/torproject/onionoo/server/PerformanceMetrics.java b/src/main/java/org/torproject/onionoo/server/PerformanceMetrics.java index a12e5c7..7adad76 100644 --- a/src/main/java/org/torproject/onionoo/server/PerformanceMetrics.java +++ b/src/main/java/org/torproject/onionoo/server/PerformanceMetrics.java @@ -30,6 +30,7 @@ class Counter { this.value++; } + @Override public String toString() { return String.valueOf(this.value); } @@ -52,6 +53,7 @@ class MostFrequentString { } } + @Override public String toString() { SortedMap<Integer, SortedSet<String>> sortedFrequencies = new TreeMap<Integer, SortedSet<String>>( @@ -98,6 +100,7 @@ class IntegerDistribution { logValues[64 - Long.numberOfLeadingZeros(value)]++; } + @Override public String toString() { StringBuilder sb = new StringBuilder(); int totalValues = 0; @@ -172,6 +175,9 @@ public class PerformanceMetrics { private static IntegerDistribution buildResponseMillis = new IntegerDistribution(); + /** Collects aggregate statistics on a given request for periodic + * request statistics, and logs requests taking longer than expected to + * process. */ public static void logStatistics(long receivedRequestMillis, String resourceType, Collection<String> parameterKeys, long parsedRequestMillis, int relayDocumentsWritten, diff --git a/src/main/java/org/torproject/onionoo/server/RequestHandler.java b/src/main/java/org/torproject/onionoo/server/RequestHandler.java index 85a6ff1..eaa4fe2 100644 --- a/src/main/java/org/torproject/onionoo/server/RequestHandler.java +++ b/src/main/java/org/torproject/onionoo/server/RequestHandler.java @@ -111,6 +111,7 @@ public class RequestHandler { private int[] firstSeenDays; + @SuppressWarnings("checkstyle:javadocmethod") public void setFirstSeenDays(int[] firstSeenDays) { this.firstSeenDays = new int[firstSeenDays.length]; System.arraycopy(firstSeenDays, 0, this.firstSeenDays, 0, @@ -119,6 +120,7 @@ public class RequestHandler { private int[] lastSeenDays; + @SuppressWarnings("checkstyle:javadocmethod") public void setLastSeenDays(int[] lastSeenDays) { this.lastSeenDays = new int[lastSeenDays.length]; System.arraycopy(lastSeenDays, 0, this.lastSeenDays, 0, @@ -137,6 +139,8 @@ public class RequestHandler { private Map<String, SummaryDocument> filteredBridges = new HashMap<String, SummaryDocument>(); + /** Handles this request by filtering by all given parameters and then + * possibly ordering, offsetting, and limiting results. */ public void handleRequest() { this.filteredRelays.putAll( this.nodeIndex.getRelayFingerprintSummaryLines()); diff --git a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java index 4fd38b7..5b3ab69 100644 --- a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java +++ b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java @@ -31,6 +31,7 @@ public class ResourceServlet extends HttpServlet { private boolean maintenanceMode = false; /* Called by servlet container, not by test class. */ + @Override public void init(ServletConfig config) throws ServletException { super.init(config); this.maintenanceMode = config.getInitParameter("maintenance") != null @@ -39,6 +40,7 @@ public class ResourceServlet extends HttpServlet { private static final long INDEX_WAITING_TIME = 10L * 1000L; + @Override public long getLastModified(HttpServletRequest request) { if (this.maintenanceMode) { return super.getLastModified(request); @@ -48,6 +50,7 @@ public class ResourceServlet extends HttpServlet { } } + @Override public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { HttpServletRequestWrapper requestWrapper = @@ -72,6 +75,9 @@ public class ResourceServlet extends HttpServlet { new HashSet<String>(Arrays.asList(("search,fingerprint,order,limit," + "offset,fields").split(","))); + /** Handles the HTTP GET request in the wrapped <code>request</code> by + * writing an HTTP GET response to the likewise <code>response</code>, + * both of which are wrapped to facilitate testing. */ public void doGet(HttpServletRequestWrapper request, HttpServletResponseWrapper response) throws IOException { diff --git a/src/main/java/org/torproject/onionoo/server/ServerMain.java b/src/main/java/org/torproject/onionoo/server/ServerMain.java index 22e315b..fd4dc90 100644 --- a/src/main/java/org/torproject/onionoo/server/ServerMain.java +++ b/src/main/java/org/torproject/onionoo/server/ServerMain.java @@ -14,6 +14,7 @@ public class ServerMain { private static final Logger log = LoggerFactory.getLogger( ServerMain.class); + /** Starts the web server listening for incoming client connections. */ public static void main(String[] args) { try { Resource onionooXml = Resource.newSystemResource("jetty.xml"); diff --git a/src/main/java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java index 1c0c181..3aa3dd1 100644 --- a/src/main/java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java @@ -16,6 +16,9 @@ public class BandwidthStatusUpdater implements DescriptorListener, private DocumentStore documentStore; + /** Initializes a new status updater, obtains references to all relevant + * singleton instances, and registers as listener at the (singleton) + * descriptor source. */ public BandwidthStatusUpdater() { this.descriptorSource = DescriptorSourceFactory.getDescriptorSource(); this.documentStore = DocumentStoreFactory.getDocumentStore(); @@ -29,12 +32,14 @@ public class BandwidthStatusUpdater implements DescriptorListener, DescriptorType.BRIDGE_EXTRA_INFOS); } + @Override public void processDescriptor(Descriptor descriptor, boolean relay) { if (descriptor instanceof ExtraInfoDescriptor) { this.parseDescriptor((ExtraInfoDescriptor) descriptor); } } + @Override public void updateStatuses() { /* Status files are already updated while processing descriptors. */ } @@ -59,6 +64,7 @@ public class BandwidthStatusUpdater implements DescriptorListener, } } + @Override public String getStatsString() { /* TODO Add statistics string. */ return null; diff --git a/src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java index 492b67b..28b8ea7 100644 --- a/src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java @@ -43,6 +43,9 @@ public class ClientsStatusUpdater implements DescriptorListener, private DocumentStore documentStore; + /** Initializes a new status updater, obtains references to all relevant + * singleton instances, and registers as listener at the (singleton) + * descriptor source. */ public ClientsStatusUpdater() { this.descriptorSource = DescriptorSourceFactory.getDescriptorSource(); this.documentStore = DocumentStoreFactory.getDocumentStore(); @@ -54,6 +57,7 @@ public class ClientsStatusUpdater implements DescriptorListener, DescriptorType.BRIDGE_EXTRA_INFOS); } + @Override public void processDescriptor(Descriptor descriptor, boolean relay) { if (descriptor instanceof ExtraInfoDescriptor && !relay) { this.processBridgeExtraInfoDescriptor( @@ -143,6 +147,7 @@ public class ClientsStatusUpdater implements DescriptorListener, return weightedResponses; } + @Override public void updateStatuses() { for (Map.Entry<String, SortedSet<ClientsHistory>> e : this.newResponses.entrySet()) { @@ -161,6 +166,7 @@ public class ClientsStatusUpdater implements DescriptorListener, } } + @Override public String getStatsString() { int newIntervals = 0; for (SortedSet<ClientsHistory> hist : this.newResponses.values()) { diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java index b9d07b4..176a17c 100644 --- a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java +++ b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java @@ -32,6 +32,7 @@ public class DescriptorSource { private DescriptorQueue archiveDescriptorQueue; + /** Instantiates a new descriptor source. */ public DescriptorSource() { this.descriptorQueues = new ArrayList<DescriptorQueue>(); this.descriptorListeners = @@ -53,6 +54,7 @@ public class DescriptorSource { private Map<DescriptorType, Set<DescriptorListener>> descriptorListeners; + /** Registers a descriptor listener for a given descriptor type. */ public void registerDescriptorListener(DescriptorListener listener, DescriptorType descriptorType) { if (!this.descriptorListeners.containsKey(descriptorType)) { @@ -62,6 +64,7 @@ public class DescriptorSource { this.descriptorListeners.get(descriptorType).add(listener); } + /** Downloads descriptors from CollecTor. */ public void downloadDescriptors() { for (DescriptorType descriptorType : DescriptorType.values()) { log.info("Loading: " + descriptorType); @@ -87,6 +90,8 @@ public class DescriptorSource { this.deletedLocalFiles += descriptorDownloader.deleteOldLocalFiles(); } + /** Reads archived and recent descriptors from disk and feeds them into + * any registered listeners. */ public void readDescriptors() { this.readArchivedDescriptors(); log.debug("Reading recent " + DescriptorType.RELAY_SERVER_DESCRIPTORS @@ -154,6 +159,8 @@ public class DescriptorSource { } } + /** Reads archived descriptors from disk and feeds them into any + * registered listeners. */ public void readArchivedDescriptors() { if (!this.inArchiveDir.exists()) { return; @@ -208,6 +215,7 @@ public class DescriptorSource { log.info("Read archived descriptors"); } + /** Writes parse histories for recent descriptors to disk. */ public void writeHistoryFiles() { log.debug("Writing parse histories for recent descriptors..."); for (DescriptorQueue descriptorQueue : this.descriptorQueues) { @@ -215,6 +223,8 @@ public class DescriptorSource { } } + /** Returns a string with statistics on the number of processed + * descriptors during the current execution. */ public String getStatsString() { StringBuilder sb = new StringBuilder(); sb.append(" " + this.localFilesBefore + " recent descriptor files " diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorSourceFactory.java b/src/main/java/org/torproject/onionoo/updater/DescriptorSourceFactory.java index 65c5622..4974488 100644 --- a/src/main/java/org/torproject/onionoo/updater/DescriptorSourceFactory.java +++ b/src/main/java/org/torproject/onionoo/updater/DescriptorSourceFactory.java @@ -7,11 +7,16 @@ public class DescriptorSourceFactory { private static DescriptorSource descriptorSourceInstance; + /** Sets a custom singleton descriptor source instance that will be + * returned by {@link #getDescriptorSource()} rather than creating an + * instance upon first invocation. */ public static void setDescriptorSource( DescriptorSource descriptorSource) { descriptorSourceInstance = descriptorSource; } + /** Returns the singleton descriptor source instance that gets created + * upon first invocation of this method. */ public static DescriptorSource getDescriptorSource() { if (descriptorSourceInstance == null) { descriptorSourceInstance = new DescriptorSource(); diff --git a/src/main/java/org/torproject/onionoo/updater/LookupService.java b/src/main/java/org/torproject/onionoo/updater/LookupService.java index 85d56e8..2b0993f 100644 --- a/src/main/java/org/torproject/onionoo/updater/LookupService.java +++ b/src/main/java/org/torproject/onionoo/updater/LookupService.java @@ -97,6 +97,9 @@ public class LookupService { return addressNumber; } + /** Looks up address strings in the configured + * <code>GeoLite2-City-*.csv</code> and <code>GeoIPASNum2.csv</code> + * files and returns all lookup results. */ public SortedMap<String, LookupResult> lookup( SortedSet<String> addressStrings) { @@ -363,6 +366,8 @@ public class LookupService { private int addressesResolved = 0; + /** Returns a string with the number of addresses looked up and + * resolved. */ public String getStatsString() { StringBuilder sb = new StringBuilder(); sb.append(" " + FormattingUtils.formatDecimalNumber( diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java index 9bceaa1..d873072 100644 --- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java @@ -95,6 +95,9 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, private int bridgeStatusesProcessed = 0; + /** Initializes a new status updater, obtains references to all relevant + * singleton instances, and registers as listener at the (singleton) + * descriptor source. */ public NodeDetailsStatusUpdater( ReverseDomainNameResolver reverseDomainNameResolver, LookupService lookupService) { @@ -125,6 +128,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, private SortedSet<String> updatedNodes = new TreeSet<String>(); + @Override public void processDescriptor(Descriptor descriptor, boolean relay) { if (descriptor instanceof ServerDescriptor && relay) { this.processRelayServerDescriptor((ServerDescriptor) descriptor); @@ -385,6 +389,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, this.bridgeStatusesProcessed++; } + @Override public void updateStatuses() { this.readNodeStatuses(); log.info("Read node statuses"); @@ -927,6 +932,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, } } + @Override public String getStatsString() { StringBuilder sb = new StringBuilder(); sb.append(" " + FormattingUtils.formatDecimalNumber( diff --git a/src/main/java/org/torproject/onionoo/updater/RdnsLookupRequest.java b/src/main/java/org/torproject/onionoo/updater/RdnsLookupRequest.java index e2ee5e1..75ad315 100644 --- a/src/main/java/org/torproject/onionoo/updater/RdnsLookupRequest.java +++ b/src/main/java/org/torproject/onionoo/updater/RdnsLookupRequest.java @@ -28,6 +28,7 @@ class RdnsLookupRequest extends Thread { this.address = address; } + @Override public void run() { this.lookupStartedMillis = this.reverseDomainNameResolver.time.currentTimeMillis(); diff --git a/src/main/java/org/torproject/onionoo/updater/RdnsLookupWorker.java b/src/main/java/org/torproject/onionoo/updater/RdnsLookupWorker.java index 0c48be5..165507b 100644 --- a/src/main/java/org/torproject/onionoo/updater/RdnsLookupWorker.java +++ b/src/main/java/org/torproject/onionoo/updater/RdnsLookupWorker.java @@ -11,6 +11,7 @@ class RdnsLookupWorker extends Thread { this.reverseDomainNameResolver = reverseDomainNameResolver; } + @Override public void run() { while (this.reverseDomainNameResolver.time.currentTimeMillis() - ReverseDomainNameResolver.RDNS_LOOKUP_MAX_DURATION_MILLIS diff --git a/src/main/java/org/torproject/onionoo/updater/ReverseDomainNameResolver.java b/src/main/java/org/torproject/onionoo/updater/ReverseDomainNameResolver.java index 0f3caad..3214c74 100644 --- a/src/main/java/org/torproject/onionoo/updater/ReverseDomainNameResolver.java +++ b/src/main/java/org/torproject/onionoo/updater/ReverseDomainNameResolver.java @@ -48,6 +48,8 @@ public class ReverseDomainNameResolver { this.addressLastLookupTimes = addressLastLookupTimes; } + /** Starts reverse domain name lookups in one or more background + * threads and returns immediately. */ public void startReverseDomainNameLookups() { this.startedRdnsLookups = this.time.currentTimeMillis(); this.rdnsLookupJobs = new HashSet<String>(); @@ -69,6 +71,8 @@ public class ReverseDomainNameResolver { } } + /** Joins all background threads performing reverse domain name lookups + * and returns as soon as they have all finished. */ public void finishReverseDomainNameLookups() { for (RdnsLookupWorker rdnsLookupWorker : this.rdnsLookupWorkers) { try { @@ -80,16 +84,21 @@ public class ReverseDomainNameResolver { } } + /** Returns reverse domain name lookup results. */ public Map<String, String> getLookupResults() { synchronized (this.rdnsLookupResults) { return new HashMap<String, String>(this.rdnsLookupResults); } } + /** Returns the time in milliseconds since the epoch when reverse domain + * lookups have been started. */ public long getLookupStartMillis() { return this.startedRdnsLookups; } + /** Returns a string with the number of performed reverse domain name + * lookups and some simple statistics on lookup time. */ public String getStatsString() { StringBuilder sb = new StringBuilder(); sb.append(" " + FormattingUtils.formatDecimalNumber( diff --git a/src/main/java/org/torproject/onionoo/updater/StatusUpdateRunner.java b/src/main/java/org/torproject/onionoo/updater/StatusUpdateRunner.java index 2ba8401..7aa1a5e 100644 --- a/src/main/java/org/torproject/onionoo/updater/StatusUpdateRunner.java +++ b/src/main/java/org/torproject/onionoo/updater/StatusUpdateRunner.java @@ -19,6 +19,8 @@ public class StatusUpdateRunner { private StatusUpdater[] statusUpdaters; + /** Instantiates a new status update runner with newly created instances + * of all known status updater implementations. */ public StatusUpdateRunner() { this.ls = new LookupService(new File("geoip")); this.rdnr = new ReverseDomainNameResolver(); @@ -32,6 +34,7 @@ public class StatusUpdateRunner { usu }; } + /** Lets each configured status updater update its status files. */ public void updateStatuses() { for (StatusUpdater su : this.statusUpdaters) { log.debug("Begin update of " + su.getClass().getSimpleName()); @@ -41,6 +44,7 @@ public class StatusUpdateRunner { } } + /** Logs statistics of all configured status updaters. */ public void logStatistics() { for (StatusUpdater su : this.statusUpdaters) { String statsString = su.getStatsString(); diff --git a/src/main/java/org/torproject/onionoo/updater/UptimeStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/UptimeStatusUpdater.java index f2d7dc1..d4951b7 100644 --- a/src/main/java/org/torproject/onionoo/updater/UptimeStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/UptimeStatusUpdater.java @@ -28,6 +28,9 @@ public class UptimeStatusUpdater implements DescriptorListener, private DocumentStore documentStore; + /** Initializes a new status updater, obtains references to all relevant + * singleton instances, and registers as listener at the (singleton) + * descriptor source. */ public UptimeStatusUpdater() { this.descriptorSource = DescriptorSourceFactory.getDescriptorSource(); this.documentStore = DocumentStoreFactory.getDocumentStore(); @@ -41,6 +44,7 @@ public class UptimeStatusUpdater implements DescriptorListener, DescriptorType.BRIDGE_STATUSES); } + @Override public void processDescriptor(Descriptor descriptor, boolean relay) { if (descriptor instanceof RelayNetworkStatusConsensus) { this.processRelayNetworkStatusConsensus( @@ -132,6 +136,7 @@ public class UptimeStatusUpdater implements DescriptorListener, } } + @Override public void updateStatuses() { for (Map.Entry<String, SortedMap<Long, Flags>> e : this.newRunningRelays.entrySet()) { @@ -179,6 +184,7 @@ public class UptimeStatusUpdater implements DescriptorListener, } } + @Override public String getStatsString() { StringBuilder sb = new StringBuilder(); sb.append(" " + FormattingUtils.formatDecimalNumber( diff --git a/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java index 14e425b..2442a00 100644 --- a/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java @@ -24,6 +24,9 @@ public class WeightsStatusUpdater implements DescriptorListener, private DocumentStore documentStore; + /** Initializes a new status updater, obtains references to all relevant + * singleton instances, and registers as listener at the (singleton) + * descriptor source. */ public WeightsStatusUpdater() { this.descriptorSource = DescriptorSourceFactory.getDescriptorSource(); this.documentStore = DocumentStoreFactory.getDocumentStore(); @@ -35,6 +38,7 @@ public class WeightsStatusUpdater implements DescriptorListener, DescriptorType.RELAY_CONSENSUSES); } + @Override public void processDescriptor(Descriptor descriptor, boolean relay) { if (descriptor instanceof RelayNetworkStatusConsensus) { this.processRelayNetworkConsensus( @@ -42,6 +46,7 @@ public class WeightsStatusUpdater implements DescriptorListener, } } + @Override public void updateStatuses() { /* Nothing to do. */ } @@ -193,6 +198,7 @@ public class WeightsStatusUpdater implements DescriptorListener, return pathSelectionProbabilities; } + @Override public String getStatsString() { /* TODO Add statistics string. */ return null; diff --git a/src/main/java/org/torproject/onionoo/util/FormattingUtils.java b/src/main/java/org/torproject/onionoo/util/FormattingUtils.java index 36fb838..fd83cf8 100644 --- a/src/main/java/org/torproject/onionoo/util/FormattingUtils.java +++ b/src/main/java/org/torproject/onionoo/util/FormattingUtils.java @@ -12,11 +12,14 @@ public class FormattingUtils { private static final long ONE_MINUTE = 60L * ONE_SECOND; + /** Formats the given number of milliseconds using the format + * <code>"${minutes}:${seconds}.{milliseconds} minutes"</code>. */ public static String formatMillis(long millis) { return String.format("%02d:%02d.%03d minutes", millis / ONE_MINUTE, (millis % ONE_MINUTE) / ONE_SECOND, millis % ONE_SECOND); } + /** Formats the given number of bytes as B, KiB, MiB, GiB, etc. */ public static String formatBytes(long bytes) { if (bytes < 1024) { return bytes + " B"; @@ -27,6 +30,8 @@ public class FormattingUtils { } } + /** Formats the given decimal number with a comma as thousands + * separator. */ public static String formatDecimalNumber(long decimalNumber) { return String.format("%,d", decimalNumber); } diff --git a/src/main/java/org/torproject/onionoo/util/TimeFactory.java b/src/main/java/org/torproject/onionoo/util/TimeFactory.java index e3067a4..608b595 100644 --- a/src/main/java/org/torproject/onionoo/util/TimeFactory.java +++ b/src/main/java/org/torproject/onionoo/util/TimeFactory.java @@ -7,10 +7,15 @@ public class TimeFactory { private static Time timeInstance; + /** Sets a custom singleton time instance that will be returned by + * {@link #getTime} rather than creating an instance upon first + * invocation. */ public static void setTime(Time time) { timeInstance = time; } + /** Returns the singleton node indexer instance that gets created upon + * first invocation of this method. */ public static Time getTime() { if (timeInstance == null) { timeInstance = new Time(); diff --git a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java index a844f7a..7238c1b 100644 --- a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java @@ -36,6 +36,7 @@ public class BandwidthDocumentWriter implements DocumentWriter { this.now = TimeFactory.getTime().currentTimeMillis(); } + @Override public void writeDocuments() { UpdateStatus updateStatus = this.documentStore.retrieve( UpdateStatus.class, true); @@ -196,6 +197,7 @@ public class BandwidthDocumentWriter implements DocumentWriter { return graphs; } + @Override public String getStatsString() { /* TODO Add statistics string. */ return null; diff --git a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java index 6cbd2e0..00389d0 100644 --- a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java @@ -64,6 +64,7 @@ public class ClientsDocumentWriter implements DocumentWriter { private int writtenDocuments = 0; + @Override public void writeDocuments() { UpdateStatus updateStatus = this.documentStore.retrieve( UpdateStatus.class, true); @@ -286,6 +287,7 @@ public class ClientsDocumentWriter implements DocumentWriter { } } + @Override public String getStatsString() { StringBuilder sb = new StringBuilder(); sb.append(" " + FormattingUtils.formatDecimalNumber( diff --git a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java index 44c675f..c167152 100644 --- a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java @@ -31,6 +31,7 @@ public class DetailsDocumentWriter implements DocumentWriter { this.documentStore = DocumentStoreFactory.getDocumentStore(); } + @Override public void writeDocuments() { UpdateStatus updateStatus = this.documentStore.retrieve( UpdateStatus.class, true); @@ -186,6 +187,7 @@ public class DetailsDocumentWriter implements DocumentWriter { this.documentStore.store(detailsDocument, fingerprint); } + @Override public String getStatsString() { /* TODO Add statistics string. */ return null; diff --git a/src/main/java/org/torproject/onionoo/writer/DocumentWriterRunner.java b/src/main/java/org/torproject/onionoo/writer/DocumentWriterRunner.java index 4809ed9..d9d1c47 100644 --- a/src/main/java/org/torproject/onionoo/writer/DocumentWriterRunner.java +++ b/src/main/java/org/torproject/onionoo/writer/DocumentWriterRunner.java @@ -13,6 +13,8 @@ public class DocumentWriterRunner { private DocumentWriter[] documentWriters; + /** Instantiates a new document writer runner with newly created + * instances of all known document writer implementations. */ public DocumentWriterRunner() { SummaryDocumentWriter sdw = new SummaryDocumentWriter(); DetailsDocumentWriter ddw = new DetailsDocumentWriter(); @@ -24,6 +26,7 @@ public class DocumentWriterRunner { udw }; } + /** Lets each configured document writer write its documents. */ public void writeDocuments() { for (DocumentWriter dw : this.documentWriters) { log.debug("Writing " + dw.getClass().getSimpleName()); @@ -31,6 +34,7 @@ public class DocumentWriterRunner { } } + /** Logs statistics of all configured document writers. */ public void logStatistics() { for (DocumentWriter dw : this.documentWriters) { String statsString = dw.getStatsString(); diff --git a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java index f0d7fe0..f941ee3 100644 --- a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java @@ -32,6 +32,7 @@ public class SummaryDocumentWriter implements DocumentWriter { private int deletedDocuments = 0; + @Override public void writeDocuments() { long relaysLastValidAfterMillis = -1L; long bridgesLastPublishedMillis = -1L; @@ -101,6 +102,7 @@ public class SummaryDocumentWriter implements DocumentWriter { log.info("Wrote summary document files"); } + @Override public String getStatsString() { StringBuilder sb = new StringBuilder(); sb.append(" " + FormattingUtils.formatDecimalNumber( diff --git a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java index e93627b..502c351 100644 --- a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java @@ -40,6 +40,7 @@ public class UptimeDocumentWriter implements DocumentWriter { this.now = TimeFactory.getTime().currentTimeMillis(); } + @Override public void writeDocuments() { UptimeStatus uptimeStatus = this.documentStore.retrieve( UptimeStatus.class, true); @@ -318,6 +319,7 @@ public class UptimeDocumentWriter implements DocumentWriter { } } + @Override public String getStatsString() { StringBuilder sb = new StringBuilder(); sb.append(" " + FormattingUtils.formatDecimalNumber( diff --git a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java index cabe964..326fbda 100644 --- a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java @@ -36,6 +36,7 @@ public class WeightsDocumentWriter implements DocumentWriter { this.now = TimeFactory.getTime().currentTimeMillis(); } + @Override public void writeDocuments() { UpdateStatus updateStatus = this.documentStore.retrieve( UpdateStatus.class, true); @@ -210,6 +211,7 @@ public class WeightsDocumentWriter implements DocumentWriter { } } + @Override public String getStatsString() { /* TODO Add statistics string. */ return null; diff --git a/src/test/resources/metrics_checks.xml b/src/test/resources/metrics_checks.xml index 0777894..a4af08a 100644 --- a/src/test/resources/metrics_checks.xml +++ b/src/test/resources/metrics_checks.xml @@ -34,6 +34,7 @@ <property name="eachLine" value="true"/> </module> + <module name="SuppressWarningsFilter" /> <module name="TreeWalker"> <module name="OuterTypeFilename"/> <module name="IllegalTokenText"> @@ -213,5 +214,6 @@ <property name="exceptionVariableName" value="expected"/> </module> <module name="CommentsIndentation"/> + <module name="SuppressWarningsHolder" /> </module> </module>

1 0

[tor/master] Also ship compat_time.h in release tarballs. Fixes bug #19746
by nickm＠torproject.org 25 Jul '16

25 Jul '16

commit 518c8fe0ec375fe852e350ef4455ec8defd8c6b0 Author: Peter Palfrader <peter(a)palfrader.org> Date: Mon Jul 25 09:07:29 2016 +0200 Also ship compat_time.h in release tarballs. Fixes bug #19746 --- src/common/include.am | 1 + 1 file changed, 1 insertion(+) diff --git a/src/common/include.am b/src/common/include.am index b022680..40c463c 100644 --- a/src/common/include.am +++ b/src/common/include.am @@ -142,6 +142,7 @@ COMMONHEADERS = \ src/common/compat_libevent.h \ src/common/compat_openssl.h \ src/common/compat_threads.h \ + src/common/compat_time.h \ src/common/container.h \ src/common/crypto.h \ src/common/crypto_curve25519.h \

1 0

[stem/master] Add encoding() method to stem.util.term
by atagar＠torproject.org 24 Jul '16

24 Jul '16

commit 50d884e9c9a732210825bd9079c8303ef65d10aa Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jul 24 09:22:25 2016 -0700 Add encoding() method to stem.util.term Breaking up our format() method a little by adding a helper to get ANSI escape sequences. This is something I need for nyx right now anyway... Oddly our stem.util.term module didn't have any direct test coverage. It was tested tangentially by our testing harness and interpreter tests but strange I missed unit test coverage for this one. It's an easy module to include. Oh well - threw in some basic tests. --- docs/change_log.rst | 1 + stem/util/term.py | 57 +++++++++++++++++++++++++++++++++----------------- test/settings.cfg | 1 + test/unit/util/term.py | 30 ++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 19 deletions(-) diff --git a/docs/change_log.rst b/docs/change_log.rst index 98dc18c..839c7c9 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -94,6 +94,7 @@ The following are only available within Stem's `git repository * Added an **is_ipv6** value to :class:`~stem.util.connection.Connection` instances * Added :func:`~stem.util.system.pids_by_user` * Added :func:`~stem.util.connection.address_to_int` + * Added :func:`~stem.util.term.encoding` * Added :func:`~stem.util.__init__.datetime_to_unix` * **Interpreter** diff --git a/stem/util/term.py b/stem/util/term.py index 3287bb1..a32fe93 100644 --- a/stem/util/term.py +++ b/stem/util/term.py @@ -8,6 +8,7 @@ Utilities for working with the terminal. :: + encoding - provides the ANSI escape sequence for a terminal attribute format - wrap text with ANSI for the given colors or attributes .. data:: Color (enum) @@ -65,6 +66,41 @@ CSI = '\x1B[%sm' RESET = CSI % '0' +def encoding(*attrs): + """ + Provides the ANSI escape sequence for these terminal color or attributes. + + .. versionadded:: 1.5.0 + + :param list attr: :data:`~stem.util.terminal.Color`, + :data:`~stem.util.terminal.BgColor`, or :data:`~stem.util.terminal.Attr` to + provide an ecoding for + + :return: **str** of the ANSI escape sequence, **None** no attributes are + recognized + """ + + term_encodings = [] + + for attr in attrs: + # TODO: Account for an earlier misspelled attribute. This should be dropped + # in Stem. 2.0.x. + + if attr == 'HILIGHT': + attr = 'HIGHLIGHT' + + attr = stem.util.str_tools._to_camel_case(attr) + term_encoding = FG_ENCODING.get(attr, None) + term_encoding = BG_ENCODING.get(attr, term_encoding) + term_encoding = ATTR_ENCODING.get(attr, term_encoding) + + if term_encoding: + term_encodings.append(term_encoding) + + if term_encodings: + return CSI % ';'.join(term_encodings) + + def format(msg, *attr): """ Simple terminal text formatting using `ANSI escape sequences @@ -93,26 +129,9 @@ def format(msg, *attr): if RESET in msg: return ''.join([format(comp, *attr) for comp in msg.split(RESET)]) - encodings = [] - - for text_attr in attr: - # TODO: Account for an earlier misspelled attribute. This should be dropped - # in Stem. 2.0.x. - - if text_attr == 'HILIGHT': - text_attr = 'HIGHLIGHT' - - text_attr, encoding = stem.util.str_tools._to_camel_case(text_attr), None - encoding = FG_ENCODING.get(text_attr, encoding) - encoding = BG_ENCODING.get(text_attr, encoding) - encoding = ATTR_ENCODING.get(text_attr, encoding) - - if encoding: - encodings.append(encoding) - - if encodings: - prefix, suffix = CSI % ';'.join(encodings), RESET + prefix, suffix = encoding(*attr), RESET + if prefix: if Attr.READLINE_ESCAPE in attr: prefix = '\001%s\002' % prefix suffix = '\001%s\002' % suffix diff --git a/test/settings.cfg b/test/settings.cfg index fd199e1..10e2ccf 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -167,6 +167,7 @@ test.unit_tests |test.unit.util.proc.TestProc |test.unit.util.str_tools.TestStrTools |test.unit.util.system.TestSystem +|test.unit.util.term.TestTerminal |test.unit.util.tor_tools.TestTorTools |test.unit.util.__init__.TestBaseUtil |test.unit.descriptor.export.TestExport diff --git a/test/unit/util/term.py b/test/unit/util/term.py new file mode 100644 index 0000000..758db22 --- /dev/null +++ b/test/unit/util/term.py @@ -0,0 +1,30 @@ +""" +Unit tests for the stem.util.term functions. +""" + +import unittest + +import stem.util.term + +from stem.util.term import Color, Attr + + +class TestTerminal(unittest.TestCase): + def test_encoding(self): + """ + Exercises our encoding function. + """ + + self.assertEqual(None, stem.util.term.encoding()) + self.assertEqual('\x1b[31m', stem.util.term.encoding(Color.RED)) + self.assertEqual('\x1b[31;1m', stem.util.term.encoding(Color.RED, Attr.BOLD)) + + def test_format(self): + """ + Exercises our format function. + """ + + self.assertEqual('hi!', stem.util.term.format('hi!')) + self.assertEqual('\x1b[31mhi!\x1b[0m', stem.util.term.format('hi!', Color.RED)) + self.assertEqual('\x1b[31;1mhi!\x1b[0m', stem.util.term.format('hi!', Color.RED, Attr.BOLD)) + self.assertEqual('\001\x1b[31m\002hi!\001\x1b[0m\002', stem.util.term.format('hi!', Color.RED, Attr.READLINE_ESCAPE))

1 0

[tor-browser-bundle/maint-6.0] Release preparations for 6.0.3
by gk＠torproject.org 22 Jul '16

22 Jul '16

commit 033fe948a692f545d50132ebe2046704be5dff7d Author: Georg Koppen <gk(a)torproject.org> Date: Fri Jul 22 16:40:03 2016 +0000 Release preparations for 6.0.3 --- Bundle-Data/Docs/ChangeLog.txt | 18 ++++++++++++++++++ gitian/versions | 2 +- tools/update-responses/config.yml | 13 ++++++------- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/Bundle-Data/Docs/ChangeLog.txt b/Bundle-Data/Docs/ChangeLog.txt index 6325c23..2abd47e 100644 --- a/Bundle-Data/Docs/ChangeLog.txt +++ b/Bundle-Data/Docs/ChangeLog.txt @@ -1,3 +1,21 @@ +Tor Browser 6.0.3 -- August 2 + * All Platforms + * Update Firefox to 45.3.0esr + * Update Torbutton to 1.9.5.6 + * Bug 19417: Disable asmjs for now + * Bug 19689: Use proper parent window for plugin prompt + * Update HTTPS-Everywhere to 5.2.1 + * Bug 19417: Disable asmjs for now + * Bug 19715: Disable the meek-google pluggable transport option + * Bug 19714: Remove mercurius4 obfs4 bridge + * Bug 19585: Fix regression test for keyboard layout fingerprinting + * Bug 19515: Tor Browser is crashing in graphics code + * Bug 18513: Favicon requests can bypass New Identity + * OS X + * Bug 19269: Icon doesn't appear in Applications folder or Dock + * Android + * Bug 19484: Avoid compilation error when MOZ_UPDATER is not defined + Tor Browser 6.0.2 -- June 21 * All Platforms * Update Torbutton to 1.9.5.5 diff --git a/gitian/versions b/gitian/versions index 6e54edd..c46c735 100755 --- a/gitian/versions +++ b/gitian/versions @@ -18,7 +18,7 @@ TORBROWSER_TAG=tor-browser-${FIREFOX_VERSION}-6.0-1-build2 TOR_TAG=tor-0.2.7.6 TORLAUNCHER_TAG=0.2.9.3 TORBUTTON_TAG=1.9.5.5 -HTTPSE_TAG=5.1.9 +HTTPSE_TAG=5.2.1 NSIS_TAG=v0.3.1 ZLIB_TAG=v1.2.8 LIBEVENT_TAG=release-2.0.22-stable diff --git a/tools/update-responses/config.yml b/tools/update-responses/config.yml index 4e89b8a..16843df 100644 --- a/tools/update-responses/config.yml +++ b/tools/update-responses/config.yml @@ -13,15 +13,14 @@ build_targets: osx64: Darwin_x86_64-gcc3 channels: alpha: 6.0a5 - release: 6.0.2 + release: 6.0.3 versions: - 6.0.2: - platformVersion: 45.2.0 - detailsURL: https://blog.torproject.org/blog/tor-browser-602-released - download_url: https://cdn.torproject.org/aus1/torbrowser/6.0.2 + 6.0.3: + platformVersion: 45.3.0 + detailsURL: https://blog.torproject.org/blog/tor-browser-603-released + download_url: https://cdn.torproject.org/aus1/torbrowser/6.0.3 incremental_from: - - 6.0 - - 6.0.1 + - 6.0.2 migrate_archs: osx32: osx64 osx32:

1 0

[torbutton/maint-1.9.5] Release preparations for 1.9.5.6
by gk＠torproject.org 22 Jul '16

22 Jul '16

commit 7b983fe74fca4bc29d68e0a4ed3c2eeae2ba1113 Author: Georg Koppen <gk(a)torproject.org> Date: Fri Jul 22 14:33:41 2016 +0000 Release preparations for 1.9.5.6 --- src/CHANGELOG | 4 ++++ src/install.rdf | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/CHANGELOG b/src/CHANGELOG index 77ba290..28cff54 100644 --- a/src/CHANGELOG +++ b/src/CHANGELOG @@ -1,3 +1,7 @@ +1.9.5.6 + * Bug 19417: Disable asmjs for now + * Bug 19689: Use proper parent windows for plugin prompt + 1.9.5.5 * Bug 19417: Clear asmjscache diff --git a/src/install.rdf b/src/install.rdf index ac45b1d..9098991 100644 --- a/src/install.rdf +++ b/src/install.rdf @@ -6,7 +6,7 @@ <em:name>Torbutton</em:name> <em:creator>Mike Perry</em:creator> <em:id>torbutton(a)torproject.org</em:id> - <em:version>1.9.5.5</em:version> + <em:version>1.9.5.6</em:version> <em:homepageURL>https://www.torproject.org/projects/torbrowser.html.en</em:homepageURL> <em:optionsURL>chrome://torbutton/content/preferences.xul</em:optionsURL> <em:iconURL>chrome://torbutton/skin/tor.png</em:iconURL>

1 0

[collector/master] Implements task-19727: make exitlist url configurable and correct default setting.
by karsten＠torproject.org 22 Jul '16

22 Jul '16

commit 7dc17f8e14b3e87f26bd34e1d7c4649546e3476a Author: iwakeh <iwakeh(a)torproject.org> Date: Fri Jul 22 09:13:28 2016 +0200 Implements task-19727: make exitlist url configurable and correct default setting. --- .../org/torproject/collector/conf/Configuration.java | 16 ++++++++++++++++ src/main/java/org/torproject/collector/conf/Key.java | 2 ++ .../collector/exitlists/ExitListDownloader.java | 4 +--- src/main/resources/collector.properties | 1 + .../org/torproject/collector/conf/ConfigurationTest.java | 9 ++++++++- 5 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/torproject/collector/conf/Configuration.java b/src/main/java/org/torproject/collector/conf/Configuration.java index 4e05ad4..9295811 100644 --- a/src/main/java/org/torproject/collector/conf/Configuration.java +++ b/src/main/java/org/torproject/collector/conf/Configuration.java @@ -3,6 +3,8 @@ package org.torproject.collector.conf; +import java.net.MalformedURLException; +import java.net.URL; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Properties; @@ -113,4 +115,18 @@ public class Configuration extends Properties { } } + /** + * Returns a {@code URL} property, e.g. + * {@code urlProperty = https://my.url.here}. + */ + public URL getUrl(Key key) throws ConfigurationException { + try { + checkClass(key, URL.class); + return new URL(getProperty(key.name())); + } catch (MalformedURLException mue) { + throw new ConfigurationException("Corrupt property: " + key + + " reason: " + mue.getMessage(), mue); + } + } + } diff --git a/src/main/java/org/torproject/collector/conf/Key.java b/src/main/java/org/torproject/collector/conf/Key.java index 4f5df64..b4119b6 100644 --- a/src/main/java/org/torproject/collector/conf/Key.java +++ b/src/main/java/org/torproject/collector/conf/Key.java @@ -1,5 +1,6 @@ package org.torproject.collector.conf; +import java.net.URL; import java.nio.file.Path; /** @@ -9,6 +10,7 @@ import java.nio.file.Path; public enum Key { ExitlistOutputDirectory(Path.class), + ExitlistUrl(URL.class), InstanceBaseUrl(String.class), LockFilePath(Path.class), ArchivePath(Path.class), diff --git a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java index 70de74f..79fe19f 100644 --- a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java +++ b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java @@ -84,9 +84,7 @@ public class ExitListDownloader extends Thread { sb.append("@type tordnsel 1.0\n"); sb.append("Downloaded " + dateTimeFormat.format(downloadedDate) + "\n"); - String exitAddressesUrl = - "http://exitlist.torproject.org/exit-addresses"; - URL url = new URL(exitAddressesUrl); + URL url = config.getUrl(Key.ExitlistUrl); HttpURLConnection huc = (HttpURLConnection) url.openConnection(); huc.setRequestMethod("GET"); huc.connect(); diff --git a/src/main/resources/collector.properties b/src/main/resources/collector.properties index fb9e1b4..76ae6bd 100644 --- a/src/main/resources/collector.properties +++ b/src/main/resources/collector.properties @@ -98,6 +98,7 @@ SanitizedBridgesWriteDirectory = out/bridge-descriptors/ # ## ExitlistOutputDirectory = out/exit-lists/ +ExitlistUrl = https://check.torproject.org/exit-addresses ######## Torperf downloader ######## # diff --git a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java index 57dda75..ac9fd4f 100644 --- a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java +++ b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java @@ -22,7 +22,7 @@ public class ConfigurationTest { public void testKeyCount() throws Exception { assertEquals("The number of properties keys in enum Key changed." + "\n This test class should be adapted.", - 32, Key.values().length); + 33, Key.values().length); } @Test() @@ -129,6 +129,13 @@ public class ConfigurationTest { } @Test( expected = ConfigurationException.class) + public void testUrlValueException() throws Exception { + Configuration conf = new Configuration(); + conf.load(new ByteArrayInputStream("ExitlistUrl = xxx://y.y.y".getBytes())); + conf.getUrl(Key.ExitlistUrl); + } + + @Test( expected = ConfigurationException.class) public void testIntValueException() throws Exception { Configuration conf = new Configuration(); conf.load(new ByteArrayInputStream("BridgeDescriptorMappingsLimit = y7".getBytes()));

1 0

[torspec/master] Remove the prop#216 "Curve25519 group" sentence from NewHope proposal.
by isis＠torproject.org 22 Jul '16

22 Jul '16

commit d04f771f8b8bcb7b5b6c27d1df352061282ac2ac Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun May 8 16:08:16 2016 +0000 Remove the prop#216 "Curve25519 group" sentence from NewHope proposal. --- proposals/XXX-newhope-hybrid-handshake.txt | 8 -------- 1 file changed, 8 deletions(-) diff --git a/proposals/XXX-newhope-hybrid-handshake.txt b/proposals/XXX-newhope-hybrid-handshake.txt index 607b533..6c234cb 100644 --- a/proposals/XXX-newhope-hybrid-handshake.txt +++ b/proposals/XXX-newhope-hybrid-handshake.txt @@ -218,14 +218,6 @@ Depends: prop#220 prop#249 prop#264 if AUTH == H(auth_input, T_MAC) return NTOR_KEY - Both parties check that none of the EXP() operations produced the point at - infinity. [NOTE: This is an adequate replacement for checking Y for group - membership, if the group is Curve25519.] - - [XXX: This doesn't sound exactly right. You need the scalar tweaking of - X25519 for this to work and also, the point at infinity is obviously an - element of the group --isis, peter] - Both parties now have a shared value for NTOR_KEY. They expand this into the keys needed for the Tor relay protocol.

1 0

[torspec/master] Add NewHope + X25519 handshake proposal draft.
by isis＠torproject.org 22 Jul '16

22 Jul '16

commit 0f5ddf6ca863eaa748733557a3a96c44f3361085 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Apr 22 19:23:43 2016 +0000 Add NewHope + X25519 handshake proposal draft. --- proposals/XXX-newhope-hybrid-handshake.txt | 768 +++++++++++++++++++++++++++++ 1 file changed, 768 insertions(+) diff --git a/proposals/XXX-newhope-hybrid-handshake.txt b/proposals/XXX-newhope-hybrid-handshake.txt new file mode 100644 index 0000000..2a5e076 --- /dev/null +++ b/proposals/XXX-newhope-hybrid-handshake.txt @@ -0,0 +1,768 @@ +Filename: XXX-newhope-hybrid-handshake.txt +Title: Post-Quantum Secure Hybrid Handshake Based on NewHope +Author: Isis Lovecruft, Peter Schwabe +Created: 16 Apr 2016 +Updated: 4 May 2016 +Status: Draft +Depends: prop#220 prop#249 prop#264 + +§0. Introduction + + NewHope is a post-quantum-secure lattice-based key-exchange protocol based + on the ring-learning-with-errors (Ring-LWE) problem. We propose a hybrid + handshake for Tor, based on a combination of Tor's current NTor handshake + and a shared key derived through a NewHope ephemeral key exchange. + + For further details on the NewHope key exchange, the reader is referred to + "Post-quantum key exchange - a new hope" by Alkim, Ducas, Pöppelmann, and + Schwabe [0][1]. + + For the purposes of brevity, we consider that NTor is currently the only + handshake protocol in Tor; the older TAP protocol is ignored completely, due + to the fact that it is currently deprecated and nearly entirely unused. + + +§1. Motivation + + An attacker currently monitoring and storing circuit-layer NTor handshakes + who later has the ability to run Shor's algorithm on a quantum computer will + be able to break Tor's current handshake protocol and decrypt previous + communications. + + It is unclear if and when such attackers equipped with large quantum + computers will exist, but various estimates by researchers in quantum + physics and quantum engineering give estimates of only 1 to 2 decades. + Clearly, the security requirements of many Tor users include secrecy of + their messages beyond this time span, which means that Tor needs to update + the key exchange to protect against such attackers as soon as possible. + + +§2. Design + + An initiator and responder, in parallel, conduct two handshakes: + + - An X25519 key exchange, as described in the description of the NTor + handshake in Tor proposal #216. + - A NewHope key exchange. + + The shared keys derived from these two handshakes are then concatenated and + used as input to the SHAKE-256 extendable output function (XOF), as decribed + in FIPS-PUB-202 [2], in order to produce a shared key of the desired length. + The testvectors in §C assume that this key has a length of 32 bytes, but the + use of a XOF allows arbitrary lengths to easily support future updates of + the symmetric primitives using the key. See also §3.3.1. + + +§3. Specification + +§3.1. Notation + + Let `a || b` be the concatenation of a with b. + + Let `a^b` denote the exponentiation of a to the bth power. + + Let `a == b` denote the equality of a with b, and vice versa. + + Let `a := b` be the assignment of the value of b to the variable a. + + Let `H(x)` be 32-bytes of output of the SHAKE-256 XOF (as described in + FIPS-PUB-202) applied to message x. + + Let X25519 refer to the curve25519-based key agreement protocol described + in RFC7748 §6.1. [3] + + Let `EXP(a, b) == X25519(., b, a)` with `g == 9`. Let X25519_KEYGEN() do + the appropriate manipulations when generating the secret key (clearing the + low bits, twidding the high bits). + + [XXX match RFC7748 notation more. --isis] + + Let `X25519_KEYID(B) == B` where B is a valid X25519 public key. + + When representing an element of the Curve25519 subgroup as a byte string, + use the standard (32-byte, little-endian, x-coordinate-only) representation + for Curve25519 points. + + Let `ID` be a router's identity key taken from the router microdescriptor. + In the case for relays possessing Ed25519 identity keys (c.f. Tor proposal + #220), this is a 32-byte string representing the public Ed25519 identity key. + For backwards and forwards compatibility with routers which do not possess + Ed25519 identity keys, this is a 32-byte string created via the output of + H(ID). + + We refer to the router as the handshake "responder", and the client (which + may be an OR or an OP) as the "initiator". + + + ID_LENGTH [32 bytes] + H_LENGTH [32 bytes] + G_LENGTH [32 bytes] + + PROTOID := "pqtor-x25519-newhope-shake256-1" + T_MAC := PROTOID || ":mac" + T_KEY := PROTOID || ":key_extract" + T_VERIFY := PROTOID || ":verify" + + (X25519_SK, X25519_PK) := X25519_KEYGEN() + + +§3.2. Protocol + + ======================================================================================== + | | + | Fig. 1: The NewHope-X25519 Hybrid Handshake. | + | | + | Before the handshake the Initiator is assumed to know Z, a public X25519 key for | + | the Responder, as well as the Responder's ID. | + ---------------------------------------------------------------------------------------- + | | + | Initiator Responder | + | | + | SEED := H(randombytes(32)) | + | x, X := X25519_KEYGEN() | + | a, A := NEWHOPE_KEYGEN(SEED) | + | CLIENT_HDATA := ID || Z || X || A | + | | + | --- CLIENT_HDATA ---> | + | | + | y, Y := X25519_KEYGEN() | + | NTOR_KEY, AUTH := NTOR_SHAREDB(X,y,Y,z,Z,ID,B) | + | M, NEWHOPE_KEY := NEWHOPE_SHAREDB(A) | + | SERVER_HDATA := Y || AUTH || M | + | sk := SHAKE-256(NTOR_KEY || NEWHOPE_KEY) | + | | + | <-- SERVER_HDATA ---- | + | | + | NTOR_KEY := NTOR_SHAREDA(x, X, Y, Z, ID, AUTH) | + | NEWHOPE_KEY := NEWHOPE_SHAREDA(M, a) | + | sk := SHAKE-256(NTOR_KEY, NEWHOPE_KEY) | + | | + ======================================================================================== + + +§3.2.1. The NTor Handshake + +§3.2.1.1. Prologue + + Take a router with identity ID. As setup, the router generates a secret key z, + and a public onion key Z with: + + z, Z := X25519_KEYGEN() + + The router publishes Z in its server descriptor in the "ntor-onion-key" entry. + Henceforward, we refer to this router as the "responder". + + +§3.2.1.2. Initiator + + To send a create cell, the initiator generates a keypair: + + x, X := X25519_KEYGEN() + + and creates the NTor portion of a CREATE2V cell's HDATA section: + + CLIENT_NTOR := ID || Z || X [96 bytes] + + The initiator includes the responder's ID and Z in the CLIENT_NTOR so that, in + the event the responder OR has recently rotated keys, the responder can + determine which keypair to use. + + The initiator then concatenates CLIENT_NTOR with CLIENT_NEWHOPE (see §3.2.2), + to create CLIENT_HDATA, and creates and sends a CREATE2V cell (see §A.1) + to the responder. + + CLIENT_NEWHOPE [1824 bytes] (see §3.2.2) + CLIENT_HDATA := CLIENT_NTOR || CLIENT_NEWHOPE [1920 bytes] + + If the responder does not respond with a CREATED2V cell, the initiator SHOULD + NOT attempt to extend the circuit through the responder by sending fragmented + EXTEND2 cells, since the responder's lack of support for CREATE2V cells is + assumed to imply the responder also lacks support for fragmented EXTEND2 + cells. Alternatively, for initiators with a sufficiently late consensus + method, the initiator MUST check that "proto" line in the responder's + descriptor (c.f. Tor proposal #264) advertises support for the "Relay" + subprotocol version 3 (see §5). + + +§3.2.1.3. Responder + + The responder generates a keypair of y, Y = X25519_KEYGEN(), and does + NTOR_SHAREDB() as follows: + + (NTOR_KEY, AUTH) ← NTOR_SHAREDB(X, y, Y, z, Z, ID, B): + secret_input := EXP(X, y) || EXP(X, z) || ID || B || Z || Y || PROTOID + NTOR_KEY := H(secret_input, T_KEY) + verify := H(secret_input, T_VERIFY) + auth_input := verify || ID || Z || Y || X || PROTOID || "Server" + AUTH := H(auth_input, T_MAC) + + The responder sends a CREATED2V cell containing: + + SERVER_NTOR := Y || AUTH [64 bytes] + SERVER_NEWHOPE [2048 bytes] (see §3.2.2) + SERVER_HDATA := SERVER_NTOR || SERVER_NEWHOPE [2112 bytes] + + and sends this to the initiator. + + +§3.2.1.4. Finalisation + + The initiator then checks Y is in G^* [see NOTE below], and does + NTOR_SHAREDA() as follows: + + (NTOR_KEY) ← NTOR_SHAREDA(x, X, Y, Z, ID, AUTH) + secret_input := EXP(Y, x) || EXP(Z, x) || ID || Z || X || Y || PROTOID + NTOR_KEY := H(secret_input, T_KEY) + verify := H(secret_input, T_VERIFY) + auth_input := verify || ID || Z || Y || X || PROTOID || "Server" + if AUTH == H(auth_input, T_MAC) + return NTOR_KEY + + Both parties check that none of the EXP() operations produced the point at + infinity. [NOTE: This is an adequate replacement for checking Y for group + membership, if the group is Curve25519.] + + [XXX: This doesn't sound exactly right. You need the scalar tweaking of + X25519 for this to work and also, the point at infinity is obviously an + element of the group --isis, peter] + + Both parties now have a shared value for NTOR_KEY. They expand this into + the keys needed for the Tor relay protocol. + + [XXX We think we want to omit the final hashing in the production of NTOR_KEY + here, and instead put all the inputs through SHAKE-256. --isis, peter] + + [XXX We probably want to remove ID and B from the input to the shared key + material, since they serve for authentication but, as pre-established + "prologue" material to the handshake, they should not be used in attempts to + strengthen the cryptographic suitability of the shared key. Also, their + inclusion is implicit in the DH exponentiations. I should probably ask Ian + about the reasoning for the original design choice. --isis] + + +§3.2.2. The NewHope Handshake + +§3.2.2.1. Parameters & Mathematical Structures + + Let ℤ be the ring of rational integers. Let ℤq, for q ≥ 1, denote the quotient + ring ℤ/qℤ. We define R = ℤ[X]/((X^n)+1) as the ring of integer polynomials + modulo ((X^n)+1), and Rq = ℤq[X]/((X^n)+1) as the ring of integer polynomials + modulo ((X^n)+1) where each coefficient is reduced modulo q. When we refer to + a polynomial, we mean an element of Rq. + + n := 1024 + q := 12289 + + SEED [32 Bytes] + NEWHOPE_POLY [1792 Bytes] + NEWHOPE_REC [256 Bytes] + NEWHOPE_KEY [32 Bytes] + + NEWHOPE_MSGA := (NEWHOPE_POLY || SEED) + NEWHOPE_MSGB := (NEWHOPE_POLY || NEWHOPE_REC) + + +§3.2.2.2. High-level Description of Newhope API Functions + + For a description of internal functions, see §B. + + (NEWHOPE_POLY, NEWHOPE_MSGA) ← NEWHOPE_KEYGEN(SEED): + â := gen_a(seed) + s := poly_getnoise() + e := poly_getnoise() + ŝ := poly_ntt(s) + ê := poly_ntt(e) + b̂ := pointwise(â, ŝ) + ê + sp := poly_tobytes(ŝ) + bp := poly_tobytes(b̂) + return (sp, (bp || seed)) + + (NEWHOPE_MSGB, NEWHOPE_KEY) ← NEWHOPE_SHAREDB(NEWHOPE_MSGA): + s' := poly_getnoise() + e' := poly_getnoise() + e" := poly_getnoise() + b̂ := poly_frombytes(bp) + â := gen_a(seed) + ŝ' := poly_ntt(s') + ê' := poly_ntt(e') + û := poly_pointwise(â, ŝ') + ê' + v := poly_invntt(poly_pointwise(b̂,ŝ')) + e" + r := helprec(v) + up := poly_tobytes(û) + k := rec(v, r) + return ((up || r), k) + + NEWHOPE_KEY ← NEWHOPE_SHAREDA(NEWHOPE_MSGB, NEWHOPE_POLY): + û := poly_frombytes(up) + ŝ := poly_frombytes(sp) + v' := poly_invntt(poly_pointwise(û, ŝ)) + k := rec(v', r) + return k + + When a client uses a SEED within a CREATE2V cell, the client SHOULD NOT use + that SEED in any other CREATE2V or EXTEND2 cells. See §4 for further + discussion. + + +§3.3. Key Expansion + + The client and server derive a shared key, SHARED, by: + + HKDFID := "THESE ARENT THE DROIDS YOURE LOOKING FOR" + SHARED := SHAKE_256(HKDFID || NTorKey || NewHopeKey) + + +§3.3.1. Note on the Design Choice + + The reader may wonder why one would use SHAKE-256 to produce a 256-bit + output, since the security strength in bits for SHAKE-256 is min(d/2,256) + for collision resistance and min(d,256) for first- and second-order + preimages, where d is the output length. + + The reasoning is that we should be aiming for 256-bit security for all of + our symmetric cryptography. One could then argue that we should just use + SHA3-256 for the KDF. We choose SHAKE-256 instead in order to provide an + easy way to derive longer shared secrets in the future without requiring a + new handshake. The construction is odd, but the future is bright. + As we are already using SHAKE-256 for the 32-byte output hash, we are also + using it for all other 32-byte hashes involved in the protocol. Note that + the only difference between SHA3-256 and SHAKE-256 with 32-byte output is + one domain-separation byte. + + [XXX why would you want 256-bit security for the symmetric side? Are you + talking pre- or post-quantum security? --peter] + + +§4. Security & Anonymity Implications + + This handshake protocol is one-way authenticated. That is, the server is + authenticated, while the client remains anonymous. + + The client MUST NOT cache and reuse SEED. Doing so gives non-trivial + adversarial advantages w.r.t. all-for-the-price-of-one attacks during the + caching period. More importantly, if the SEED used to generate NEWHOPE_MSGA + is reused for handshakes along the same circuit or multiple different + circuits, an adversary conducting a sybil attack somewhere along the path(s) + will be able to correlate the identity of the client across circuits or + hops. + + +§5. Compatibility + + Because our proposal requires both the client and server to send more than + the 505 bytes possible within a CREATE2 cell's HDATA section, it depends + upon the implementation of a mechanism for allowing larger CREATE cells + (c.f. Tor proposal #249). + + We reserve the following handshake type for use in CREATE2V/CREATED2V and + EXTEND2V/EXTENDED2V cells: + + 0x0003 [NEWHOPE + X25519 HYBRID HANDSHAKE] + + We introduce a new sub-protocol number, "Relay=3", (c.f. Tor proposal #264 + §5.3) to signify support this handshake, and hence for the CREATE2V and + fragmented EXTEND2 cells which it requires. + + There are no additional entries or changes required within either router + descriptors or microdescriptors to support this handshake method, due to the + NewHope keys being ephemeral and derived on-the-fly, and due to the NTor X25519 + public keys already being in included within the "ntor-onion-key" entry. + + Add a "UseNewHopeKEX" configuration option and a corresponding consensus + parameter to control whether clients prefer using this NewHope hybrid + handshake or some previous handshake protocol. If the configuration option + is "auto", clients SHOULD obey the consensus parameter. The default + configuration SHOULD be "auto" and the consensus value SHOULD initially be "0". + + +§6. Implementation + + The paper by Alkim, Ducas, Pöppelmann and Schwabe describes two software + implementations of NewHope, one C reference implementation and an optimized + implementation using AVX2 vector instructions. Those implementations are + available at [1]. + + Additionally, there are implementations in Go by Yawning Angel, available + from [4] and in Rust by Isis Lovecruft, available from [5]. + + The software used to generate the test vectors in §C is based on the C + reference implementation and available from: + + https://code.ciph.re/isis/newhope-tor-testvectors + https://github.com/isislovecruft/newhope-tor-testvectors + + +§7. Performance & Scalability + + The computationally expensive part in the current NTor handshake is the + X25519 key-pair generation and the X25519 shared-key computation. The + current implementation in Tor is a wrapper to support various highly optimized + implementations on different architectures. On Intel Haswell processors, the + fastest implementation of X25519, as reported by the eBACS benchmarking + project [6], takes 169920 cycles for key-pair generation and 161648 cycles + for shared-key computation; these add up to a total of 331568 cycles on each + side (initiator and responder). + + The C reference implementation of NewHope, also benchmarked on Intel + Haswell, takes 358234 cycles for the initiator and 402058 cycles for the + Responder. The core computation of the proposed combination of NewHope and + X25519 will thus mean a slowdown of about a factor of 2.1 for the Initiator + and a slowdown by a factor of 2.2 for the Responder compared to the current + NTor handshake. These numbers assume a fully optimized implementation of the + NTor handshake and a C reference implementation of NewHope. With optimized + implementations of NewHope, such as the one for Intel Haswell described in + [0], the computational slowdown will be considerably smaller than a factor + of 2. + + +§8. References + +[0]: https://cryptojedi.org/papers/newhope-20160328.pdf +[1]: https://cryptojedi.org/crypto/#newhope +[2]: http://www.nist.gov/customcf/get_pdf.cfm?pub_id=919061 +[3]: https://tools.ietf.org/html/rfc7748#section-6.1 +[4]: https://github.com/Yawning/newhope +[5]: https://code.ciph.re/isis/newhopers +[6]: http://bench.cr.yp.to + + +§A. Cell Formats + +§A.1. CREATE2V Cells + + The client portion of the handshake should send CLIENT_HDATA, formatted + into a CREATE2V cell as follows: + + CREATE2V { [2114 bytes] + HTYPE := 0x0003 [2 bytes] + HLEN := 0x0780 [2 bytes] + HDATA := CLIENT_HDATA [1920 bytes] + IGNORED := 0x00 [194 bytes] + } + + [XXX do we really want to pad with IGNORED to make CLIENT_HDATA the + same number of bytes as SERVER_HDATA? --isis] + +§A.2. CREATED2V Cells + + The server responds to the client's CREATE2V cell with SERVER_HDATA, + formatted into a CREATED2V cell as follows: + + CREATED2V { [2114 bytes] + HLEN := 0x0800 [2 bytes] + HDATA := SERVER_HDATA [2112 bytes] + IGNORED := 0x00 [0 bytes] + } + +§A.3. Fragmented EXTEND2 Cells + + When the client wishes to extend a circuit, the client should fragment + CLIENT_HDATA into four EXTEND2 cells: + + EXTEND2 { + NSPEC := 0x02 { [1 byte] + LINK_ID_SERVER [22 bytes] XXX + LINK_ADDRESS_SERVER [8 bytes] XXX + } + HTYPE := 0x0003 [2 bytes] + HLEN := 0x0780 [2 bytes] + HDATA := CLIENT_HDATA[0,461] [462 bytes] + } + EXTEND2 { + NSPEC := 0x00 [1 byte] + HTYPE := 0xFFFF [2 bytes] + HLEN := 0x0000 [2 bytes] + HDATA := CLIENT_HDATA[462,954] [492 bytes] + } + EXTEND2 { + NSPEC := 0x00 [1 byte] + HTYPE := 0xFFFF [2 bytes] + HLEN := 0x0000 [2 bytes] + HDATA := CLIENT_HDATA[955,1447] [492 bytes] + } + EXTEND2 { + NSPEC := 0x00 [1 byte] + HTYPE := 0xFFFF [2 bytes] + HLEN := 0x0000 [2 bytes] + HDATA := CLIENT_HDATA[1448,1919] || 0x00[20] [492 bytes] + } + EXTEND2 { + NSPEC := 0x00 [1 byte] + HTYPE := 0xFFFF [2 bytes] + HLEN := 0x0000 [2 bytes] + HDATA := 0x00[172] [172 bytes] + } + + The client sends this to the server to extend the circuit from, and that + server should format the fragmented EXTEND2 cells into a CREATE2V cell, as + described in §A.1. + +§A.4. Fragmented EXTENDED2 Cells + + EXTENDED2 { + NSPEC := 0x02 { [1 byte] + LINK_ID_SERVER [22 bytes] XXX + LINK_ADDRESS_SERVER [8 bytes] XXX + } + HTYPE := 0x0003 [2 bytes] + HLEN := 0x0800 [2 bytes] + HDATA := SERVER_HDATA[0,461] [462 bytes] + } + EXTENDED2 { + NSPEC := 0x00 [1 byte] + HTYPE := 0xFFFF [2 bytes] + HLEN := 0x0000 [2 bytes] + HDATA := SERVER_HDATA[462,954] [492 bytes] + } + EXTEND2 { + NSPEC := 0x00 [1 byte] + HTYPE := 0xFFFF [2 bytes] + HLEN := 0x0000 [2 bytes] + HDATA := SERVER_HDATA[955,1447] [492 bytes] + } + EXTEND2 { + NSPEC := 0x00 [1 byte] + HTYPE := 0xFFFF [2 bytes] + HLEN := 0x0000 [2 bytes] + HDATA := SERVER_HDATA[1448,1939] [492 bytes] + } + EXTEND2 { + NSPEC := 0x00 [1 byte] + HTYPE := 0xFFFF [2 bytes] + HLEN := 0x0000 [2 bytes] + HDATA := SERVER_HDATA[1940,2112] [172 bytes] + } + + +§B. NewHope Internal Functions + + gen_a(SEED): returns a uniformly random poly + poly_getnoise(): returns a poly sampled from a centered binomial + poly_ntt(poly): number-theoretic transform; returns a poly + poly_invntt(poly): inverse number-theoretic transform; returns a poly + poly_pointwise(poly, poly): pointwise multiplication; returns a poly + poly_tobytes(poly): packs a poly to a NEWHOPE_POLY byte array + poly_frombytes(NEWHOPE_POLY): unpacks a NEWHOPE_POLY byte array to a poly + + helprec(poly): returns a NEWHOPE_REC byte array + rec(poly, NEWHOPE_REC): returns a NEWHOPE_KEY + + + --- Description of the Newhope internal functions --- + + gen_a(SEED seed) receives as input a 32-byte (public) seed. It expands + this seed through SHAKE-128 from the FIPS202 standard. The output of SHAKE-128 + is considered a sequence of 16-bit little-endian integers. This sequence is + used to initialize the coefficients of the returned polynomial from the least + significant (coefficient of X^0) to the most significant (coefficient of + X^1023) coefficient. For each of the 16-bit integers first eliminate the + highest two bits (to make it a 14-bit integer) and then use it as the next + coefficient if it is smaller than q=12289. + Note that the amount of output required from SHAKE to initialize all 1024 + coefficients of the polynomial varies depending on the input seed. + Note further that this function does not process any secret data and thus does + not need any timing-attack protection. + + + poly_getnoise() first generates 4096 Bytes of uniformly random data. This can + be done by reading these bytes from the system's RNG; efficient + implementations will typically only read a 32-byte seed from the system's RNG + and expand it through some fast PRNG (for example, ChaCha20 or AES-256 in CTR + mode). The output of the PRG is considered an array of 2048 16-bit integers + r[0],...,r[2047]. The coefficients of the output polynomial are computed as + HW(r[0])-HW(r[1]), HW(r[2])-HW(r[3]),...,HW(r[2046])-HW(r[2047]), where HW + stands for Hamming weight. + Note that the choice of RNG is a local decision; different implementations are + free to use different RNGs. + Note further that the output of this function is secret; the PRG (and the + computation of HW) need to be protected against timing attacks. + + + poly_ntt(poly f): For a mathematical description of poly_ntt see the [0]; a + pseudocode description of a very naive inplace transformation of an input + polynomial f = f[0] + f[1]*X + f[2]*X^2 + ... + f[1023]*X^1023 is the + following code (all arithmetic on coefficients performed modulo q): + + psi = 7 + omega = 49 + + for i in range(0,n): + t[i] = f[i] * psi^i + + for i in range(0,n): + f[i] = 0 + for j in range(0,n): + f[i] += t[j] * omega^((i*j)%n) + + Note that this is not how poly_ntt should be implemented if performance is + an issue; in particular, efficient algorithms for the number-theoretic + transform take time O(n*log(n)) and not O(n^2) + Note further that all arithmetic in poly_ntt has to be protected against + timing attacks. + + + poly_invntt(poly f): For a mathematical description of poly_invntt see the + [0]; a pseudocode description of a very naive inplace transformation of an + input polynomial f = f[0] + f[1]*X + f[2]*X^2 + ... + f[1023]*X^1023 is the + following code (all arithmetic on coefficients performed modulo q): + + invpsi = 8778; + invomega = 1254; + invn = 12277; + + for i in range(0,n): + t[i] = f[i]; + + for i in range(0,n): + f[i]=0; + for j in range(0,n): + f[i] += t[j] * invomega^((i*j)%n) + f[i] *= invpsi^i + f[i] *= invn + + Note that this is not how poly_invntt should be implemented if performance + is an issue; in particular, efficient algorithms for the inverse + number-theoretic transform take time O(n*log(n)) and not O(n^2) + Note further that all arithmetic in poly_invntt has to be protected against + timing attacks. + + + poly_pointwise(poly f, poly g) performs pointwise multiplication of the two + polynomials. This means that for f = (f0 + f1*X + f2*X^2 + ... + + f1023*X^1023) and g = (g0 + g1*X + g2*X^2 + ... + g1023*X^1023) it computes + and returns h = (h0 + h1*X + h2*X^2 + ... + h1023*X^1023) with h0 = f0*g0, + h1 = f1*g1,..., h1023 = f1023*g1023. + + + poly_tobytes(poly f) first reduces all coefficents of f modulo q, i.e., + brings them to the interval [0,q-1]. Denote these reduced coefficients as + f0,..., f1023; note that they all fit into 14 bits. The function then packs + those coefficients into an array of 1792 bytes r[0],..., r[1792] in "packed + little-endian representation", i.e., + r[0] = f[0] & 0xff; + r[1] = (f[0] >> 8) & ((f[1] & 0x03) << 6) + r[2] = (f[1] >> 2) & 0xff; + r[3] = (f[1] >> 10) & ((f[2] & 0x0f) << 4) + . + . + . + r[1790] = (f[1022]) >> 12) & ((f[1023] & 0x3f) << 2) + r[1791] = f[1023] >> 6 + Note that this function needs to be protected against timing attacks. In + particular, avoid non-constant-time conditional subtractions (or other + non-constant-time expressions) in the reduction modulo q of the coefficients. + + + poly_frombytes(NEWHOPE_POLY b) is the inverse of poly_tobytes; it receives + as input an array of 1792 bytes and coverts it into the internal + representation of a poly. Note that poly_frombytes does not need to check + whether the coefficients are reduced modulo q or reduce coefficients modulo + q. Note further that the function must not leak any information about its + inputs through timing information, as it is also applied to the secret key + of the initiator. + + + helprec(poly f) computes 256 bytes of reconciliation information from the + input poly f. Internally, one byte of reconciliation information is computed + from four coefficients of f by a function helprec4. Let the input polynomial f + = (f0 + f1*X + f2*X^2 + ... + f1023*X^1023); let the output byte array be + r[0],...r[256]. This output byte array is computed as + r[0] = helprec4(f0,f256,f512,f768) + r[1] = helprec4(f1,f257,f513,f769) + r[2] = helprec4(f2,f258,f514,f770) + . + . + . + r[255] = helprec4(f255,f511,f767,f1023), where helprec4 does the following: + + helprec4(x0,x1,x2,x3): + b = randombit() + r0,r1,r2,r3 = CVPD4(8*x0+4*b,8*x1+4*b,8*x2+4*b,8*x3+4*b) + r = (r0 & 0x03) | ((r1 & 0x03) << 2) | ((r2 & 0x03) << 4) | ((r3 & 0x03) << 6) + return r + + The function CVPD4 does the following: + + CVPD4(y0,y1,y2,y3): + v00 = round(y0/2q) + v01 = round(y1/2q) + v02 = round(y2/2q) + v03 = round(y3/2q) + v10 = round((y0-1)/2q) + v11 = round((y1-1)/2q) + v12 = round((y2-1)/2q) + v13 = round((y3-1)/2q) + t = abs(y0 - 2q*v00) + t += abs(y1 - 2q*v01) + t += abs(y2 - 2q*v02) + t += abs(y3 - 2q*v03) + if(t < 2q): + v0 = v00 + v1 = v01 + v2 = v02 + v3 = v03 + k = 0 + else + v0 = v10 + v1 = v11 + v2 = v12 + v3 = v13 + r = 1 + return (v0-v3,v1-v3,v2-v3,k+2*v3) + + In this description, round() returns the closest integer and abs() returns the + absolute value. + Note that all computations involved in helprec operate on secret data and must + be protected against timing attacks. + + + rec(poly f, NEWHOPE_REC r) computes the pre-hash (see paper) Newhope key from + f and r. Specifically, it computes one bit of key from 4 coefficients of f and + one byte of r. Let f = f0 + f1*X + f2*X^2 + ... + f1023*X^1023 and let r = + r[0],r[1],...,r[255]. Let the bytes of the output by k[0],...,k[31] and let + the bits of the output by k0,...,k255, where + k0 = k[0] & 0x01 + k1 = (k[0] >> 1) & 0x01 + k2 = (k[0] >> 2) & 0x01 + . + . + . + k8 = k[1] & 0x01 + k9 = (k[1] >> 1) & 0x01 + . + . + . + k255 = (k[32] >> 7) + The function rec computes k0,...,k255 as + k0 = rec4(f0,f256,f512,f768,r[0]) + k1 = rec4(f1,f257,f513,f769,r[1]) + . + . + . + k255 = rec4(f255,f511,f767,f1023,r[255]) + + The function rec4 does the following: + + rec4(y0,y1,y2,y3,r): + r0 = r & 0x03 + r1 = (r >> 2) & 0x03 + r2 = (r >> 4) & 0x03 + r3 = (r >> 6) & 0x03 + Decode(8*y0-2q*r0, 8*y1-2q*r1, 8*y2-2q*r2, 8*y3-q*r3) + + The function Decode does the following: + + Decode(v0,v1,v2,v3): + t0 = round(v0/8q) + t1 = round(v1/8q) + t2 = round(v2/8q) + t3 = round(v3/8q) + t = abs(v0 - 8q*t0) + t += abs(v0 - 8q*t0) + t += abs(v0 - 8q*t0) + t += abs(v0 - 8q*t0) + if(t > 1) return 1 + else return 0 + + +§C. Test Vectors

1 0

[torspec/master] Fix several typos found in the NewHope proposal.
by isis＠torproject.org 22 Jul '16

22 Jul '16

commit 5c11590522923f8cbca307f6be92357087a5ca2d Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun May 8 15:05:36 2016 +0000 Fix several typos found in the NewHope proposal. * THANKS TO eikovi(a)sigaint.org for pointing them out. --- proposals/XXX-newhope-hybrid-handshake.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/proposals/XXX-newhope-hybrid-handshake.txt b/proposals/XXX-newhope-hybrid-handshake.txt index 2a5e076..d11fbd2 100644 --- a/proposals/XXX-newhope-hybrid-handshake.txt +++ b/proposals/XXX-newhope-hybrid-handshake.txt @@ -46,7 +46,7 @@ Depends: prop#220 prop#249 prop#264 - A NewHope key exchange. The shared keys derived from these two handshakes are then concatenated and - used as input to the SHAKE-256 extendable output function (XOF), as decribed + used as input to the SHAKE-256 extendable output function (XOF), as described in FIPS-PUB-202 [2], in order to produce a shared key of the desired length. The testvectors in §C assume that this key has a length of 32 bytes, but the use of a XOF allows arbitrary lengths to easily support future updates of @@ -84,7 +84,7 @@ Depends: prop#220 prop#249 prop#264 for Curve25519 points. Let `ID` be a router's identity key taken from the router microdescriptor. - In the case for relays possessing Ed25519 identity keys (c.f. Tor proposal + In the case for relays possessing Ed25519 identity keys (cf. Tor proposal #220), this is a 32-byte string representing the public Ed25519 identity key. For backwards and forwards compatibility with routers which do not possess Ed25519 identity keys, this is a 32-byte string created via the output of @@ -180,7 +180,7 @@ Depends: prop#220 prop#249 prop#264 assumed to imply the responder also lacks support for fragmented EXTEND2 cells. Alternatively, for initiators with a sufficiently late consensus method, the initiator MUST check that "proto" line in the responder's - descriptor (c.f. Tor proposal #264) advertises support for the "Relay" + descriptor (cf. Tor proposal #264) advertises support for the "Relay" subprotocol version 3 (see §5). @@ -352,14 +352,14 @@ Depends: prop#220 prop#249 prop#264 Because our proposal requires both the client and server to send more than the 505 bytes possible within a CREATE2 cell's HDATA section, it depends upon the implementation of a mechanism for allowing larger CREATE cells - (c.f. Tor proposal #249). + (cf. Tor proposal #249). We reserve the following handshake type for use in CREATE2V/CREATED2V and EXTEND2V/EXTENDED2V cells: 0x0003 [NEWHOPE + X25519 HYBRID HANDSHAKE] - We introduce a new sub-protocol number, "Relay=3", (c.f. Tor proposal #264 + We introduce a new sub-protocol number, "Relay=3", (cf. Tor proposal #264 §5.3) to signify support this handshake, and hence for the CREATE2V and fragmented EXTEND2 cells which it requires. @@ -564,10 +564,10 @@ Depends: prop#220 prop#249 prop#264 not need any timing-attack protection. - poly_getnoise() first generates 4096 Bytes of uniformly random data. This can + poly_getnoise() first generates 4096 bytes of uniformly random data. This can be done by reading these bytes from the system's RNG; efficient implementations will typically only read a 32-byte seed from the system's RNG - and expand it through some fast PRNG (for example, ChaCha20 or AES-256 in CTR + and expand it through some fast PRG (for example, ChaCha20 or AES-256 in CTR mode). The output of the PRG is considered an array of 2048 16-bit integers r[0],...,r[2047]. The coefficients of the output polynomial are computed as HW(r[0])-HW(r[1]), HW(r[2])-HW(r[3]),...,HW(r[2046])-HW(r[2047]), where HW @@ -579,7 +579,7 @@ Depends: prop#220 prop#249 prop#264 poly_ntt(poly f): For a mathematical description of poly_ntt see the [0]; a - pseudocode description of a very naive inplace transformation of an input + pseudocode description of a very naive in-place transformation of an input polynomial f = f[0] + f[1]*X + f[2]*X^2 + ... + f[1023]*X^1023 is the following code (all arithmetic on coefficients performed modulo q):

1 0

Jump to page: