tor-commits

Download

tor-commits@lists.torproject.org

July 2016

21 participants
1271 discussions

[collector/master] Don't add META-DATA files from other jars to the collector jar.
by karsten＠torproject.org 25 Jul '16

25 Jul '16

commit 7517480e4475f35730c560cab5dadcfbfc011118 Author: iwakeh <iwakeh(a)torproject.org> Date: Wed Jul 20 16:23:13 2016 +0200 Don't add META-DATA files from other jars to the collector jar. --- build.xml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/build.xml b/build.xml index ba50fbf..eccae8b 100644 --- a/build.xml +++ b/build.xml @@ -141,9 +141,18 @@ <include name="collector.properties"/> <include name="logback.xml"/> </fileset> - <zipgroupfileset dir="${libs}" > - <patternset refid="runtime" /> - </zipgroupfileset> + <restrict> + <not> + <name name="META-INF/*" /> + </not> + <archives> + <zips> + <fileset dir="${libs}"> + <patternset refid="runtime" /> + </fileset> + </zips> + </archives> + </restrict> <manifest> <attribute name="Created-By" value="The Tor Project" /> <attribute name="Implementation-Title" value="${name}"/>

1 0

[collector/master] Implements #19018 'Run CollecTor without crontab'. Added scheduler logic, adapted and added tests, adapted coverage check.
by karsten＠torproject.org 25 Jul '16

25 Jul '16

commit 997d4a4a2e6fca2bc95134537fcfebe930db8de1 Author: iwakeh <iwakeh(a)torproject.org> Date: Wed Jul 20 13:23:18 2016 +0200 Implements #19018 'Run CollecTor without crontab'. Added scheduler logic, adapted and added tests, adapted coverage check. --- build.xml | 8 +- src/main/java/org/torproject/collector/Main.java | 54 ++++------- .../bridgedescs/SanitizedBridgesWriter.java | 32 +------ .../torproject/collector/cron/CollecTorMain.java | 28 ++++++ .../org/torproject/collector/cron/Scheduler.java | 102 +++++++++++++++++++++ .../collector/exitlists/ExitListDownloader.java | 27 +----- .../collector/index/CreateIndexJson.java | 29 ++++-- .../org/torproject/collector/main/LockFile.java | 66 ------------- .../collector/relaydescs/ArchiveWriter.java | 61 ++++-------- .../collector/torperf/TorperfDownloader.java | 33 ++----- .../java/org/torproject/collector/MainTest.java | 84 +++++++++++++++-- .../java/org/torproject/collector/cron/Dummy.java | 15 +++ .../torproject/collector/cron/SchedulerTest.java | 54 +++++++++++ src/test/resources/junittest.policy | 1 + 14 files changed, 355 insertions(+), 239 deletions(-) diff --git a/build.xml b/build.xml index 51471f6..ba50fbf 100644 --- a/build.xml +++ b/build.xml @@ -210,7 +210,7 @@ <classpath refid="cobertura.test.classpath" /> <formatter type="xml" /> <batchtest toDir="${testresult}" > - <fileset dir="${testclasses}" /> + <fileset dir="${testclasses}" includes="**/*Test.class" /> </batchtest> </junit> <cobertura-report format="html" destdir="${coverageresult}" > @@ -218,8 +218,10 @@ <include name="**/*.java" /> </fileset> </cobertura-report> - <cobertura-check branchrate="0" totallinerate="15" totalbranchrate="5" > + <cobertura-check branchrate="0" totallinerate="4" totalbranchrate="1" > <regex pattern="org.torproject.collector.conf.*" branchrate="100" linerate="100"/> + <regex pattern="org.torproject.collector.cron.*" branchrate="66" linerate="73"/> + <regex pattern="org.torproject.collector.Main" branchrate="66" linerate="94"/> </cobertura-check> </target> <target name="test" depends="compile,compile-tests"> @@ -231,7 +233,7 @@ <classpath refid="test.classpath"/> <formatter type="plain" usefile="false"/> <batchtest> - <fileset dir="${testclasses}" /> + <fileset dir="${testclasses}" includes="**/*Test.class" /> </batchtest> </junit> </target> diff --git a/src/main/java/org/torproject/collector/Main.java b/src/main/java/org/torproject/collector/Main.java index 81234c3..97c7a0c 100644 --- a/src/main/java/org/torproject/collector/Main.java +++ b/src/main/java/org/torproject/collector/Main.java @@ -5,6 +5,9 @@ package org.torproject.collector; import org.torproject.collector.bridgedescs.SanitizedBridgesWriter; import org.torproject.collector.conf.Configuration; +import org.torproject.collector.conf.Key; +import org.torproject.collector.cron.CollecTorMain; +import org.torproject.collector.cron.Scheduler; import org.torproject.collector.exitlists.ExitListDownloader; import org.torproject.collector.index.CreateIndexJson; import org.torproject.collector.relaydescs.ArchiveWriter; @@ -37,34 +40,31 @@ public class Main { /** All possible main classes. * If a new CollecTorMain class is available, just add it to this map. */ - static final Map<String, Class> collecTorMains = new HashMap<>(); + static final Map<Key, Class<? extends CollecTorMain>> collecTorMains = new HashMap<>(); static { // add a new main class here - collecTorMains.put("bridgedescs", SanitizedBridgesWriter.class); - collecTorMains.put("exitlists", ExitListDownloader.class); - collecTorMains.put("updateindex", CreateIndexJson.class); - collecTorMains.put("relaydescs", ArchiveWriter.class); - collecTorMains.put("torperf", TorperfDownloader.class); + collecTorMains.put(Key.BridgedescsActivated, SanitizedBridgesWriter.class); + collecTorMains.put(Key.ExitlistsActivated, ExitListDownloader.class); + collecTorMains.put(Key.UpdateindexActivated, CreateIndexJson.class); + collecTorMains.put(Key.RelaydescsActivated, ArchiveWriter.class); + collecTorMains.put(Key.TorperfActivated, TorperfDownloader.class); } - private static final String modules = collecTorMains.keySet().toString() - .replace("[", "").replace("]", "").replaceAll(", ", "|"); - private static Configuration conf = new Configuration(); /** - * One argument is necessary. + * At most one argument. * See class description {@link Main}. */ public static void main(String[] args) throws Exception { File confFile = null; - if (null == args || args.length < 1 || args.length > 2) { - printUsage("CollecTor needs one or two arguments."); - return; - } else if (args.length == 1) { + if (args == null || args.length == 0) { confFile = new File(CONF_FILE); - } else if (args.length == 2) { - confFile = new File(args[1]); + } else if (args.length == 1) { + confFile = new File(args[0]); + } else { + printUsage("CollecTor takes at most one argument."); + return; } if (!confFile.exists() || confFile.length() < 1L) { writeDefaultConfig(confFile); @@ -72,12 +72,12 @@ public class Main { } else { readConfigurationFrom(confFile); } - invokeGivenMain(args[0]); + Scheduler.getInstance().scheduleModuleRuns(collecTorMains, conf); } private static void printUsage(String msg) { final String usage = "Usage:\njava -jar collector.jar " - + "<" + modules + "> [path/to/configFile]"; + + "[path/to/configFile]"; System.out.println(msg + "\n" + usage); } @@ -105,23 +105,5 @@ public class Main { } } - private static void invokeGivenMain(String mainId) { - Class clazz = collecTorMains.get(mainId); - if (null == clazz) { - printUsage("Unknown argument: " + mainId); - } - invokeMainOnClass(clazz); - } - - private static void invokeMainOnClass(Class clazz) { - try { - clazz.getMethod("main", new Class[] { Configuration.class }) - .invoke(null, (Object) conf); - } catch (NoSuchMethodException | IllegalAccessException - | InvocationTargetException e) { - log.error("Cannot invoke 'main' method on " - + clazz.getName() + ". " + e, e); - } - } } diff --git a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java index 121f8ca..5c33566 100644 --- a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java +++ b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java @@ -6,7 +6,7 @@ package org.torproject.collector.bridgedescs; import org.torproject.collector.conf.Configuration; import org.torproject.collector.conf.ConfigurationException; import org.torproject.collector.conf.Key; -import org.torproject.collector.main.LockFile; +import org.torproject.collector.cron.CollecTorMain; import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Base64; @@ -49,36 +49,12 @@ import java.util.TreeMap; * by the bridge to advertise their capabilities), and extra-info * descriptors (published by the bridge, mainly for statistical analysis). */ -public class SanitizedBridgesWriter extends Thread { +public class SanitizedBridgesWriter extends CollecTorMain { private static Logger logger = LoggerFactory.getLogger(SanitizedBridgesWriter.class); - /** Executes the bridge-descriptors module using the given - * configuration. */ - public static void main(Configuration config) throws ConfigurationException { - - logger.info("Starting bridge-descriptors module of CollecTor."); - - // Use lock file to avoid overlapping runs - LockFile lf = new LockFile(config.getPath(Key.LockFilePath).toString(), "bridge-descriptors"); - lf.acquireLock(); - - // Sanitize bridge descriptors - new SanitizedBridgesWriter(config).run(); - - // Remove lock file - lf.releaseLock(); - - logger.info("Terminating bridge-descriptors module of CollecTor."); - } - - private Configuration config; - - /** - * Initializes this class. - */ public SanitizedBridgesWriter(Configuration config) { - this.config = config; + super(config); } private String rsyncCatString; @@ -106,12 +82,14 @@ public class SanitizedBridgesWriter extends Thread { @Override public void run() { + logger.info("Starting bridge-descriptors module of CollecTor."); try { startProcessing(); } catch (ConfigurationException ce) { logger.error("Configuration failed: " + ce, ce); throw new RuntimeException(ce); } + logger.info("Terminating bridge-descriptors module of CollecTor."); } private void startProcessing() throws ConfigurationException { diff --git a/src/main/java/org/torproject/collector/cron/CollecTorMain.java b/src/main/java/org/torproject/collector/cron/CollecTorMain.java new file mode 100644 index 0000000..7a00e68 --- /dev/null +++ b/src/main/java/org/torproject/collector/cron/CollecTorMain.java @@ -0,0 +1,28 @@ +/* Copyright 2016 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.collector.cron; + +import org.torproject.collector.conf.Configuration; +import org.torproject.collector.conf.Key; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.InvocationTargetException; +import java.util.Calendar; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +public abstract class CollecTorMain implements Runnable { + + protected Configuration config; + + public CollecTorMain( Configuration conf) { + this.config = conf; + } + +} + diff --git a/src/main/java/org/torproject/collector/cron/Scheduler.java b/src/main/java/org/torproject/collector/cron/Scheduler.java new file mode 100644 index 0000000..e4f2aa3 --- /dev/null +++ b/src/main/java/org/torproject/collector/cron/Scheduler.java @@ -0,0 +1,102 @@ +/* Copyright 2016 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.collector.cron; + +import org.torproject.collector.conf.Configuration; +import org.torproject.collector.conf.ConfigurationException; +import org.torproject.collector.conf.Key; +import org.torproject.collector.cron.CollecTorMain; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.InvocationTargetException; +import java.util.Calendar; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * Scheduler that starts the modules configured in collector.properties. + */ +public class Scheduler { + + public static final String ACTIVATED = "Activated"; + public static final String PERIODMIN = "PeriodMinutes"; + public static final String OFFSETMIN = "OffsetMinutes"; + + private final Logger log = LoggerFactory.getLogger(Scheduler.class); + + private final ScheduledExecutorService scheduler = + Executors.newScheduledThreadPool(1); + + private static Scheduler instance = new Scheduler(); + + private Scheduler(){} + + public static Scheduler getInstance() { + return instance; + } + + /** + * Schedule all classes given according to the parameters in the + * the configuration. + */ + public void scheduleModuleRuns(Map<Key, + Class<? extends CollecTorMain>> collecTorMains, Configuration conf) { + for ( Map.Entry<Key, Class<? extends CollecTorMain>> ctmEntry + : collecTorMains.entrySet() ) { + try { + if ( conf.getBool(ctmEntry.getKey()) ) { + String prefix = ctmEntry.getKey().name().replace(ACTIVATED, ""); + CollecTorMain ctm = ctmEntry.getValue() + .getConstructor(Configuration.class).newInstance(conf); + scheduleExecutions(ctm, + conf.getInt(Key.valueOf(prefix + OFFSETMIN)), + conf.getInt(Key.valueOf(prefix + PERIODMIN))); + } + } catch (ConfigurationException | IllegalAccessException + | InstantiationException | InvocationTargetException + | NoSuchMethodException | RuntimeException ex) { + log.error("Cannot schedule " + ctmEntry.getValue().getName() + + ". Reason: " + ex.getMessage(), ex); + shutdownScheduler(); + throw new RuntimeException("Halted scheduling.", ex); + } + } + } + + private void scheduleExecutions(CollecTorMain ctm, int offset, int period) { + this.log.info("Periodic updater started for " + ctm.getClass().getName() + + "; offset=" + offset + ", period=" + period + "."); + int currentMinute = Calendar.getInstance().get(Calendar.MINUTE); + int initialDelay = (60 - currentMinute + offset) % 60; + + /* Run after initialDelay delay and then every period min. */ + this.log.info("Periodic updater will start every " + period + "th min " + + "at minute " + ((currentMinute + initialDelay) % 60) + "."); + this.scheduler.scheduleAtFixedRate(ctm, initialDelay, 60, + TimeUnit.MINUTES); + } + + /** + * Try to shutdown smoothly, i.e., wait for running tasks to terminate. + */ + public void shutdownScheduler() { + try { + scheduler.shutdown(); + scheduler.awaitTermination(20L, java.util.concurrent.TimeUnit.MINUTES); + log.info("Shutdown of all scheduled tasks completed successfully."); + } catch ( InterruptedException ie ) { + List<Runnable> notTerminated = scheduler.shutdownNow(); + log.error("Regular shutdown failed for: " + notTerminated); + if ( !notTerminated.isEmpty() ) { + log.error("Forced shutdown failed for: " + notTerminated); + } + } + } +} + diff --git a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java index 79fe19f..e6720cd 100644 --- a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java +++ b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java @@ -6,7 +6,7 @@ package org.torproject.collector.exitlists; import org.torproject.collector.conf.Configuration; import org.torproject.collector.conf.ConfigurationException; import org.torproject.collector.conf.Key; -import org.torproject.collector.main.LockFile; +import org.torproject.collector.cron.CollecTorMain; import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.DescriptorParser; @@ -32,42 +32,25 @@ import java.util.Stack; import java.util.TimeZone; import java.util.TreeSet; -public class ExitListDownloader extends Thread { +public class ExitListDownloader extends CollecTorMain { private static Logger logger = LoggerFactory.getLogger(ExitListDownloader.class); - /** Execute the exit-lists module using the given configuration. */ - public static void main(Configuration config) throws ConfigurationException { - logger.info("Starting exit-lists module of CollecTor."); - - // Use lock file to avoid overlapping runs - LockFile lf = new LockFile(config.getPath(Key.LockFilePath).toString(), "exit-lists"); - lf.acquireLock(); - - // Download exit list and store it to disk - new ExitListDownloader(config).run(); - - // Remove lock file - lf.releaseLock(); - - logger.info("Terminating exit-lists module of CollecTor."); - } - - private Configuration config; - /** Instanciate the exit-lists module using the given configuration. */ public ExitListDownloader(Configuration config) { - this.config = config; + super(config); } @Override public void run() { + logger.info("Starting exit-lists module of CollecTor."); try { startProcessing(); } catch (ConfigurationException ce) { logger.error("Configuration failed: " + ce, ce); throw new RuntimeException(ce); } + logger.info("Terminating exit-lists module of CollecTor."); } private void startProcessing() throws ConfigurationException { diff --git a/src/main/java/org/torproject/collector/index/CreateIndexJson.java b/src/main/java/org/torproject/collector/index/CreateIndexJson.java index 7da2b5e..80f183c 100644 --- a/src/main/java/org/torproject/collector/index/CreateIndexJson.java +++ b/src/main/java/org/torproject/collector/index/CreateIndexJson.java @@ -6,6 +6,7 @@ package org.torproject.collector.index; import org.torproject.collector.conf.Configuration; import org.torproject.collector.conf.ConfigurationException; import org.torproject.collector.conf.Key; +import org.torproject.collector.cron.CollecTorMain; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -35,7 +36,7 @@ import java.util.zip.GZIPOutputStream; * cache index parts of directories or files that haven't changed. * Example: if we parse include cryptographic hashes or @type information, * we'll likely have to do that. */ -public class CreateIndexJson { +public class CreateIndexJson extends CollecTorMain { private static File indexJsonFile; @@ -51,14 +52,24 @@ public class CreateIndexJson { /** Creates indexes of directories containing archived and recent * descriptors and write index files to disk. */ - public static void main(Configuration config) - throws ConfigurationException, IOException { - indexJsonFile = new File(config.getPath(Key.IndexPath).toFile(), "index.json"); - basePath = config.getProperty(Key.InstanceBaseUrl.name()); - indexedDirectories = new File[] { - config.getPath(Key.ArchivePath).toFile(), - config.getPath(Key.RecentPath).toFile() }; - writeIndex(indexDirectories()); + public CreateIndexJson(Configuration conf) { + super(conf); + } + + @Override + public void run() { + try { + indexJsonFile = new File(config.getPath(Key.IndexPath).toFile(), + "index.json"); + basePath = config.getProperty(Key.InstanceBaseUrl.name()); + indexedDirectories = new File[] { + config.getPath(Key.ArchivePath).toFile(), + config.getPath(Key.RecentPath).toFile() }; + writeIndex(indexDirectories()); + } catch (Exception e) { + throw new RuntimeException("Cannot run index creation: " + e.getMessage(), + e); + } } static class DirectoryNode implements Comparable<DirectoryNode> { diff --git a/src/main/java/org/torproject/collector/main/LockFile.java b/src/main/java/org/torproject/collector/main/LockFile.java deleted file mode 100644 index 977b0b9..0000000 --- a/src/main/java/org/torproject/collector/main/LockFile.java +++ /dev/null @@ -1,66 +0,0 @@ -/* Copyright 2010--2016 The Tor Project - * See LICENSE for licensing information */ - -package org.torproject.collector.main; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; - -public class LockFile { - - private final File lockFile; - private final String moduleName; - private final Logger logger = LoggerFactory.getLogger(LockFile.class); - - public LockFile(String moduleName) { - this("lock", moduleName); - } - - public LockFile(String lockFilePath, String moduleName) { - this.lockFile = new File(lockFilePath, moduleName); - this.moduleName = moduleName; - } - - /** Acquires the lock by checking whether a lock file already exists, - * and if not, by creating one with the current system time as - * content. */ - public boolean acquireLock() { - this.logger.debug("Trying to acquire lock..."); - try { - if (this.lockFile.exists()) { - BufferedReader br = new BufferedReader(new FileReader( - this.lockFile)); - long runStarted = Long.parseLong(br.readLine()); - br.close(); - if (System.currentTimeMillis() - runStarted < 55L * 60L * 1000L) { - throw new RuntimeException("Cannot acquire lock for " + moduleName); - } - } - this.lockFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter( - this.lockFile)); - bw.append("" + System.currentTimeMillis() + "\n"); - bw.close(); - this.logger.debug("Acquired lock."); - return true; - } catch (IOException e) { - throw new RuntimeException("Caught exception while trying to acquire " - + "lock for " + moduleName); - } - } - - /** Releases the lock by deleting the lock file, if present. */ - public void releaseLock() { - this.logger.debug("Releasing lock..."); - this.lockFile.delete(); - this.logger.debug("Released lock."); - } -} - diff --git a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java index db05bc5..cbab5ea 100644 --- a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java +++ b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java @@ -6,7 +6,7 @@ package org.torproject.collector.relaydescs; import org.torproject.collector.conf.Configuration; import org.torproject.collector.conf.ConfigurationException; import org.torproject.collector.conf.Key; -import org.torproject.collector.main.LockFile; +import org.torproject.collector.cron.CollecTorMain; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.DescriptorParser; import org.torproject.descriptor.DescriptorSourceFactory; @@ -38,12 +38,10 @@ import java.util.Stack; import java.util.TimeZone; import java.util.TreeMap; -public class ArchiveWriter extends Thread { +public class ArchiveWriter extends CollecTorMain { private static Logger logger = LoggerFactory.getLogger(ArchiveWriter.class); - private Configuration config; - private long now = System.currentTimeMillis(); private String outputDirectory; private String rsyncCatString; @@ -99,64 +97,45 @@ public class ArchiveWriter extends Thread { private StringBuilder intermediateStats = new StringBuilder(); - private static Path recentPath; - private static String recentPathName; + private Path recentPath; + private String recentPathName; private static final String RELAY_DESCRIPTORS = "relay-descriptors"; private static final String MICRO = "micro"; private static final String CONSENSUS_MICRODESC = "consensus-microdesc"; private static final String MICRODESC = "microdesc"; private static final String MICRODESCS = "microdescs"; - /** Executes the relay-descriptors module using the given - * configuration. */ - public static void main(Configuration config) throws ConfigurationException { - - logger.info("Starting relay-descriptors module of CollecTor."); - - // Use lock file to avoid overlapping runs - LockFile lf = new LockFile(config.getPath(Key.LockFilePath).toString(), RELAY_DESCRIPTORS); - lf.acquireLock(); - - recentPath = config.getPath(Key.RecentPath); - recentPathName = recentPath.toString(); - - // Import/download relay descriptors from the various sources - new ArchiveWriter(config).run(); - - new ReferenceChecker( - recentPath.toFile(), - new File(config.getPath(Key.StatsPath).toFile(), "references"), - new File(config.getPath(Key.StatsPath).toFile(), "references-history")).check(); - - // Remove lock file - lf.releaseLock(); - - logger.info("Terminating relay-descriptors module of CollecTor."); - } - /** Initialize an archive writer with a given configuration. */ public ArchiveWriter(Configuration config) throws ConfigurationException { - this.config = config; - storedServerDescriptorsFile = - new File(config.getPath(Key.StatsPath).toFile(), "stored-server-descriptors"); - storedExtraInfoDescriptorsFile = - new File(config.getPath(Key.StatsPath).toFile(), "stored-extra-info-descriptors"); - storedMicrodescriptorsFile = - new File(config.getPath(Key.StatsPath).toFile(), "stored-microdescriptors"); + super(config); } @Override public void run() { + logger.info("Starting relay-descriptors module of CollecTor."); try { + recentPath = config.getPath(Key.RecentPath); + recentPathName = recentPath.toString(); + File statsDir = config.getPath(Key.StatsPath).toFile(); + storedServerDescriptorsFile = + new File(statsDir, "stored-server-descriptors"); + storedExtraInfoDescriptorsFile = + new File(statsDir, "stored-extra-info-descriptors"); + storedMicrodescriptorsFile = + new File(statsDir, "stored-microdescriptors"); + startProcessing(); + new ReferenceChecker(recentPath.toFile(), + new File(statsDir, "references"), + new File(statsDir, "references-history")).check(); } catch (ConfigurationException ce) { logger.error("Configuration failed: " + ce, ce); throw new RuntimeException(ce); } + logger.info("Terminating relay-descriptors module of CollecTor."); } private void startProcessing() throws ConfigurationException { - File statsDirectory = new File("stats"); this.outputDirectory = config.getPath(Key.DirectoryArchivesOutputDirectory).toString(); SimpleDateFormat rsyncCatFormat = new SimpleDateFormat( diff --git a/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java b/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java index 7616dd8..1fa2d41 100644 --- a/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java +++ b/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java @@ -6,7 +6,7 @@ package org.torproject.collector.torperf; import org.torproject.collector.conf.Configuration; import org.torproject.collector.conf.ConfigurationException; import org.torproject.collector.conf.Key; -import org.torproject.collector.main.LockFile; +import org.torproject.collector.cron.CollecTorMain; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,30 +32,11 @@ import java.util.TreeMap; /* Download possibly truncated Torperf .data and .extradata files from * configured sources, append them to the files we already have, and merge * the two files into the .tpf format. */ -public class TorperfDownloader extends Thread { +public class TorperfDownloader extends CollecTorMain { private static Logger logger = LoggerFactory.getLogger(TorperfDownloader.class); - /** Executes the torperf module using the given configuration. */ - public static void main(Configuration config) throws ConfigurationException { - logger.info("Starting torperf module of CollecTor."); - - // Use lock file to avoid overlapping runs - LockFile lf = new LockFile(config.getPath(Key.LockFilePath).toString(), "torperf"); - lf.acquireLock(); - - // Process Torperf files - new TorperfDownloader(config).run(); - - // Remove lock file - lf.releaseLock(); - - logger.info("Terminating torperf module of CollecTor."); - } - - private Configuration config; - public TorperfDownloader(Configuration config) { - this.config = config; + super(config); } private File torperfOutputDirectory = null; @@ -66,12 +47,14 @@ public class TorperfDownloader extends Thread { @Override public void run() { + logger.info("Starting torperf module of CollecTor."); try { startProcessing(); } catch (ConfigurationException ce) { logger.error("Configuration failed: " + ce, ce); throw new RuntimeException(ce); } + logger.info("Terminating torperf module of CollecTor."); } private void startProcessing() throws ConfigurationException { @@ -309,9 +292,6 @@ public class TorperfDownloader extends Thread { private String mergeFiles(File dataFile, File extradataFile, String source, int fileSize, String skipUntil) throws IOException, ConfigurationException { - SortedMap<String, String> config = new TreeMap<String, String>(); - config.put("SOURCE", source); - config.put("FILESIZE", String.valueOf(fileSize)); if (!dataFile.exists() || !extradataFile.exists()) { this.logger.warn("File " + dataFile.getAbsolutePath() + " or " + extradataFile.getAbsolutePath() + " is missing."); @@ -426,11 +406,12 @@ public class TorperfDownloader extends Thread { /* Write output line to .tpf file. */ SortedMap<String, String> keysAndValues = new TreeMap<String, String>(); + keysAndValues.put("SOURCE", source); + keysAndValues.put("FILESIZE", String.valueOf(fileSize)); if (extradata != null) { keysAndValues.putAll(extradata); } keysAndValues.putAll(data); - keysAndValues.putAll(config); this.logger.debug("Writing " + dataFile.getName() + ":" + skippedLineCount++ + "."); lineD = brD.readLine(); diff --git a/src/test/java/org/torproject/collector/MainTest.java b/src/test/java/org/torproject/collector/MainTest.java index 5991f78..b48a0a9 100644 --- a/src/test/java/org/torproject/collector/MainTest.java +++ b/src/test/java/org/torproject/collector/MainTest.java @@ -4,16 +4,19 @@ package org.torproject.collector; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import org.torproject.collector.conf.Key; +import org.torproject.collector.cron.Scheduler; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import java.io.BufferedWriter; +import java.io.IOException; import java.io.File; import java.nio.file.Files; import java.util.List; @@ -27,23 +30,54 @@ public class MainTest { @Rule public TemporaryFolder tmpf = new TemporaryFolder(); + @Test(expected = IOException.class) + public void testInitializationConfigException() throws Exception { + File conf = new File(Main.CONF_FILE); + assertFalse("Please remove " + Main.CONF_FILE + " before running tests!", conf.exists()); + Main.main(new String[] {"/tmp/"}); + assertTrue(conf.exists()); + assertTrue(conf.delete()); + } + + @Test() + public void testInitializationNullArgs() throws Exception { + File conf = new File(Main.CONF_FILE); + assertFalse("Please remove " + Main.CONF_FILE + " before running tests!", conf.exists()); + Main.main(null); + assertTrue(conf.exists()); + assertTrue(conf.delete()); + } + + @Test() + public void testInitializationEmptyArgs() throws Exception { + File conf = new File(Main.CONF_FILE); + assertFalse("Please remove " + Main.CONF_FILE + " before running tests!", conf.exists()); + Main.main(new String[]{}); + assertTrue(conf.exists()); + assertTrue(conf.delete()); + } + + @Test() + public void testInitializationTooManyArgs() throws Exception { + File conf = new File(Main.CONF_FILE); + assertFalse("Please remove " + Main.CONF_FILE + " before running tests!", conf.exists()); + Main.main(new String[]{"x", "y"}); + assertFalse(conf.exists()); + } + @Test() public void testSmoke() throws Exception { - System.out.println("\n!!!! Three ERROR log messages are expected." - + "\nOne each from: ExitListDownloader, " - + "TorperfDownloader, and CreateIndexJson.\n"); File conf = tmpf.newFile("test.conf"); File lockPath = tmpf.newFolder("test.lock"); assertEquals(0L, conf.length()); - Main.main(new String[]{"relaydescs", conf.toString()}); + Main.main(new String[]{conf.toString()}); assertTrue(4_000L <= conf.length()); - changeLockFilePath(conf, lockPath); - for ( String key : Main.collecTorMains.keySet()) { - Main.main(new String[]{key, conf.toString()}); - } + changeFilePathsAndSetActivation(conf, lockPath, "TorperfActivated"); + Main.main(new String[]{conf.toString()}); + for(int t=0; t<1_000_000; t++) { } } - private void changeLockFilePath(File f, File l) throws Exception { + private void changeFilePathsAndSetActivation(File f, File l, String a) throws Exception { List<String> lines = Files.readAllLines(f.toPath()); BufferedWriter bw = Files.newBufferedWriter(f.toPath()); File in = tmpf.newFolder(); @@ -57,6 +91,8 @@ public class MainTest { line = line.replace(inStr, in.toString() + inStr); } else if (line.contains(outStr)) { line = line.replace(outStr, out.toString() + outStr); + } else if (line.contains(a)) { + line = line.replace("false", "true"); } bw.write(line); bw.newLine(); @@ -86,5 +122,35 @@ public class MainTest { } } } + + /* Verifies that every collecTorMain class is configured in the + * default collector.properties file and the other way around. */ + @Test() + public void testRunConfiguration() throws Exception { + Properties props = new Properties(); + props.load(getClass().getClassLoader().getResourceAsStream( + Main.CONF_FILE)); + String[] runConfigSettings = new String[] {Scheduler.ACTIVATED, + Scheduler.PERIODMIN, Scheduler.OFFSETMIN}; + for (Key key : Main.collecTorMains.keySet()) { + for ( String part : runConfigSettings ){ + String key2 = key.name().replace("Activated", part); + assertNotNull("Property '" + key2 + "' not specified in " + + Main.CONF_FILE + ".", + props.getProperty(key2)); + } + } + for (String propName : props.stringPropertyNames()) { + for ( String part : runConfigSettings ){ + if( propName.contains(part) ){ + String key2 = propName.replace(part, ""); + assertTrue("CollecTorMain '" + key2 + + "' not specified in Main.class.", + Main.collecTorMains.containsKey(Key.valueOf(key2 + "Activated"))); + } + } + } + } + } diff --git a/src/test/java/org/torproject/collector/cron/Dummy.java b/src/test/java/org/torproject/collector/cron/Dummy.java new file mode 100644 index 0000000..0231e69 --- /dev/null +++ b/src/test/java/org/torproject/collector/cron/Dummy.java @@ -0,0 +1,15 @@ +package org.torproject.collector.cron; + +import org.torproject.collector.conf.Configuration; + +public class Dummy extends CollecTorMain { + + public Dummy(Configuration c) { + super(c); + } + + @Override + public void run() { + + } +} diff --git a/src/test/java/org/torproject/collector/cron/SchedulerTest.java b/src/test/java/org/torproject/collector/cron/SchedulerTest.java new file mode 100644 index 0000000..0c4e922 --- /dev/null +++ b/src/test/java/org/torproject/collector/cron/SchedulerTest.java @@ -0,0 +1,54 @@ +/* Copyright 2016 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.collector.cron; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.torproject.collector.conf.Key; +import org.torproject.collector.conf.Configuration; +import org.torproject.collector.cron.Scheduler; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.BufferedWriter; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.util.List; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.Random; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; + +public class SchedulerTest { + + private static final String runConfigProperties = "TorperfActivated=true\n" + + "TorperfPeriodMinutes=10\nTorperfOffsetMinutes=7\n"; + + @Test() + public void testSimpleSchedule() throws Exception { + Map<Key, Class<? extends CollecTorMain>> ctms = new HashMap<>(); + Configuration conf = new Configuration(); + conf.load(new ByteArrayInputStream(runConfigProperties.getBytes())); + ctms.put(Key.TorperfActivated, Dummy.class); + Field schedulerField = Scheduler.class.getDeclaredField("scheduler"); + schedulerField.setAccessible(true); + ScheduledThreadPoolExecutor stpe = (ScheduledThreadPoolExecutor) + schedulerField.get(Scheduler.getInstance()); + assertTrue(stpe.getQueue().isEmpty()); + Scheduler.getInstance().scheduleModuleRuns(ctms, conf); + assertEquals(stpe.getQueue().size(), 1); + Scheduler.getInstance().shutdownScheduler(); + assertTrue(stpe.isShutdown()); + } + +} + diff --git a/src/test/resources/junittest.policy b/src/test/resources/junittest.policy index 208a172..35c30c0 100644 --- a/src/test/resources/junittest.policy +++ b/src/test/resources/junittest.policy @@ -5,6 +5,7 @@ grant { permission java.util.PropertyPermission "*", "read, write"; permission java.lang.RuntimePermission "setIO"; permission java.lang.RuntimePermission "accessDeclaredMembers"; + permission java.lang.RuntimePermission "modifyThread"; permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; permission java.lang.RuntimePermission "shutdownHooks"; permission java.lang.RuntimePermission "accessClassInPackage.sun.reflect";

1 0

[collector/master] Added new properties for scheduled module runs and adapted tests accordingly. Scheduler implementation open. Partially implements task-19018.
by karsten＠torproject.org 25 Jul '16

25 Jul '16

commit 62c94124d1e7bcf7bfa55465434818b19749c0ad Author: iwakeh <iwakeh(a)torproject.org> Date: Wed Jul 20 11:11:37 2016 +0200 Added new properties for scheduled module runs and adapted tests accordingly. Scheduler implementation open. Partially implements task-19018. --- .../java/org/torproject/collector/conf/Key.java | 15 ++++++++++ src/main/resources/collector.properties | 32 +++++++++++++++++++++- .../collector/conf/ConfigurationTest.java | 2 +- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/torproject/collector/conf/Key.java b/src/main/java/org/torproject/collector/conf/Key.java index b4119b6..d8c87ac 100644 --- a/src/main/java/org/torproject/collector/conf/Key.java +++ b/src/main/java/org/torproject/collector/conf/Key.java @@ -17,6 +17,21 @@ public enum Key { RecentPath(Path.class), IndexPath(Path.class), StatsPath(Path.class), + BridgedescsActivated(Boolean.class), + BridgedescsOffsetMinutes(Integer.class), + BridgedescsPeriodMinutes(Integer.class), + ExitlistsActivated(Boolean.class), + ExitlistsOffsetMinutes(Integer.class), + ExitlistsPeriodMinutes(Integer.class), + RelaydescsActivated(Boolean.class), + RelaydescsOffsetMinutes(Integer.class), + RelaydescsPeriodMinutes(Integer.class), + TorperfActivated(Boolean.class), + TorperfOffsetMinutes(Integer.class), + TorperfPeriodMinutes(Integer.class), + UpdateindexActivated(Boolean.class), + UpdateindexOffsetMinutes(Integer.class), + UpdateindexPeriodMinutes(Integer.class), BridgeSnapshotsDirectory(Path.class), CachedRelayDescriptorsDirectories(String[].class), CompressRelayDescriptorDownloads(Boolean.class), diff --git a/src/main/resources/collector.properties b/src/main/resources/collector.properties index 76ae6bd..8996a1f 100644 --- a/src/main/resources/collector.properties +++ b/src/main/resources/collector.properties @@ -1,5 +1,36 @@ ######## Collector Properties # +######## Run Configuration ######## +## the following defines, if this module is activated +BridgedescsActivated = false +# period in minutes +BridgedescsPeriodMinutes = 60 +# offset in minutes since the epoch and +BridgedescsOffsetMinutes = 9 +## the following defines, if this module is activated +ExitlistsActivated = false +# period in minutes +ExitlistsPeriodMinutes = 60 +# offset in minutes since the epoch and +ExitlistsOffsetMinutes = 2 +## the following defines, if this module is activated +RelaydescsActivated = false +# period in minutes +RelaydescsPeriodMinutes = 30 +# offset in minutes since the epoch and +RelaydescsOffsetMinutes = 5 +## the following defines, if this module is activated +TorperfActivated = false +# period in minutes +TorperfPeriodMinutes = 360 +# offset in minutes since the epoch and +TorperfOffsetMinutes = 1 +# the following defines, if this module is activated +UpdateindexActivated = false +# period in minutes +UpdateindexPeriodMinutes = 2 +# offset in minutes since the epoch and +UpdateindexOffsetMinutes = 0 ######## General Properties ######## InstanceBaseUrl = "https://collector.torproject.org" LockFilePath = lock @@ -7,7 +38,6 @@ IndexPath = out/index ArchivePath = out/archive RecentPath = out/recent StatsPath = out/stats - ######## Relay descriptors ######## # ## Read cached-* files from a local Tor data directory diff --git a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java index ac9fd4f..437939b 100644 --- a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java +++ b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java @@ -22,7 +22,7 @@ public class ConfigurationTest { public void testKeyCount() throws Exception { assertEquals("The number of properties keys in enum Key changed." + "\n This test class should be adapted.", - 33, Key.values().length); + 48, Key.values().length); } @Test()

1 0

[collector/master] Make some trivial whitespace fixes.
by karsten＠torproject.org 25 Jul '16

25 Jul '16

commit 69535b64e20049e7a2f2f97b616322c857991e96 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Jul 25 17:00:45 2016 +0200 Make some trivial whitespace fixes. --- .../collector/bridgedescs/SanitizedBridgesWriter.java | 2 +- .../java/org/torproject/collector/cron/CollecTorMain.java | 2 +- .../java/org/torproject/collector/cron/Scheduler.java | 15 +++++++-------- .../collector/exitlists/ExitListDownloader.java | 2 +- .../org/torproject/collector/index/CreateIndexJson.java | 2 +- src/test/java/org/torproject/collector/MainTest.java | 12 ++++++------ .../org/torproject/collector/conf/ConfigurationTest.java | 12 ++++++------ 7 files changed, 23 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java index e00f70b..476f80b 100644 --- a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java +++ b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java @@ -392,7 +392,7 @@ public class SanitizedBridgesWriter extends CollecTorMain { if (this.bridgeSanitizingCutOffTimestamp .compareTo(publicationTime) > 0) { - String text = "Sanitizing and storing network status with " + String text = "Sanitizing and storing network status with " + "publication time outside our descriptor sanitizing " + "interval."; if (this.haveWarnedAboutInterval) { diff --git a/src/main/java/org/torproject/collector/cron/CollecTorMain.java b/src/main/java/org/torproject/collector/cron/CollecTorMain.java index 21d8948..5fe6376 100644 --- a/src/main/java/org/torproject/collector/cron/CollecTorMain.java +++ b/src/main/java/org/torproject/collector/cron/CollecTorMain.java @@ -23,7 +23,7 @@ public abstract class CollecTorMain implements Runnable { protected Configuration config; - public CollecTorMain( Configuration conf) { + public CollecTorMain(Configuration conf) { this.config = conf; } diff --git a/src/main/java/org/torproject/collector/cron/Scheduler.java b/src/main/java/org/torproject/collector/cron/Scheduler.java index 6bc90ca..5535a8a 100644 --- a/src/main/java/org/torproject/collector/cron/Scheduler.java +++ b/src/main/java/org/torproject/collector/cron/Scheduler.java @@ -53,10 +53,10 @@ public class Scheduler implements ThreadFactory { */ public void scheduleModuleRuns(Map<Key, Class<? extends CollecTorMain>> collecTorMains, Configuration conf) { - for ( Map.Entry<Key, Class<? extends CollecTorMain>> ctmEntry - : collecTorMains.entrySet() ) { + for (Map.Entry<Key, Class<? extends CollecTorMain>> ctmEntry + : collecTorMains.entrySet()) { try { - if ( conf.getBool(ctmEntry.getKey()) ) { + if (conf.getBool(ctmEntry.getKey())) { String prefix = ctmEntry.getKey().name().replace(ACTIVATED, ""); CollecTorMain ctm = ctmEntry.getValue() .getConstructor(Configuration.class).newInstance(conf); @@ -76,9 +76,9 @@ public class Scheduler implements ThreadFactory { private void scheduleExecutions(CollecTorMain ctm, int offset, int period) { this.log.info("Periodic updater started for " + ctm.getClass().getName() - + "; offset=" + offset + ", period=" + period + "."); + + "; offset=" + offset + ", period=" + period + "."); int currentMinute = Calendar.getInstance().get(Calendar.MINUTE); - int initialDelay = (period - (currentMinute % period) + offset) % period; + int initialDelay = (period - (currentMinute % period) + offset) % period; /* Run after initialDelay delay and then every period min. */ this.log.info("Periodic updater will start every " + period + "th min " @@ -96,10 +96,10 @@ public class Scheduler implements ThreadFactory { scheduler.shutdown(); scheduler.awaitTermination(20L, java.util.concurrent.TimeUnit.MINUTES); log.info("Shutdown of all scheduled tasks completed successfully."); - } catch ( InterruptedException ie ) { + } catch (InterruptedException ie) { List<Runnable> notTerminated = scheduler.shutdownNow(); log.error("Regular shutdown failed for: " + notTerminated); - if ( !notTerminated.isEmpty() ) { + if (!notTerminated.isEmpty()) { log.error("Forced shutdown failed for: " + notTerminated); } } @@ -115,6 +115,5 @@ public class Scheduler implements ThreadFactory { log.info("New Thread created: " + newThread.getName()); return newThread; } - } diff --git a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java index 8b5277b..9c02032 100644 --- a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java +++ b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java @@ -77,7 +77,7 @@ public class ExitListDownloader extends CollecTorMain { byte[] data = new byte[1024]; while ((len = in.read(data, 0, 1024)) >= 0) { sb.append(new String(data, 0, len)); - } + } in.close(); downloadedExitList = sb.toString(); logger.debug("Finished downloading exit list."); diff --git a/src/main/java/org/torproject/collector/index/CreateIndexJson.java b/src/main/java/org/torproject/collector/index/CreateIndexJson.java index 08c28c6..39069f1 100644 --- a/src/main/java/org/torproject/collector/index/CreateIndexJson.java +++ b/src/main/java/org/torproject/collector/index/CreateIndexJson.java @@ -68,7 +68,7 @@ public class CreateIndexJson extends CollecTorMain { @Override protected void startProcessing() throws ConfigurationException { try { - indexJsonFile = new File(config.getPath(Key.IndexPath).toFile(), + indexJsonFile = new File(config.getPath(Key.IndexPath).toFile(), "index.json"); basePath = config.getProperty(Key.InstanceBaseUrl.name()); indexedDirectories = new File[] { diff --git a/src/test/java/org/torproject/collector/MainTest.java b/src/test/java/org/torproject/collector/MainTest.java index b48a0a9..6b90978 100644 --- a/src/test/java/org/torproject/collector/MainTest.java +++ b/src/test/java/org/torproject/collector/MainTest.java @@ -52,7 +52,7 @@ public class MainTest { public void testInitializationEmptyArgs() throws Exception { File conf = new File(Main.CONF_FILE); assertFalse("Please remove " + Main.CONF_FILE + " before running tests!", conf.exists()); - Main.main(new String[]{}); + Main.main(new String[] { }); assertTrue(conf.exists()); assertTrue(conf.delete()); } @@ -61,7 +61,7 @@ public class MainTest { public void testInitializationTooManyArgs() throws Exception { File conf = new File(Main.CONF_FILE); assertFalse("Please remove " + Main.CONF_FILE + " before running tests!", conf.exists()); - Main.main(new String[]{"x", "y"}); + Main.main(new String[] { "x", "y" }); assertFalse(conf.exists()); } @@ -74,7 +74,7 @@ public class MainTest { assertTrue(4_000L <= conf.length()); changeFilePathsAndSetActivation(conf, lockPath, "TorperfActivated"); Main.main(new String[]{conf.toString()}); - for(int t=0; t<1_000_000; t++) { } + for(int t = 0; t < 1_000_000; t++) { } } private void changeFilePathsAndSetActivation(File f, File l, String a) throws Exception { @@ -133,7 +133,7 @@ public class MainTest { String[] runConfigSettings = new String[] {Scheduler.ACTIVATED, Scheduler.PERIODMIN, Scheduler.OFFSETMIN}; for (Key key : Main.collecTorMains.keySet()) { - for ( String part : runConfigSettings ){ + for (String part : runConfigSettings) { String key2 = key.name().replace("Activated", part); assertNotNull("Property '" + key2 + "' not specified in " + Main.CONF_FILE + ".", @@ -141,8 +141,8 @@ public class MainTest { } } for (String propName : props.stringPropertyNames()) { - for ( String part : runConfigSettings ){ - if( propName.contains(part) ){ + for (String part : runConfigSettings) { + if (propName.contains(part)) { String key2 = propName.replace(part, ""); assertTrue("CollecTorMain '" + key2 + "' not specified in Main.class.", diff --git a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java index 437939b..8671ef4 100644 --- a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java +++ b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java @@ -100,42 +100,42 @@ public class ConfigurationTest { assertArrayEquals(sourceStrings, conf.getStringArrayArray(Key.TorperfSources)); } - @Test( expected = ConfigurationException.class) + @Test(expected = ConfigurationException.class) public void testArrayArrayValueException() throws Exception { Configuration conf = new Configuration(); conf.load(new ByteArrayInputStream("CachedRelayDescriptorsDirectories".getBytes())); conf.getStringArrayArray(Key.TorperfOutputDirectory); } - @Test( expected = ConfigurationException.class) + @Test(expected = ConfigurationException.class) public void testArrayValueException() throws Exception { Configuration conf = new Configuration(); conf.load(new ByteArrayInputStream("CachedRelayDescriptorsDirectories".getBytes())); conf.getStringArray(Key.TorperfSources); } - @Test( expected = ConfigurationException.class) + @Test(expected = ConfigurationException.class) public void testBoolValueException() throws Exception { Configuration conf = new Configuration(); conf.load(new ByteArrayInputStream("TorperfSource = http://x.y.z".getBytes())); conf.getBool(Key.CachedRelayDescriptorsDirectories); } - @Test( expected = ConfigurationException.class) + @Test(expected = ConfigurationException.class) public void testPathValueException() throws Exception { Configuration conf = new Configuration(); conf.load(new ByteArrayInputStream("DirectoryArchivesDirectory = \\u0000:".getBytes())); conf.getPath(Key.DirectoryArchivesDirectory); } - @Test( expected = ConfigurationException.class) + @Test(expected = ConfigurationException.class) public void testUrlValueException() throws Exception { Configuration conf = new Configuration(); conf.load(new ByteArrayInputStream("ExitlistUrl = xxx://y.y.y".getBytes())); conf.getUrl(Key.ExitlistUrl); } - @Test( expected = ConfigurationException.class) + @Test(expected = ConfigurationException.class) public void testIntValueException() throws Exception { Configuration conf = new Configuration(); conf.load(new ByteArrayInputStream("BridgeDescriptorMappingsLimit = y7".getBytes()));

1 0

[collector/master] Changed another hard-coded directory reference.
by karsten＠torproject.org 25 Jul '16

25 Jul '16

commit 40ae5159959d07bb5b6b83494e687ef24b63bc90 Author: iwakeh <iwakeh(a)torproject.org> Date: Wed Jul 20 16:30:31 2016 +0200 Changed another hard-coded directory reference. --- src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java index cbab5ea..4ad87f5 100644 --- a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java +++ b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java @@ -136,7 +136,7 @@ public class ArchiveWriter extends CollecTorMain { } private void startProcessing() throws ConfigurationException { - File statsDirectory = new File("stats"); + File statsDirectory = config.getPath(Key.StatsPath).toFile(); this.outputDirectory = config.getPath(Key.DirectoryArchivesOutputDirectory).toString(); SimpleDateFormat rsyncCatFormat = new SimpleDateFormat( "yyyy-MM-dd-HH-mm-ss");

1 0

[collector/master] Add missing = in properties template.
by karsten＠torproject.org 25 Jul '16

25 Jul '16

commit 50463327de8cd35c6b2625b0cdb32ae00b3238e8 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Jul 25 17:01:09 2016 +0200 Add missing = in properties template. --- src/main/resources/collector.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/collector.properties b/src/main/resources/collector.properties index 8996a1f..e7d41ae 100644 --- a/src/main/resources/collector.properties +++ b/src/main/resources/collector.properties @@ -94,7 +94,7 @@ DownloadMissingMicrodescriptors = true # ## Download all server descriptors from the directory authorities at most ## once a day (only if DownloadRelayDescriptors is true) -DownloadAllServerDescriptors false +DownloadAllServerDescriptors = false # ## Download all extra-info descriptors from the directory authorities at ## most once a day (only if DownloadRelayDescriptors is true)

1 0

[onionoo/master] Upgrade to metrics-lib 1.3.0.
by karsten＠torproject.org 25 Jul '16

25 Jul '16

commit c0c7ee989580b67a6147487d14aa986526180744 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Jul 20 09:46:33 2016 +0200 Upgrade to metrics-lib 1.3.0. --- build.xml | 2 +- src/test/java/org/torproject/onionoo/updater/DummyConsensus.java | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/build.xml b/build.xml index f7f6bc0..467c991 100644 --- a/build.xml +++ b/build.xml @@ -3,7 +3,7 @@ <property name="onionoo.protocol.version" value="3.1"/> <property name="release.version" value="${onionoo.protocol.version}.0"/> - <property name="descriptorversion" value="1.2.0"/> + <property name="descriptorversion" value="1.3.0"/> <property name="javasources" value="src/main/java"/> <property name="tests" value="src/test/java"/> <property name="generated" value="generated/"/> diff --git a/src/test/java/org/torproject/onionoo/updater/DummyConsensus.java b/src/test/java/org/torproject/onionoo/updater/DummyConsensus.java index 0f01a5e..af03ccf 100644 --- a/src/test/java/org/torproject/onionoo/updater/DummyConsensus.java +++ b/src/test/java/org/torproject/onionoo/updater/DummyConsensus.java @@ -115,5 +115,13 @@ public class DummyConsensus implements RelayNetworkStatusConsensus { public String getConsensusDigest() { return null; } + + public List<String> getPackageLines() { + return null; + } + + public List<DirectorySignature> getSignatures() { + return null; + } }

1 0

[onionoo/master] Resolve or suppress remaining checkstyle warnings.
by karsten＠torproject.org 25 Jul '16

25 Jul '16

commit 01c53dd0a261acfdad467fc5450abf5b12d42063 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jul 19 16:42:15 2016 +0200 Resolve or suppress remaining checkstyle warnings. Implements the rest of #19613. --- .../torproject/onionoo/docs/BandwidthDocument.java | 1 + .../torproject/onionoo/docs/ClientsDocument.java | 1 + .../torproject/onionoo/docs/DetailsDocument.java | 41 +++---- .../org/torproject/onionoo/docs/DetailsStatus.java | 49 +++++---- .../org/torproject/onionoo/docs/DocumentStore.java | 4 +- .../org/torproject/onionoo/docs/NodeStatus.java | 18 +-- .../torproject/onionoo/docs/SummaryDocument.java | 15 +-- .../torproject/onionoo/docs/UptimeDocument.java | 1 + .../torproject/onionoo/docs/WeightsDocument.java | 1 + .../org/torproject/onionoo/docs/WeightsStatus.java | 6 +- .../org/torproject/onionoo/server/Counter.java | 22 ++++ .../onionoo/server/HttpServletRequestWrapper.java | 1 + .../onionoo/server/IntegerDistribution.java | 50 +++++++++ .../onionoo/server/MostFrequentString.java | 65 +++++++++++ .../org/torproject/onionoo/server/NodeIndex.java | 12 +- .../org/torproject/onionoo/server/NodeIndexer.java | 25 +++-- .../onionoo/server/PerformanceMetrics.java | 122 --------------------- .../torproject/onionoo/server/RequestHandler.java | 18 +-- .../torproject/onionoo/server/ResourceServlet.java | 46 ++++---- .../onionoo/updater/DescriptorDownloader.java | 10 +- .../onionoo/updater/DescriptorSource.java | 6 + .../torproject/onionoo/updater/LookupService.java | 27 ++--- .../onionoo/updater/NodeDetailsStatusUpdater.java | 8 +- .../onionoo/writer/BandwidthDocumentWriter.java | 4 +- .../onionoo/writer/DetailsDocumentWriter.java | 4 +- .../onionoo/writer/SummaryDocumentWriter.java | 8 +- 26 files changed, 301 insertions(+), 264 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/docs/BandwidthDocument.java b/src/main/java/org/torproject/onionoo/docs/BandwidthDocument.java index 6fe3240..1990734 100644 --- a/src/main/java/org/torproject/onionoo/docs/BandwidthDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/BandwidthDocument.java @@ -5,6 +5,7 @@ package org.torproject.onionoo.docs; import java.util.Map; +@SuppressWarnings("checkstyle:membername") public class BandwidthDocument extends Document { @SuppressWarnings("unused") diff --git a/src/main/java/org/torproject/onionoo/docs/ClientsDocument.java b/src/main/java/org/torproject/onionoo/docs/ClientsDocument.java index 89729d2..f8b0c84 100644 --- a/src/main/java/org/torproject/onionoo/docs/ClientsDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/ClientsDocument.java @@ -5,6 +5,7 @@ package org.torproject.onionoo.docs; import java.util.Map; +@SuppressWarnings("checkstyle:membername") public class ClientsDocument extends Document { @SuppressWarnings("unused") diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java index 85c0154..a9257af 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java @@ -10,6 +10,7 @@ import java.util.List; import java.util.Map; import java.util.SortedSet; +@SuppressWarnings("checkstyle:membername") public class DetailsDocument extends Document { /* We must ensure that details files only contain ASCII characters @@ -20,12 +21,12 @@ public class DetailsDocument extends Document { * to write six characters '\', 'u', '0', '0', 'F', '2'. The only thing * we'll have to do is to change back the '\\' that Gson writes for the * '\'. */ - private static String escapeJSON(String s) { - return StringEscapeUtils.escapeJava(s); + private static String escapeJson(String stringToEscape) { + return StringEscapeUtils.escapeJava(stringToEscape); } - private static String unescapeJSON(String s) { - return StringEscapeUtils.unescapeJava(s); + private static String unescapeJson(String stringToUnescape) { + return StringEscapeUtils.unescapeJava(stringToUnescape); } private String nickname; @@ -154,31 +155,31 @@ public class DetailsDocument extends Document { private String country_name; public void setCountryName(String countryName) { - this.country_name = escapeJSON(countryName); + this.country_name = escapeJson(countryName); } public String getCountryName() { - return unescapeJSON(this.country_name); + return unescapeJson(this.country_name); } private String region_name; public void setRegionName(String regionName) { - this.region_name = escapeJSON(regionName); + this.region_name = escapeJson(regionName); } public String getRegionName() { - return unescapeJSON(this.region_name); + return unescapeJson(this.region_name); } private String city_name; public void setCityName(String cityName) { - this.city_name = escapeJSON(cityName); + this.city_name = escapeJson(cityName); } public String getCityName() { - return unescapeJSON(this.city_name); + return unescapeJson(this.city_name); } private Float latitude; @@ -204,21 +205,21 @@ public class DetailsDocument extends Document { private String as_number; public void setAsNumber(String asNumber) { - this.as_number = escapeJSON(asNumber); + this.as_number = escapeJson(asNumber); } public String getAsNumber() { - return unescapeJSON(this.as_number); + return unescapeJson(this.as_number); } private String as_name; public void setAsName(String asName) { - this.as_name = escapeJSON(asName); + this.as_name = escapeJson(asName); } public String getAsName() { - return unescapeJSON(this.as_name); + return unescapeJson(this.as_name); } private Long consensus_weight; @@ -234,11 +235,11 @@ public class DetailsDocument extends Document { private String host_name; public void setHostName(String hostName) { - this.host_name = escapeJSON(hostName); + this.host_name = escapeJson(hostName); } public String getHostName() { - return unescapeJSON(this.host_name); + return unescapeJson(this.host_name); } private String last_restarted; @@ -328,21 +329,21 @@ public class DetailsDocument extends Document { private String contact; public void setContact(String contact) { - this.contact = escapeJSON(contact); + this.contact = escapeJson(contact); } public String getContact() { - return unescapeJSON(this.contact); + return unescapeJson(this.contact); } private String platform; public void setPlatform(String platform) { - this.platform = escapeJSON(platform); + this.platform = escapeJson(platform); } public String getPlatform() { - return unescapeJSON(this.platform); + return unescapeJson(this.platform); } private SortedSet<String> alleged_family; diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java index 1a3c05d..7258054 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java @@ -10,6 +10,7 @@ import java.util.Map; import java.util.SortedSet; import java.util.TreeSet; +@SuppressWarnings("checkstyle:membername") public class DetailsStatus extends Document { /* We must ensure that details files only contain ASCII characters @@ -20,12 +21,12 @@ public class DetailsStatus extends Document { * to write six characters '\', 'u', '0', '0', 'F', '2'. The only thing * we'll have to do is to change back the '\\' that Gson writes for the * '\'. */ - private static String escapeJSON(String s) { - return StringEscapeUtils.escapeJava(s); + private static String escapeJson(String stringToEscape) { + return StringEscapeUtils.escapeJava(stringToEscape); } - private static String unescapeJSON(String s) { - return StringEscapeUtils.unescapeJava(s); + private static String unescapeJson(String stringToUnescape) { + return StringEscapeUtils.unescapeJava(stringToUnescape); } /* From most recently published server descriptor: */ @@ -105,21 +106,21 @@ public class DetailsStatus extends Document { private String contact; public void setContact(String contact) { - this.contact = escapeJSON(contact); + this.contact = escapeJson(contact); } public String getContact() { - return unescapeJSON(this.contact); + return unescapeJson(this.contact); } private String platform; public void setPlatform(String platform) { - this.platform = escapeJSON(platform); + this.platform = escapeJson(platform); } public String getPlatform() { - return unescapeJSON(this.platform); + return unescapeJson(this.platform); } private SortedSet<String> alleged_family; @@ -470,51 +471,51 @@ public class DetailsStatus extends Document { private String country_name; public void setCountryName(String countryName) { - this.country_name = escapeJSON(countryName); + this.country_name = escapeJson(countryName); } public String getCountryName() { - return unescapeJSON(this.country_name); + return unescapeJson(this.country_name); } private String region_name; public void setRegionName(String regionName) { - this.region_name = escapeJSON(regionName); + this.region_name = escapeJson(regionName); } public String getRegionName() { - return unescapeJSON(this.region_name); + return unescapeJson(this.region_name); } private String city_name; public void setCityName(String cityName) { - this.city_name = escapeJSON(cityName); + this.city_name = escapeJson(cityName); } public String getCityName() { - return unescapeJSON(this.city_name); + return unescapeJson(this.city_name); } private String as_name; - public void setASName(String aSName) { - this.as_name = escapeJSON(aSName); + public void setAsName(String asName) { + this.as_name = escapeJson(asName); } - public String getASName() { - return unescapeJSON(this.as_name); + public String getAsName() { + return unescapeJson(this.as_name); } private String as_number; - public void setASNumber(String aSNumber) { - this.as_number = escapeJSON(aSNumber); + public void setAsNumber(String asNumber) { + this.as_number = escapeJson(asNumber); } - public String getASNumber() { - return unescapeJSON(this.as_number); + public String getAsNumber() { + return unescapeJson(this.as_number); } /* Reverse DNS lookup result: */ @@ -522,11 +523,11 @@ public class DetailsStatus extends Document { private String host_name; public void setHostName(String hostName) { - this.host_name = escapeJSON(hostName); + this.host_name = escapeJson(hostName); } public String getHostName() { - return unescapeJSON(this.host_name); + return unescapeJson(this.host_name); } } diff --git a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java index 42c75aa..e1be777 100644 --- a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java +++ b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java @@ -420,7 +420,7 @@ public class DocumentStore { String nickname = detailsDocument.getNickname(); List<String> addresses = new ArrayList<String>(); String countryCode = null; - String aSNumber = null; + String asNumber = null; String contact = null; for (String orAddressAndPort : detailsDocument.getOrAddresses()) { if (!orAddressAndPort.contains(":")) { @@ -451,7 +451,7 @@ public class DocumentStore { SummaryDocument summaryDocument = new SummaryDocument(isRelay, nickname, fingerprint, addresses, lastSeenMillis, running, relayFlags, consensusWeight, countryCode, firstSeenMillis, - aSNumber, contact, family, family); + asNumber, contact, family, family); return summaryDocument; } diff --git a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java index afdd6c6..87cacb4 100644 --- a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java @@ -67,9 +67,9 @@ public class NodeStatus extends Document { String[] stringArray = null; if (collection != null && !collection.isEmpty()) { stringArray = new String[collection.size()]; - int i = 0; + int index = 0; for (String string : collection) { - stringArray[i++] = string; + stringArray[index++] = string; } } return stringArray; @@ -337,14 +337,14 @@ public class NodeStatus extends Document { return this.countryCode; } - private String aSNumber; + private String asNumber; - public void setASNumber(String aSNumber) { - this.aSNumber = aSNumber; + public void setAsNumber(String asNumber) { + this.asNumber = asNumber; } - public String getASNumber() { - return this.aSNumber; + public String getAsNumber() { + return this.asNumber; } /* Reverse DNS lookup result */ @@ -501,7 +501,7 @@ public class NodeStatus extends Document { } nodeStatus.addLastAddresses(lastChangedAddresses, address, orPort, dirPort, orAddressesAndPorts); - nodeStatus.setASNumber(parts[19]); + nodeStatus.setAsNumber(parts[19]); nodeStatus.setContact(parts[20]); if (!parts[21].equals("null")) { nodeStatus.setRecommendedVersion(parts[21].equals("true")); @@ -588,7 +588,7 @@ public class NodeStatus extends Document { sb.append("\t" + DateTimeHelper.format( this.getLastChangedOrAddressOrPort(), DateTimeHelper.ISO_DATETIME_TAB_FORMAT)); - sb.append("\t" + (this.aSNumber != null ? this.aSNumber : "null")); + sb.append("\t" + (this.asNumber != null ? this.asNumber : "null")); } else { sb.append("\tnull\tnull\tnull"); } diff --git a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java index c7d4774..133f616 100644 --- a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java @@ -16,6 +16,7 @@ import java.util.SortedSet; import java.util.TreeSet; import java.util.regex.Pattern; +@SuppressWarnings("checkstyle:membername") public class SummaryDocument extends Document { private boolean t; @@ -133,9 +134,9 @@ public class SummaryDocument extends Document { String[] stringArray = null; if (collection != null && !collection.isEmpty()) { stringArray = new String[collection.size()]; - int i = 0; + int index = 0; for (String string : collection) { - stringArray[i++] = string; + stringArray[index++] = string; } } return stringArray; @@ -171,11 +172,11 @@ public class SummaryDocument extends Document { private String as; - public void setASNumber(String aSNumber) { - this.as = aSNumber; + public void setAsNumber(String asNumber) { + this.as = asNumber; } - public String getASNumber() { + public String getAsNumber() { return this.as; } @@ -274,7 +275,7 @@ public class SummaryDocument extends Document { public SummaryDocument(boolean isRelay, String nickname, String fingerprint, List<String> addresses, long lastSeenMillis, boolean running, SortedSet<String> relayFlags, long consensusWeight, - String countryCode, long firstSeenMillis, String aSNumber, + String countryCode, long firstSeenMillis, String asNumber, String contact, SortedSet<String> familyFingerprints, SortedSet<String> effectiveFamily) { this.setRelay(isRelay); @@ -287,7 +288,7 @@ public class SummaryDocument extends Document { this.setConsensusWeight(consensusWeight); this.setCountryCode(countryCode); this.setFirstSeenMillis(firstSeenMillis); - this.setASNumber(aSNumber); + this.setAsNumber(asNumber); this.setContact(contact); this.setFamilyFingerprints(familyFingerprints); this.setEffectiveFamily(effectiveFamily); diff --git a/src/main/java/org/torproject/onionoo/docs/UptimeDocument.java b/src/main/java/org/torproject/onionoo/docs/UptimeDocument.java index 505d66c..98884b6 100644 --- a/src/main/java/org/torproject/onionoo/docs/UptimeDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/UptimeDocument.java @@ -6,6 +6,7 @@ package org.torproject.onionoo.docs; import java.util.Map; import java.util.SortedMap; +@SuppressWarnings("checkstyle:membername") public class UptimeDocument extends Document { @SuppressWarnings("unused") diff --git a/src/main/java/org/torproject/onionoo/docs/WeightsDocument.java b/src/main/java/org/torproject/onionoo/docs/WeightsDocument.java index 6e2ee02..9b1c8ef 100644 --- a/src/main/java/org/torproject/onionoo/docs/WeightsDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/WeightsDocument.java @@ -5,6 +5,7 @@ package org.torproject.onionoo.docs; import java.util.Map; +@SuppressWarnings("checkstyle:membername") public class WeightsDocument extends Document { @SuppressWarnings("unused") diff --git a/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java b/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java index efb7c25..acd44a1 100644 --- a/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java @@ -32,8 +32,10 @@ public class WeightsStatus extends Document { private SortedMap<long[], double[]> history = new TreeMap<long[], double[]>( new Comparator<long[]>() { - public int compare(long[] a, long[] b) { - return a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0; + public int compare(long[] first, long[] second) { + return first[0] < second[0] ? -1 + : first[0] > second[0] ? 1 + : 0; } } ); diff --git a/src/main/java/org/torproject/onionoo/server/Counter.java b/src/main/java/org/torproject/onionoo/server/Counter.java new file mode 100644 index 0000000..655a56c --- /dev/null +++ b/src/main/java/org/torproject/onionoo/server/Counter.java @@ -0,0 +1,22 @@ +/* Copyright 2014--2016 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.onionoo.server; + +class Counter { + + int value = 0; + + void increment() { + this.value++; + } + + @Override + public String toString() { + return String.valueOf(this.value); + } + + void clear() { + this.value = 0; + } +} diff --git a/src/main/java/org/torproject/onionoo/server/HttpServletRequestWrapper.java b/src/main/java/org/torproject/onionoo/server/HttpServletRequestWrapper.java index 1aa964a..e819a1a 100644 --- a/src/main/java/org/torproject/onionoo/server/HttpServletRequestWrapper.java +++ b/src/main/java/org/torproject/onionoo/server/HttpServletRequestWrapper.java @@ -15,6 +15,7 @@ public class HttpServletRequestWrapper { this.request = request; } + @SuppressWarnings("abbreviationaswordinname") protected String getRequestURI() { return this.request.getRequestURI(); } diff --git a/src/main/java/org/torproject/onionoo/server/IntegerDistribution.java b/src/main/java/org/torproject/onionoo/server/IntegerDistribution.java new file mode 100644 index 0000000..c9e3a29 --- /dev/null +++ b/src/main/java/org/torproject/onionoo/server/IntegerDistribution.java @@ -0,0 +1,50 @@ +/* Copyright 2014--2016 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.onionoo.server; + +import java.util.Arrays; + +class IntegerDistribution { + + int[] logValues = new int[64]; + + void addLong(long value) { + logValues[64 - Long.numberOfLeadingZeros(value)]++; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + int totalValues = 0; + for (int i = 0; i < logValues.length; i++) { + totalValues += logValues[i]; + } + int[] permilles = new int[] { 500, 900, 990, 999 }; + if (totalValues > 0) { + int seenValues = 0; + for (int i = 0, j = 0; i < logValues.length; i++) { + seenValues += logValues[i]; + while (j < permilles.length + && (seenValues * 1000 > totalValues * permilles[j])) { + sb.append((j > 0 ? ", " : "") + "." + permilles[j] + + (i < logValues.length - 1 ? "<" + (1L << i) + : ">=" + (1L < 0 ? ", " : "") + "." + permilles[j] + "<null"); + } + } + return sb.toString(); + } + + void clear() { + Arrays.fill(logValues, 0, logValues.length - 1, 0); + } +} diff --git a/src/main/java/org/torproject/onionoo/server/MostFrequentString.java b/src/main/java/org/torproject/onionoo/server/MostFrequentString.java new file mode 100644 index 0000000..12b5e04 --- /dev/null +++ b/src/main/java/org/torproject/onionoo/server/MostFrequentString.java @@ -0,0 +1,65 @@ +/* Copyright 2014--2016 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.onionoo.server; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; + +class MostFrequentString { + + Map<String, Integer> stringFrequencies = new HashMap<String, Integer>(); + + void addString(String string) { + if (!this.stringFrequencies.containsKey(string)) { + this.stringFrequencies.put(string, 1); + } else { + this.stringFrequencies.put(string, + this.stringFrequencies.get(string) + 1); + } + } + + @Override + public String toString() { + SortedMap<Integer, SortedSet<String>> sortedFrequencies = + new TreeMap<Integer, SortedSet<String>>( + Collections.reverseOrder()); + if (this.stringFrequencies.isEmpty()) { + return "null (0)"; + } + for (Map.Entry<String, Integer> e : stringFrequencies.entrySet()) { + if (!sortedFrequencies.containsKey(e.getValue())) { + sortedFrequencies.put(e.getValue(), new TreeSet<String>( + Arrays.asList(new String[] { e.getKey() } ))); + } else { + sortedFrequencies.get(e.getValue()).add(e.getKey()); + } + } + StringBuilder sb = new StringBuilder(); + int stringsToAdd = 3; + int written = 0; + for (Map.Entry<Integer, SortedSet<String>> e : + sortedFrequencies.entrySet()) { + for (String string : e.getValue()) { + if (stringsToAdd-- > 0) { + sb.append((written++ > 0 ? ", " : "") + string + " (" + + e.getKey() + ")"); + } + } + if (stringsToAdd == 0) { + break; + } + } + return sb.toString(); + } + + void clear() { + this.stringFrequencies.clear(); + } +} diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndex.java b/src/main/java/org/torproject/onionoo/server/NodeIndex.java index c000bba..7871690 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndex.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndex.java @@ -86,15 +86,15 @@ class NodeIndex { return relaysByCountryCode; } - private Map<String, Set<String>> relaysByASNumber = null; + private Map<String, Set<String>> relaysByAsNumber = null; - public void setRelaysByASNumber( - Map<String, Set<String>> relaysByASNumber) { - this.relaysByASNumber = relaysByASNumber; + public void setRelaysByAsNumber( + Map<String, Set<String>> relaysByAsNumber) { + this.relaysByAsNumber = relaysByAsNumber; } - public Map<String, Set<String>> getRelaysByASNumber() { - return relaysByASNumber; + public Map<String, Set<String>> getRelaysByAsNumber() { + return relaysByAsNumber; } private Map<String, Set<String>> relaysByFlag = null; diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java index 93b5af7..99d1aee 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java @@ -71,6 +71,8 @@ public class NodeIndexer implements ServletContextListener, Runnable { try { this.wait(timeoutMillis); } catch (InterruptedException e) { + /* Nothing that we could handle, just return what we have + * below. */ } } return this.lastIndexed; @@ -84,6 +86,8 @@ public class NodeIndexer implements ServletContextListener, Runnable { try { this.wait(timeoutMillis); } catch (InterruptedException e) { + /* Nothing that we could handle, just return what we have + * below. */ } } return this.latestNodeIndex; @@ -110,6 +114,8 @@ public class NodeIndexer implements ServletContextListener, Runnable { try { Thread.sleep(ONE_MINUTE); } catch (InterruptedException e) { + /* Nothing that we could handle, just check if there's new data + * to index now. */ } } } @@ -137,14 +143,13 @@ public class NodeIndexer implements ServletContextListener, Runnable { } } documentStore.invalidateDocumentCache(); - List<String> newRelaysByConsensusWeight = new ArrayList<String>(); Map<String, SummaryDocument> newRelayFingerprintSummaryLines = new HashMap<String, SummaryDocument>(); Map<String, SummaryDocument> newBridgeFingerprintSummaryLines = new HashMap<String, SummaryDocument>(); Map<String, Set<String>> newRelaysByCountryCode = new HashMap<String, Set<String>>(); - Map<String, Set<String>> newRelaysByASNumber = + Map<String, Set<String>> newRelaysByAsNumber = new HashMap<String, Set<String>>(); Map<String, Set<String>> newRelaysByFlag = new HashMap<String, Set<String>>(); @@ -208,13 +213,13 @@ public class NodeIndexer implements ServletContextListener, Runnable { newRelaysByCountryCode.get(countryCode).add(fingerprint); newRelaysByCountryCode.get(countryCode).add(hashedFingerprint); } - if (entry.getASNumber() != null) { - String aSNumber = entry.getASNumber(); - if (!newRelaysByASNumber.containsKey(aSNumber)) { - newRelaysByASNumber.put(aSNumber, new HashSet<String>()); + if (entry.getAsNumber() != null) { + String asNumber = entry.getAsNumber(); + if (!newRelaysByAsNumber.containsKey(asNumber)) { + newRelaysByAsNumber.put(asNumber, new HashSet<String>()); } - newRelaysByASNumber.get(aSNumber).add(fingerprint); - newRelaysByASNumber.get(aSNumber).add(hashedFingerprint); + newRelaysByAsNumber.get(asNumber).add(fingerprint); + newRelaysByAsNumber.get(asNumber).add(hashedFingerprint); } for (String flag : entry.getRelayFlags()) { String flagLowerCase = flag.toLowerCase(); @@ -261,7 +266,7 @@ public class NodeIndexer implements ServletContextListener, Runnable { newRelaysByContact.get(contact).add(hashedFingerprint); } Collections.sort(orderRelaysByConsensusWeight); - newRelaysByConsensusWeight = new ArrayList<String>(); + List<String> newRelaysByConsensusWeight = new ArrayList<String>(); for (String relay : orderRelaysByConsensusWeight) { newRelaysByConsensusWeight.add(relay.split(" ")[1]); } @@ -325,7 +330,7 @@ public class NodeIndexer implements ServletContextListener, Runnable { newNodeIndex.setBridgeFingerprintSummaryLines( newBridgeFingerprintSummaryLines); newNodeIndex.setRelaysByCountryCode(newRelaysByCountryCode); - newNodeIndex.setRelaysByASNumber(newRelaysByASNumber); + newNodeIndex.setRelaysByAsNumber(newRelaysByAsNumber); newNodeIndex.setRelaysByFlag(newRelaysByFlag); newNodeIndex.setBridgesByFlag(newBridgesByFlag); newNodeIndex.setRelaysByContact(newRelaysByContact); diff --git a/src/main/java/org/torproject/onionoo/server/PerformanceMetrics.java b/src/main/java/org/torproject/onionoo/server/PerformanceMetrics.java index 7adad76..7a621cf 100644 --- a/src/main/java/org/torproject/onionoo/server/PerformanceMetrics.java +++ b/src/main/java/org/torproject/onionoo/server/PerformanceMetrics.java @@ -11,130 +11,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.text.SimpleDateFormat; -import java.util.Arrays; import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.SortedMap; -import java.util.SortedSet; import java.util.TimeZone; -import java.util.TreeMap; -import java.util.TreeSet; - -class Counter { - - int value = 0; - - void increment() { - this.value++; - } - - @Override - public String toString() { - return String.valueOf(this.value); - } - - void clear() { - this.value = 0; - } -} - -class MostFrequentString { - - Map<String, Integer> stringFrequencies = new HashMap<String, Integer>(); - - void addString(String string) { - if (!this.stringFrequencies.containsKey(string)) { - this.stringFrequencies.put(string, 1); - } else { - this.stringFrequencies.put(string, - this.stringFrequencies.get(string) + 1); - } - } - - @Override - public String toString() { - SortedMap<Integer, SortedSet<String>> sortedFrequencies = - new TreeMap<Integer, SortedSet<String>>( - Collections.reverseOrder()); - if (this.stringFrequencies.isEmpty()) { - return "null (0)"; - } - for (Map.Entry<String, Integer> e : stringFrequencies.entrySet()) { - if (!sortedFrequencies.containsKey(e.getValue())) { - sortedFrequencies.put(e.getValue(), new TreeSet<String>( - Arrays.asList(new String[] { e.getKey() } ))); - } else { - sortedFrequencies.get(e.getValue()).add(e.getKey()); - } - } - StringBuilder sb = new StringBuilder(); - int stringsToAdd = 3; - int written = 0; - for (Map.Entry<Integer, SortedSet<String>> e : - sortedFrequencies.entrySet()) { - for (String string : e.getValue()) { - if (stringsToAdd-- > 0) { - sb.append((written++ > 0 ? ", " : "") + string + " (" - + e.getKey() + ")"); - } - } - if (stringsToAdd == 0) { - break; - } - } - return sb.toString(); - } - - void clear() { - this.stringFrequencies.clear(); - } -} - -class IntegerDistribution { - - int[] logValues = new int[64]; - - void addLong(long value) { - logValues[64 - Long.numberOfLeadingZeros(value)]++; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - int totalValues = 0; - for (int i = 0; i < logValues.length; i++) { - totalValues += logValues[i]; - } - int[] permilles = new int[] { 500, 900, 990, 999 }; - if (totalValues > 0) { - int seenValues = 0; - for (int i = 0, j = 0; i < logValues.length; i++) { - seenValues += logValues[i]; - while (j < permilles.length - && (seenValues * 1000 > totalValues * permilles[j])) { - sb.append((j > 0 ? ", " : "") + "." + permilles[j] - + (i < logValues.length - 1 ? "<" + (1L << i) - : ">=" + (1L < 0 ? ", " : "") + "." + permilles[j] + "<null"); - } - } - return sb.toString(); - } - - void clear() { - Arrays.fill(logValues, 0, logValues.length - 1, 0); - } -} public class PerformanceMetrics { diff --git a/src/main/java/org/torproject/onionoo/server/RequestHandler.java b/src/main/java/org/torproject/onionoo/server/RequestHandler.java index eaa4fe2..49b3b94 100644 --- a/src/main/java/org/torproject/onionoo/server/RequestHandler.java +++ b/src/main/java/org/torproject/onionoo/server/RequestHandler.java @@ -153,7 +153,7 @@ public class RequestHandler { this.filterByLookup(); this.filterByFingerprint(); this.filterByCountryCode(); - this.filterByASNumber(); + this.filterByAsNumber(); this.filterByFlag(); this.filterNodesByFirstSeenDays(); this.filterNodesByLastSeenDays(); @@ -369,23 +369,23 @@ public class RequestHandler { this.filteredBridges.clear(); } - private void filterByASNumber() { + private void filterByAsNumber() { if (this.as == null) { /* Not filtering by AS number. */ return; } - String aSNumber = this.as.toUpperCase(); - if (!aSNumber.startsWith("AS")) { - aSNumber = "AS" + aSNumber; + String asNumber = this.as.toUpperCase(); + if (!asNumber.startsWith("AS")) { + asNumber = "AS" + asNumber; } - if (!this.nodeIndex.getRelaysByASNumber().containsKey(aSNumber)) { + if (!this.nodeIndex.getRelaysByAsNumber().containsKey(asNumber)) { this.filteredRelays.clear(); } else { - Set<String> relaysWithASNumber = - this.nodeIndex.getRelaysByASNumber().get(aSNumber); + Set<String> relaysWithAsNumber = + this.nodeIndex.getRelaysByAsNumber().get(asNumber); Set<String> removeRelays = new HashSet<String>(); for (String fingerprint : this.filteredRelays.keySet()) { - if (!relaysWithASNumber.contains(fingerprint)) { + if (!relaysWithAsNumber.contains(fingerprint)) { removeRelays.add(fingerprint); } } diff --git a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java index 5b3ab69..92d0d81 100644 --- a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java +++ b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java @@ -78,6 +78,7 @@ public class ResourceServlet extends HttpServlet { /** Handles the HTTP GET request in the wrapped <code>request</code> by * writing an HTTP GET response to the likewise <code>response</code>, * both of which are wrapped to facilitate testing. */ + @SuppressWarnings("checkstyle:variabledeclarationusagedistance") public void doGet(HttpServletRequestWrapper request, HttpServletResponseWrapper response) throws IOException { @@ -86,15 +87,6 @@ public class ResourceServlet extends HttpServlet { return; } - long nowMillis = TimeFactory.getTime().currentTimeMillis(); - long indexWrittenMillis = - NodeIndexerFactory.getNodeIndexer().getLastIndexed( - INDEX_WAITING_TIME); - long indexAgeMillis = nowMillis - indexWrittenMillis; - long cacheMaxAgeMillis = Math.max(CACHE_MIN_TIME, - ((CACHE_MAX_TIME - indexAgeMillis) - / CACHE_INTERVAL) * CACHE_INTERVAL); - NodeIndex nodeIndex = NodeIndexerFactory.getNodeIndexer() .getLatestNodeIndex(INDEX_WAITING_TIME); if (nodeIndex == null) { @@ -230,13 +222,13 @@ public class ResourceServlet extends HttpServlet { rh.setCountry(countryCodeParameter); } if (parameterMap.containsKey("as")) { - String aSNumberParameter = this.parseASNumberParameter( + String asNumberParameter = this.parseAsNumberParameter( parameterMap.get("as")); - if (aSNumberParameter == null) { + if (asNumberParameter == null) { response.sendError(HttpServletResponse.SC_BAD_REQUEST); return; } - rh.setAs(aSNumberParameter); + rh.setAs(asNumberParameter); } if (parameterMap.containsKey("flag")) { String flagParameter = this.parseFlagParameter( @@ -343,6 +335,14 @@ public class ResourceServlet extends HttpServlet { rb.setFields(fields); } + long indexWrittenMillis = + NodeIndexerFactory.getNodeIndexer().getLastIndexed( + INDEX_WAITING_TIME); + long indexAgeMillis = receivedRequestMillis - indexWrittenMillis; + long cacheMaxAgeMillis = Math.max(CACHE_MIN_TIME, + ((CACHE_MAX_TIME - indexAgeMillis) + / CACHE_INTERVAL) * CACHE_INTERVAL); + response.setHeader("Access-Control-Allow-Origin", "*"); response.setContentType("application/json"); response.setCharacterEncoding("utf-8"); @@ -418,11 +418,11 @@ public class ResourceServlet extends HttpServlet { return parameter; } - private static Pattern aSNumberParameterPattern = + private static Pattern asNumberParameterPattern = Pattern.compile("^[asAS]{0,2}[0-9]{1,10}$"); - private String parseASNumberParameter(String parameter) { - if (!aSNumberParameterPattern.matcher(parameter).matches()) { + private String parseAsNumberParameter(String parameter) { + if (!asNumberParameterPattern.matcher(parameter).matches()) { /* AS number contains illegal character(s). */ return null; } @@ -447,30 +447,30 @@ public class ResourceServlet extends HttpServlet { /* Days contain illegal character(s). */ return null; } - int x = 0; - int y = Integer.MAX_VALUE; + int fromDays = 0; + int toDays = Integer.MAX_VALUE; try { if (!parameter.contains("-")) { - x = Integer.parseInt(parameter); - y = x; + fromDays = Integer.parseInt(parameter); + toDays = fromDays; } else { String[] parts = parameter.split("-", 2); if (parts[0].length() > 0) { - x = Integer.parseInt(parts[0]); + fromDays = Integer.parseInt(parts[0]); } if (parts.length > 1 && parts[1].length() > 0) { - y = Integer.parseInt(parts[1]); + toDays = Integer.parseInt(parts[1]); } } } catch (NumberFormatException e) { /* Invalid format. */ return null; } - if (x > y) { + if (fromDays > toDays) { /* Second number or days must exceed first number. */ return null; } - return new int[] { x, y }; + return new int[] { fromDays, toDays }; } private String[] parseContactParameter(String parameter) { diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorDownloader.java b/src/main/java/org/torproject/onionoo/updater/DescriptorDownloader.java index 71a43d2..5822b77 100644 --- a/src/main/java/org/torproject/onionoo/updater/DescriptorDownloader.java +++ b/src/main/java/org/torproject/onionoo/updater/DescriptorDownloader.java @@ -79,8 +79,8 @@ class DescriptorDownloader { String directoryUrl = this.protocolHostNameResourcePrefix + this.directory; try { - URL u = new URL(directoryUrl); - HttpURLConnection huc = (HttpURLConnection) u.openConnection(); + URL url = new URL(directoryUrl); + HttpURLConnection huc = (HttpURLConnection) url.openConnection(); huc.setRequestMethod("GET"); huc.connect(); if (huc.getResponseCode() != 200) { @@ -129,8 +129,8 @@ class DescriptorDownloader { File localFile = new File(this.inDir, this.directory + remoteFile); try { localFile.getParentFile().mkdirs(); - URL u = new URL(fileUrl); - HttpURLConnection huc = (HttpURLConnection) u.openConnection(); + URL url = new URL(fileUrl); + HttpURLConnection huc = (HttpURLConnection) url.openConnection(); huc.setRequestMethod("GET"); huc.addRequestProperty("Accept-Encoding", "gzip"); huc.connect(); @@ -140,7 +140,6 @@ class DescriptorDownloader { + huc.getResponseMessage() + ". Skipping."); continue; } - long lastModified = huc.getHeaderFieldDate("Last-Modified", -1L); InputStream is; if (huc.getContentEncoding() != null && huc.getContentEncoding().equalsIgnoreCase("gzip")) { @@ -158,6 +157,7 @@ class DescriptorDownloader { } } localTempFile.renameTo(localFile); + long lastModified = huc.getHeaderFieldDate("Last-Modified", -1L); if (lastModified >= 0) { localFile.setLastModified(lastModified); } diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java index 176a17c..c251142 100644 --- a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java +++ b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java @@ -156,6 +156,12 @@ public class DescriptorSource { case BRIDGE_EXTRA_INFOS: log.info("Read recent bridge extra-info descriptors"); break; + default: + /* We shouldn't run into this default case, but if we do, it's + * because we added a new type to DescriptorType but forgot to + * update this switch statement. It's just logging, so not the + * end of the world. */ + log.info("Read recent descriptors of type " + descriptorType); } } diff --git a/src/main/java/org/torproject/onionoo/updater/LookupService.java b/src/main/java/org/torproject/onionoo/updater/LookupService.java index 2b0993f..ec63751 100644 --- a/src/main/java/org/torproject/onionoo/updater/LookupService.java +++ b/src/main/java/org/torproject/onionoo/updater/LookupService.java @@ -39,7 +39,7 @@ public class LookupService { private File geoLite2CityLocationsEnCsvFile; - private File geoIPASNum2CsvFile; + private File geoIpAsNum2CsvFile; private boolean hasAllFiles = false; @@ -63,8 +63,8 @@ public class LookupService { + "geoip/."); return; } - this.geoIPASNum2CsvFile = new File(this.geoipDir, "GeoIPASNum2.csv"); - if (!this.geoIPASNum2CsvFile.exists()) { + this.geoIpAsNum2CsvFile = new File(this.geoipDir, "GeoIPASNum2.csv"); + if (!this.geoIpAsNum2CsvFile.exists()) { log.error("No GeoIPASNum2.csv file in geoip/."); return; } @@ -85,6 +85,7 @@ public class LookupService { try { octetValue = Integer.parseInt(parts[i]); } catch (NumberFormatException e) { + /* Handled below, because octetValue will still be -1. */ } if (octetValue < 0 || octetValue > 255) { addressNumber = -1L; @@ -228,9 +229,9 @@ public class LookupService { } /* Obtain a map from IP address numbers to ASN. */ - Map<Long, String> addressNumberASN = new HashMap<Long, String>(); + Map<Long, String> addressNumberAsn = new HashMap<Long, String>(); try (BufferedReader br = this.createBufferedReaderFromIso88591File( - this.geoIPASNum2CsvFile)) { + this.geoIpAsNum2CsvFile)) { SortedSet<Long> sortedAddressNumbers = new TreeSet<Long>( addressStringNumbers.values()); long firstAddressNumber = sortedAddressNumbers.first(); @@ -240,14 +241,14 @@ public class LookupService { String[] parts = line.replaceAll("\"", "").split(",", 3); if (parts.length != 3) { log.error("Illegal line '" + line + "' in " - + geoIPASNum2CsvFile.getAbsolutePath() + "."); + + geoIpAsNum2CsvFile.getAbsolutePath() + "."); return lookupResults; } try { long startIpNum = Long.parseLong(parts[0]); if (startIpNum <= previousStartIpNum) { log.error("Line '" + line + "' not sorted in " - + geoIPASNum2CsvFile.getAbsolutePath() + "."); + + geoIpAsNum2CsvFile.getAbsolutePath() + "."); return lookupResults; } previousStartIpNum = startIpNum; @@ -264,7 +265,7 @@ public class LookupService { while (firstAddressNumber <= endIpNum && firstAddressNumber != -1L) { if (parts[2].startsWith("AS")) { - addressNumberASN.put(firstAddressNumber, parts[2]); + addressNumberAsn.put(firstAddressNumber, parts[2]); } sortedAddressNumbers.remove(firstAddressNumber); if (sortedAddressNumbers.isEmpty()) { @@ -279,13 +280,13 @@ public class LookupService { } catch (NumberFormatException e) { log.error("Number format exception while parsing line " + "'" + line + "' in " - + geoIPASNum2CsvFile.getAbsolutePath() + "."); + + geoIpAsNum2CsvFile.getAbsolutePath() + "."); return lookupResults; } } } catch (IOException e) { log.error("I/O exception while reading " - + geoIPASNum2CsvFile.getAbsolutePath() + ": " + e); + + geoIpAsNum2CsvFile.getAbsolutePath() + ": " + e); return lookupResults; } @@ -297,7 +298,7 @@ public class LookupService { long addressNumber = addressStringNumbers.get(addressString); if (!addressNumberBlocks.containsKey(addressNumber) && !addressNumberLatLong.containsKey(addressNumber) - && !addressNumberASN.containsKey(addressNumber)) { + && !addressNumberAsn.containsKey(addressNumber)) { continue; } LookupResult lookupResult = new LookupResult(); @@ -325,8 +326,8 @@ public class LookupService { lookupResult.setLatitude(latLong[0]); lookupResult.setLongitude(latLong[1]); } - if (addressNumberASN.containsKey(addressNumber)) { - String[] parts = addressNumberASN.get(addressNumber).split(" ", + if (addressNumberAsn.containsKey(addressNumber)) { + String[] parts = addressNumberAsn.get(addressNumber).split(" ", 2); lookupResult.setAsNumber(parts[0]); lookupResult.setAsName(parts.length == 2 ? parts[1] : ""); diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java index d873072..a66aba8 100644 --- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java @@ -492,7 +492,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, updatedNodeStatus.setDefaultPolicy( nodeStatus.getDefaultPolicy()); updatedNodeStatus.setPortList(nodeStatus.getPortList()); - updatedNodeStatus.setASNumber(nodeStatus.getASNumber()); + updatedNodeStatus.setAsNumber(nodeStatus.getAsNumber()); updatedNodeStatus.setRecommendedVersion( nodeStatus.getRecommendedVersion()); } @@ -862,10 +862,10 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, detailsStatus.setCityName(lookupResult.getCityName()); detailsStatus.setLatitude(lookupResult.getLatitude()); detailsStatus.setLongitude(lookupResult.getLongitude()); - detailsStatus.setASNumber(lookupResult.getAsNumber()); - detailsStatus.setASName(lookupResult.getAsName()); + detailsStatus.setAsNumber(lookupResult.getAsNumber()); + detailsStatus.setAsName(lookupResult.getAsName()); nodeStatus.setCountryCode(lookupResult.getCountryCode()); - nodeStatus.setASNumber(lookupResult.getAsNumber()); + nodeStatus.setAsNumber(lookupResult.getAsNumber()); } if (this.consensusWeightFractions.containsKey(fingerprint)) { diff --git a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java index 7238c1b..33b943e 100644 --- a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java @@ -107,12 +107,11 @@ public class BandwidthDocumentWriter implements DocumentWriter { long totalMillis = 0L; long totalBandwidth = 0L; for (long[] v : history.values()) { - long startMillis = v[0]; long endMillis = v[1]; - long bandwidth = v[2]; if (endMillis < intervalStartMillis) { continue; } + long startMillis = v[0]; if (endMillis - startMillis > dataPointInterval) { /* This history interval is too long for this graph's data point * interval. Maybe the next graph will contain it, but not this @@ -128,6 +127,7 @@ public class BandwidthDocumentWriter implements DocumentWriter { totalMillis = 0L; intervalStartMillis += dataPointInterval; } + long bandwidth = v[2]; totalBandwidth += bandwidth; totalMillis += (endMillis - startMillis); } diff --git a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java index c167152..1c515e8 100644 --- a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java @@ -94,8 +94,8 @@ public class DetailsDocumentWriter implements DocumentWriter { detailsDocument.setCountryName(detailsStatus.getCountryName()); detailsDocument.setRegionName(detailsStatus.getRegionName()); detailsDocument.setCityName(detailsStatus.getCityName()); - detailsDocument.setAsNumber(detailsStatus.getASNumber()); - detailsDocument.setAsName(detailsStatus.getASName()); + detailsDocument.setAsNumber(detailsStatus.getAsNumber()); + detailsDocument.setAsName(detailsStatus.getAsName()); if (detailsStatus.isRunning()) { detailsDocument.setConsensusWeightFraction( detailsStatus.getConsensusWeightFraction()); diff --git a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java index f941ee3..e9bb8f6 100644 --- a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java @@ -65,8 +65,6 @@ public class SummaryDocumentWriter implements DocumentWriter { } continue; } - boolean isRelay = nodeStatus.isRelay(); - String nickname = nodeStatus.getNickname(); List<String> addresses = new ArrayList<String>(); addresses.add(nodeStatus.getAddress()); for (String orAddress : nodeStatus.getOrAddresses()) { @@ -81,20 +79,22 @@ public class SummaryDocumentWriter implements DocumentWriter { } long lastSeenMillis = nodeStatus.getLastSeenMillis(); SortedSet<String> relayFlags = nodeStatus.getRelayFlags(); + boolean isRelay = nodeStatus.isRelay(); boolean running = relayFlags.contains("Running") && (isRelay ? lastSeenMillis == relaysLastValidAfterMillis : lastSeenMillis == bridgesLastPublishedMillis); long consensusWeight = nodeStatus.getConsensusWeight(); String countryCode = nodeStatus.getCountryCode(); long firstSeenMillis = nodeStatus.getFirstSeenMillis(); - String aSNumber = nodeStatus.getASNumber(); + String asNumber = nodeStatus.getAsNumber(); String contact = nodeStatus.getContact(); SortedSet<String> declaredFamily = nodeStatus.getDeclaredFamily(); SortedSet<String> effectiveFamily = nodeStatus.getEffectiveFamily(); + String nickname = nodeStatus.getNickname(); SummaryDocument summaryDocument = new SummaryDocument(isRelay, nickname, fingerprint, addresses, lastSeenMillis, running, relayFlags, consensusWeight, countryCode, firstSeenMillis, - aSNumber, contact, declaredFamily, effectiveFamily); + asNumber, contact, declaredFamily, effectiveFamily); if (this.documentStore.store(summaryDocument, fingerprint)) { this.writtenDocuments++; }

1 0

[onionoo/master] Reset sorted fingerprint blocks when setting fingerprint.
by karsten＠torproject.org 25 Jul '16

25 Jul '16

commit 54ef1a7e34659ab7941b7c2d769fd7a8d851d1a6 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jul 19 10:22:15 2016 +0200 Reset sorted fingerprint blocks when setting fingerprint. Previously, we'd only have calculated sorted fingerprint blocks once, but not after setting a new fingerprint. We never ran into this bug, because we never called that setter outside of the constructor. But future uses of that setter would have caused trouble. Found while writing Javadoc comments for #19613. --- .../torproject/onionoo/docs/SummaryDocument.java | 1 + .../onionoo/docs/SummaryDocumentTest.java | 26 ++++++++++++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java index 6a520c8..c7d4774 100644 --- a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java @@ -44,6 +44,7 @@ public class SummaryDocument extends Document { this.f = fingerprint; this.hashedFingerprint = null; this.base64Fingerprint = null; + this.fingerprintSortedHexBlocks = null; } public String getFingerprint() { diff --git a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java index 3c097ae..99ae0c5 100644 --- a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java +++ b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java @@ -1,7 +1,8 @@ -/* Copyright 2015 The Tor Project +/* Copyright 2015--2016 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertTrue; import org.junit.Test; @@ -11,9 +12,8 @@ import java.util.TreeSet; public class SummaryDocumentTest { - @Test() - public void testFingerprintSortedHexBlocksAreSorted() { - SummaryDocument relayTorkaZ = new SummaryDocument(true, "TorkaZ", + private SummaryDocument createSummaryDocumentRelayTorkaZ() { + return new SummaryDocument(true, "TorkaZ", "000C5F55BD4814B917CC474BD537F1A3B33CCE2A", Arrays.asList( new String[] { "62.216.201.221", "62.216.201.222", "62.216.201.223" }), DateTimeHelper.parse("2013-04-19 05:00:00"), @@ -26,6 +26,11 @@ public class SummaryDocumentTest { "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<String>(Arrays.asList( new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" }))); + } + + @Test() + public void testFingerprintSortedHexBlocksAreSorted() { + SummaryDocument relayTorkaZ = this.createSummaryDocumentRelayTorkaZ(); String[] fingerprintSortedHexBlocks = relayTorkaZ.getFingerprintSortedHexBlocks(); for (int i = 0; i < fingerprintSortedHexBlocks.length - 1; i++) { @@ -34,5 +39,18 @@ public class SummaryDocumentTest { fingerprintSortedHexBlocks[i + 1]) <= 0); } } + + @Test() + public void testFingerprintSortedHexBlocksReset() { + SummaryDocument relayTorkaZ = this.createSummaryDocumentRelayTorkaZ(); + assertArrayEquals("Hex blocks differ", new String[] { "000C", "14B9", + "17CC", "474B", "5F55", "B33C", "BD48", "CE2A", "D537", "F1A3" }, + relayTorkaZ.getFingerprintSortedHexBlocks()); + relayTorkaZ.setFingerprint( + "A2ECC33B3A1F735D8474CC719B4184DB55F5C000"); + assertArrayEquals("Hex blocks differ", new String[] { "3A1F", "55F5", + "735D", "8474", "84DB", "9B41", "A2EC", "C000", "C33B", "CC71" }, + relayTorkaZ.getFingerprintSortedHexBlocks()); + } }

1 0

[onionoo/master] Implements #19613: changed filestructure, added coverage task,
by karsten＠torproject.org 25 Jul '16

25 Jul '16

commit 254824395abea80787ef06af32ae72d0f288c2d7 Author: iwakeh <iwakeh(a)torproject.org> Date: Thu Jul 21 15:13:16 2016 +0200 Implements #19613: changed filestructure, added coverage task, updated some dependencies to current debian stable. --- build.xml | 160 ++- etc/context.xml.template | 3 - etc/jetty.xml | 45 - etc/logback.xml | 82 -- etc/web.xml.template | 78 -- src/main/resources/context.xml | 3 + src/main/resources/jetty.xml | 45 + src/main/resources/logback.xml | 82 ++ src/main/resources/web.xml | 78 ++ src/main/resources/web/css/style.css | 36 + src/main/resources/web/favicon.ico | Bin 0 -> 1150 bytes src/main/resources/web/index.html | 141 ++ src/main/resources/web/protocol.html | 2338 ++++++++++++++++++++++++++++++++++ src/main/resources/web/robots.txt | 8 + web/css/style.css | 36 - web/favicon.ico | Bin 1150 -> 0 bytes web/index.html | 141 -- web/protocol.html | 2338 ---------------------------------- web/robots.txt | 8 - 19 files changed, 2840 insertions(+), 2782 deletions(-) diff --git a/build.xml b/build.xml index 467c991..7cfd870 100644 --- a/build.xml +++ b/build.xml @@ -5,21 +5,19 @@ value="${onionoo.protocol.version}.0"/> <property name="descriptorversion" value="1.3.0"/> <property name="javasources" value="src/main/java"/> - <property name="tests" value="src/test/java"/> - <property name="generated" value="generated/"/> - <property name="classes" value="classes"/> + <property name="testsources" value="src/test/java"/> + <property name="generated" value="generated"/> + <property name="classes" value="${generated}/classes"/> + <property name="testclasses" value="${generated}/testclasses/"/> + <property name="resources" value="src/main/resources/"/> <property name="testresources" value="src/test/resources/"/> - <property name="dist" value="dist"/> + <property name="testresult" value="${generated}/test-results"/> + <property name="coverageresult" value="${generated}/coverage-report/"/> + <property name="instrument" value="${generated}/instrument/"/> + <property name="cobertura.ser.file" value="${basedir}/cobertura.ser" /> + <property name="dist" value="${generated}/dist"/> <property name="docs" value="${generated}/javadoc/"/> - <property name="libs" value="lib"/> - <property name="config" value="etc"/> - <property name="webxmlfile" value="${config}/web.xml"/> - <property name="contextxmltemplate" - value="${config}/context.xml.template"/> - <property name="contextxml" value="${config}/context.xml"/> - <property name="webxmltemplate" - value="${config}/web.xml.template"/> - <property name="webxml" value="${config}/web.xml"/> + <property name="libs" value="${basedir}/lib"/> <property name="warfile" value="${dist}/onionoo-${release.version}.war"/> <property name="onionoo.jarfile" @@ -28,13 +26,13 @@ <patternset id="runtime" > <include name="descriptor-${descriptorversion}.jar"/> - <include name="commons-codec-1.6.jar"/> - <include name="commons-compress-1.4.1.jar"/> - <include name="commons-lang3-3.1.jar"/> - <include name="gson-2.1.jar"/> - <include name="logback-classic-1.0.4.jar"/> - <include name="logback-core-1.0.4.jar"/> - <include name="slf4j-api-1.6.5.jar"/> + <include name="commons-codec-1.9.jar"/> + <include name="commons-compress-1.9.jar"/> + <include name="commons-lang3-3.3.2.jar"/> + <include name="gson-2.2.4.jar"/> + <include name="logback-classic-1.1.2.jar"/> + <include name="logback-core-1.1.2.jar"/> + <include name="slf4j-api-1.7.7.jar"/> </patternset> <patternset id="web" > @@ -52,7 +50,7 @@ </patternset> <patternset id="test" > - <include name="junit4-4.10.jar"/> + <include name="junit4-4.11.jar"/> <include name="hamcrest-core-1.3.jar"/> </patternset> @@ -62,13 +60,14 @@ <patternset refid="runtime" /> <patternset refid="web" /> </fileset> - <fileset dir="${config}"> + <fileset dir="${resources}"> <include name="logback.xml,jetty.xml"/> </fileset> </path> - <path id="testclasspath"> + <path id="test.classpath"> <path refid="classpath" /> + <pathelement path="${testclasses}"/> <fileset dir="${libs}"> <patternset refid="runtime" /> <patternset refid="test" /> @@ -82,20 +81,42 @@ </fileset> </path> + <path id="cobertura.classpath"> + <fileset dir="${libs}"> + <include name="descriptor-${descriptorversion}.jar"/> + <include name="cobertura-2.1.1.jar" /> + <include name="slf4j-api-1.7.7.jar" /> + <include name="commons-lang3-3.3.2.jar" /> + <include name="asm4-5.0.3.jar" /> + <include name="asm4-util-5.0.3.jar" /> + <include name="asm4-tree-5.0.3.jar" /> + <include name="asm4-commons-5.0.3.jar" /> + <include name="asm4-analysis-5.0.3.jar" /> + <include name="oro-2.0.8.jar" /> + <include name="logback-core-1.1.2.jar" /> + <include name="logback-classic-1.1.2.jar" /> + </fileset> + </path> + + <path id="cobertura.test.classpath"> + <path location="${instrument}" /> + <path refid="test.classpath" /> + <path refid="cobertura.classpath" /> + </path> + <target name="init"> - <copy file="${contextxmltemplate}" tofile="${contextxml}"/> - <copy file="${webxmltemplate}" tofile="${webxml}"/> <mkdir dir="${classes}"/> + <mkdir dir="${testclasses}"/> + <mkdir dir="${testresult}"/> <mkdir dir="${docs}"/> <mkdir dir="${dist}"/> - <mkdir dir="${generated}"/> </target> <target name="clean" > - <delete includeEmptyDirs="true"> - <fileset dir="${classes}" defaultexcludes="false" includes="**" /> - <fileset dir="${dist}" defaultexcludes="false" includes="**" /> + <delete includeEmptyDirs="true" quiet="true" > + <fileset dir="${generated}" defaultexcludes="false" includes="**" /> </delete> + <delete file="${cobertura.ser.file}" quiet="true"/> </target> <target name="compile" @@ -113,6 +134,20 @@ </javac> </target> + <target name="compile-tests" depends="compile"> + <javac destdir="${testclasses}" + srcdir="${testsources}" + source="${source-and-target-java-version}" + target="${source-and-target-java-version}" + debug="true" debuglevel="lines,source" + deprecation="true" + optimize="false" + failonerror="true" + includeantruntime="false"> + <classpath refid="test.classpath"/> + </javac> + </target> + <target name="docs" depends="init"> <javadoc destdir="${docs}" footer="&copy; 2016 The Tor Project" @@ -124,23 +159,12 @@ </javadoc> </target> - <target name="test" depends="compile"> - <javac destdir="${classes}" - srcdir="${tests}" - source="${source-and-target-java-version}" - target="${source-and-target-java-version}" - debug="true" - deprecation="true" - optimize="false" - failonerror="true" - includeantruntime="false"> - <classpath refid="testclasspath" /> - </javac> - <junit fork="true" haltonfailure="true" printsummary="off"> - <classpath refid="testclasspath" /> + <target name="test" depends="compile,compile-tests"> + <junit fork="true" haltonfailure="true" printsummary="on"> + <classpath refid="test.classpath" /> <formatter type="plain" usefile="false"/> <batchtest> - <fileset dir="${classes}" + <fileset dir="${testclasses}" includes="**/*Test.class"/> </batchtest> </junit> @@ -165,8 +189,8 @@ <target name="war" depends="compile"> <war destfile="${warfile}" - webxml="${webxmlfile}"> - <fileset dir="web"/> + webxml="${resources}/web.xml"> + <fileset dir="${resources}/web"/> <restrict> <not> <name name="META-INF/*" /> @@ -183,14 +207,14 @@ <fileset dir="${classes}" includes="**/*" excludes="**/Test*.class"/> - <fileset dir="${config}" includes="jetty.xml" /> - <zipfileset dir="${config}" + <fileset dir="${resources}" includes="jetty.xml" /> + <zipfileset dir="${resources}" prefix="" includes="logback.xml"/> - <zipfileset dir="${config}" + <zipfileset dir="${resources}" prefix="WEB-INF/classes" includes="logback.xml"/> - <metainf dir="${config}" + <metainf dir="${resources}" includes="context.xml"/> <manifest> <attribute name="Created-By" value="The Tor Project" /> @@ -222,7 +246,7 @@ </restrict> <fileset dir="${classes}" excludes="org/torproject/onionoo/server/" /> - <fileset dir="${config}" includes="logback.xml" /> + <fileset dir="${resources}" includes="logback.xml" /> <exclude name="**/Test*.class"/> <manifest> <attribute name="Created-By" value="The Tor Project" /> @@ -239,5 +263,39 @@ <target name="dist" depends="test, war, jar"/> + <taskdef classpathref="cobertura.classpath" resource="tasks.properties" /> + <target name="coverage" depends="compile,compile-tests"> + <copy todir="${instrument}" > + <fileset dir="${classes}" > + <include name="**/*"/> + </fileset> + </copy> + <cobertura-instrument ignoreTrivial="true"> + <fileset dir="${instrument}"> + <include name="**/**/*.class" /> + </fileset> + </cobertura-instrument> + <junit fork="true" haltonfailure="false" printsummary="on"> + <sysproperty key="net.sourceforge.cobertura.datafile" + file="${cobertura.ser.file}" /> + <classpath refid="cobertura.test.classpath" /> + <formatter type="xml" /> + <batchtest toDir="${testresult}" > + <fileset dir="${testclasses}" includes="**/*Test.class" /> + </batchtest> + </junit> + <cobertura-report format="html" destdir="${coverageresult}" > + <fileset dir="${javasources}"> + <include name="**/*.java" /> + </fileset> + </cobertura-report> + <cobertura-check branchrate="0" linerate="0" totallinerate="34" totalbranchrate="30" > + <regex pattern="org.torproject.onionoo.server" branchrate="67" linerate="77"/> + <regex pattern="org.torproject.onionoo.docs" branchrate="19" linerate="29"/> + <regex pattern="org.torproject.onionoo.updater" branchrate="17" linerate="18"/> + <regex pattern="org.torproject.onionoo.writer" branchrate="22" linerate="20"/> + </cobertura-check> + </target> + </project> diff --git a/etc/context.xml.template b/etc/context.xml.template deleted file mode 100644 index 09e5bac..0000000 --- a/etc/context.xml.template +++ /dev/null @@ -1,3 +0,0 @@ -<Context cookies="false"> -</Context> - diff --git a/etc/jetty.xml b/etc/jetty.xml deleted file mode 100644 index c172902..0000000 --- a/etc/jetty.xml +++ /dev/null @@ -1,45 +0,0 @@ -<?xml version="1.0"?> -<!DOCTYPE Configure PUBLIC "-//Jetty//Configure//EN" - "http://www.eclipse.org/jetty/configure.dtd"> - -<Configure id="server" class="org.eclipse.jetty.server.Server" > - <Set name="dumpAfterStart">false</Set> - - <Set name="threadPool"> - <New class="org.eclipse.jetty.util.thread.QueuedThreadPool"> - <Set name="minThreads">25</Set> - <Set name="maxThreads">250</Set> - <Set name="detailedDump">false</Set> - </New> - </Set> - - <New id="webAppContext" class="org.eclipse.jetty.webapp.WebAppContext"> - <Set name="logUrlOnStart">true</Set> - <Set name="war"> - <Call class="java.lang.System" name="getProperty"> - <Arg>java.class.path</Arg> - </Call> - </Set> - </New> - - <Call name="addConnector"> - <Arg> - <New class="org.eclipse.jetty.server.nio.SelectChannelConnector"> - <Set name="port">8080</Set> - </New> - </Arg> - </Call> - - <Set name="handler"> -  - <New class="org.eclipse.jetty.server.handler.ContextHandlerCollection"> - <Call name="addHandler"> - <Arg> - <Ref id="webAppContext"/> - </Arg> - </Call> - </New> - </Set> - -</Configure> - diff --git a/etc/logback.xml b/etc/logback.xml deleted file mode 100644 index 19ae9f7..0000000 --- a/etc/logback.xml +++ /dev/null @@ -1,82 +0,0 @@ -<configuration debug="false"> - -  - <property name="logfile-base" value="${LOGBASE}/onionoo-" /> - -  - <property name="fileall-logname" value="${logfile-base}all" /> - <property name="fileerr-logname" value="${logfile-base}err" /> - <property name="filestatistics-logname" value="${logfile-base}statistics" /> - -  - <property name="utc-date-pattern" value="%date{ISO8601, UTC}" /> - -  - <appender name="FILEALL" class="ch.qos.logback.core.rolling.RollingFileAppender"> - <file>${fileall-logname}.log</file> - <encoder> - <pattern>${utc-date-pattern} %level %logger{20}:%line %msg%n</pattern> - </encoder> - <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> -  - <FileNamePattern>${fileall-logname}.%d{yyyy-MM-dd}.%i.log</FileNamePattern> - <maxHistory>10</maxHistory> - <timeBasedFileNamingAndTriggeringPolicy - class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP"> -  - <maxFileSize>1MB</maxFileSize> - </timeBasedFileNamingAndTriggeringPolicy> - </rollingPolicy> - </appender> - - <appender name="FILEERR" class="ch.qos.logback.core.FileAppender"> - <file>${fileerr-logname}.log</file> - <encoder> - <pattern>${utc-date-pattern} %level %logger{20}:%line %msg%n</pattern> - </encoder> - -  - <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> - <level>ERROR</level> - </filter> - </appender> - - <appender name="FILESTATISTICS" class="ch.qos.logback.core.FileAppender"> - <file>${filestatistics-logname}.log</file> - <encoder> - <pattern>${utc-date-pattern} %msg%n</pattern> - </encoder> - -  - <filter class="ch.qos.logback.classic.filter.LevelFilter"> - <level>INFO</level> - <onMatch>ACCEPT</onMatch> - <onMismatch>DENY</onMismatch> - </filter> - </appender> - -  - <logger name="org.torproject" > - <appender-ref ref="FILEERR" /> - </logger> - - <logger name="org.eclipse" level="INFO" /> - - <logger name="org.torproject.onionoo.cron.Main" > - <appender-ref ref="FILESTATISTICS" /> - </logger> - - <logger name="org.torproject.onionoo.server.PerformanceMetrics" > - <appender-ref ref="FILESTATISTICS" /> - </logger> - - <logger name="statistics" > - <appender-ref ref="FILESTATISTICS" /> - </logger> - - <root level="ALL"> - <appender-ref ref="FILEALL" /> - </root> - -</configuration> - diff --git a/etc/web.xml.template b/etc/web.xml.template deleted file mode 100644 index cefdfae..0000000 --- a/etc/web.xml.template +++ /dev/null @@ -1,78 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> - -<web-app version="2.4" - xmlns="http://java.sun.com/xml/ns/j2ee" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://java.sun.com/xml/ns/j2ee - http://java.sun.com/xml/ns/j2ee/web-app_2_4.xsd" > - - <servlet> - <servlet-name>Resource</servlet-name> - <servlet-class> - org.torproject.onionoo.server.ResourceServlet - </servlet-class> - <init-param> - <param-name>maintenance</param-name> - <param-value>0</param-value> - </init-param> - </servlet> - <servlet-mapping> - <servlet-name>Resource</servlet-name> - <url-pattern>/summary</url-pattern> - </servlet-mapping> - <servlet-mapping> - <servlet-name>Resource</servlet-name> - <url-pattern>/details</url-pattern> - </servlet-mapping> - <servlet-mapping> - <servlet-name>Resource</servlet-name> - <url-pattern>/bandwidth</url-pattern> - </servlet-mapping> - <servlet-mapping> - <servlet-name>Resource</servlet-name> - <url-pattern>/weights</url-pattern> - </servlet-mapping> - <servlet-mapping> - <servlet-name>Resource</servlet-name> - <url-pattern>/clients</url-pattern> - </servlet-mapping> - <servlet-mapping> - <servlet-name>Resource</servlet-name> - <url-pattern>/uptime</url-pattern> - </servlet-mapping> - - <context-param> - <param-name>outDir</param-name> - <param-value>/srv/onionoo.torproject.org/onionoo/out/</param-value> - </context-param> - - <listener> - <listener-class> - org.torproject.onionoo.server.NodeIndexer - </listener-class> - </listener> - - <filter> - <filter-name>GzipFilter</filter-name> - <filter-class>org.eclipse.jetty.servlets.GzipFilter</filter-class> - <init-param> - <param-name>mimeTypes</param-name> - <param-value>text/html,text/xml,text/plain,application/json</param-value> - </init-param> - <init-param> - <param-name>excludedAgents</param-name> - <param-value>gozilla,traviata</param-value> - </init-param> - <init-param> - <param-name>minGzipSize</param-name> - <param-value>2048</param-value> - </init-param> - </filter> - - <filter-mapping> - <filter-name>GzipFilter</filter-name> - <url-pattern>/*</url-pattern> - </filter-mapping> - -</web-app> - diff --git a/src/main/resources/context.xml b/src/main/resources/context.xml new file mode 100644 index 0000000..09e5bac --- /dev/null +++ b/src/main/resources/context.xml @@ -0,0 +1,3 @@ +<Context cookies="false"> +</Context> + diff --git a/src/main/resources/jetty.xml b/src/main/resources/jetty.xml new file mode 100644 index 0000000..c172902 --- /dev/null +++ b/src/main/resources/jetty.xml @@ -0,0 +1,45 @@ +<?xml version="1.0"?> +<!DOCTYPE Configure PUBLIC "-//Jetty//Configure//EN" + "http://www.eclipse.org/jetty/configure.dtd"> + +<Configure id="server" class="org.eclipse.jetty.server.Server" > + <Set name="dumpAfterStart">false</Set> + + <Set name="threadPool"> + <New class="org.eclipse.jetty.util.thread.QueuedThreadPool"> + <Set name="minThreads">25</Set> + <Set name="maxThreads">250</Set> + <Set name="detailedDump">false</Set> + </New> + </Set> + + <New id="webAppContext" class="org.eclipse.jetty.webapp.WebAppContext"> + <Set name="logUrlOnStart">true</Set> + <Set name="war"> + <Call class="java.lang.System" name="getProperty"> + <Arg>java.class.path</Arg> + </Call> + </Set> + </New> + + <Call name="addConnector"> + <Arg> + <New class="org.eclipse.jetty.server.nio.SelectChannelConnector"> + <Set name="port">8080</Set> + </New> + </Arg> + </Call> + + <Set name="handler"> +  + <New class="org.eclipse.jetty.server.handler.ContextHandlerCollection"> + <Call name="addHandler"> + <Arg> + <Ref id="webAppContext"/> + </Arg> + </Call> + </New> + </Set> + +</Configure> + diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml new file mode 100644 index 0000000..19ae9f7 --- /dev/null +++ b/src/main/resources/logback.xml @@ -0,0 +1,82 @@ +<configuration debug="false"> + +  + <property name="logfile-base" value="${LOGBASE}/onionoo-" /> + +  + <property name="fileall-logname" value="${logfile-base}all" /> + <property name="fileerr-logname" value="${logfile-base}err" /> + <property name="filestatistics-logname" value="${logfile-base}statistics" /> + +  + <property name="utc-date-pattern" value="%date{ISO8601, UTC}" /> + +  + <appender name="FILEALL" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${fileall-logname}.log</file> + <encoder> + <pattern>${utc-date-pattern} %level %logger{20}:%line %msg%n</pattern> + </encoder> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> +  + <FileNamePattern>${fileall-logname}.%d{yyyy-MM-dd}.%i.log</FileNamePattern> + <maxHistory>10</maxHistory> + <timeBasedFileNamingAndTriggeringPolicy + class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP"> +  + <maxFileSize>1MB</maxFileSize> + </timeBasedFileNamingAndTriggeringPolicy> + </rollingPolicy> + </appender> + + <appender name="FILEERR" class="ch.qos.logback.core.FileAppender"> + <file>${fileerr-logname}.log</file> + <encoder> + <pattern>${utc-date-pattern} %level %logger{20}:%line %msg%n</pattern> + </encoder> + +  + <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> + <level>ERROR</level> + </filter> + </appender> + + <appender name="FILESTATISTICS" class="ch.qos.logback.core.FileAppender"> + <file>${filestatistics-logname}.log</file> + <encoder> + <pattern>${utc-date-pattern} %msg%n</pattern> + </encoder> + +  + <filter class="ch.qos.logback.classic.filter.LevelFilter"> + <level>INFO</level> + <onMatch>ACCEPT</onMatch> + <onMismatch>DENY</onMismatch> + </filter> + </appender> + +  + <logger name="org.torproject" > + <appender-ref ref="FILEERR" /> + </logger> + + <logger name="org.eclipse" level="INFO" /> + + <logger name="org.torproject.onionoo.cron.Main" > + <appender-ref ref="FILESTATISTICS" /> + </logger> + + <logger name="org.torproject.onionoo.server.PerformanceMetrics" > + <appender-ref ref="FILESTATISTICS" /> + </logger> + + <logger name="statistics" > + <appender-ref ref="FILESTATISTICS" /> + </logger> + + <root level="ALL"> + <appender-ref ref="FILEALL" /> + </root> + +</configuration> + diff --git a/src/main/resources/web.xml b/src/main/resources/web.xml new file mode 100644 index 0000000..cefdfae --- /dev/null +++ b/src/main/resources/web.xml @@ -0,0 +1,78 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<web-app version="2.4" + xmlns="http://java.sun.com/xml/ns/j2ee" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://java.sun.com/xml/ns/j2ee + http://java.sun.com/xml/ns/j2ee/web-app_2_4.xsd" > + + <servlet> + <servlet-name>Resource</servlet-name> + <servlet-class> + org.torproject.onionoo.server.ResourceServlet + </servlet-class> + <init-param> + <param-name>maintenance</param-name> + <param-value>0</param-value> + </init-param> + </servlet> + <servlet-mapping> + <servlet-name>Resource</servlet-name> + <url-pattern>/summary</url-pattern> + </servlet-mapping> + <servlet-mapping> + <servlet-name>Resource</servlet-name> + <url-pattern>/details</url-pattern> + </servlet-mapping> + <servlet-mapping> + <servlet-name>Resource</servlet-name> + <url-pattern>/bandwidth</url-pattern> + </servlet-mapping> + <servlet-mapping> + <servlet-name>Resource</servlet-name> + <url-pattern>/weights</url-pattern> + </servlet-mapping> + <servlet-mapping> + <servlet-name>Resource</servlet-name> + <url-pattern>/clients</url-pattern> + </servlet-mapping> + <servlet-mapping> + <servlet-name>Resource</servlet-name> + <url-pattern>/uptime</url-pattern> + </servlet-mapping> + + <context-param> + <param-name>outDir</param-name> + <param-value>/srv/onionoo.torproject.org/onionoo/out/</param-value> + </context-param> + + <listener> + <listener-class> + org.torproject.onionoo.server.NodeIndexer + </listener-class> + </listener> + + <filter> + <filter-name>GzipFilter</filter-name> + <filter-class>org.eclipse.jetty.servlets.GzipFilter</filter-class> + <init-param> + <param-name>mimeTypes</param-name> + <param-value>text/html,text/xml,text/plain,application/json</param-value> + </init-param> + <init-param> + <param-name>excludedAgents</param-name> + <param-value>gozilla,traviata</param-value> + </init-param> + <init-param> + <param-name>minGzipSize</param-name> + <param-value>2048</param-value> + </init-param> + </filter> + + <filter-mapping> + <filter-name>GzipFilter</filter-name> + <url-pattern>/*</url-pattern> + </filter-mapping> + +</web-app> + diff --git a/src/main/resources/web/css/style.css b/src/main/resources/web/css/style.css new file mode 100644 index 0000000..e3caa4a --- /dev/null +++ b/src/main/resources/web/css/style.css @@ -0,0 +1,36 @@ +body { font-family: "Open Sans","lucida grande","Segoe UI",arial,verdana, + "lucida sans unicode",tahoma,sans-serif; background: #fafafa; + font-size: 13px; line-height: 22px; color: #222; } +h3 { color: #7D4698; position: relative } +a { color: #7D4698; text-decoration: none; font-weight: bold; } +p { margin: 0; padding: 10px; } +a[name] { padding: 0; margin: 0; } +.box { max-width: 850px; width: 100%; margin: 0 auto 30px auto; + padding-bottom: 30px; background: white; border: 1px solid #eee; } +.box > * { margin-left: 30px; margin-right: 30px; } +.box h3 a { visibility: hidden; } +.box:hover h3 a { visibility: visible; } +.api-request { border-bottom: 1px solid #eee; position: relative } +.request-url, .request-type, .request-response { padding: 8px 10px; + vertical-align: middle } +.request-type { color: #57145F; display: inline-block; } +.request-url { color: #333; font-size: 18px; } +.request-response { position: absolute; color: #666; right: 0; } +h3 .request-response { padding: 0 !important; } +.api-urls>li:last-child { border-bottom: 0; } +.required-true, .required-false, .typeof { display: inline-block; + vertical-align: middle; padding: 5px 10px; } +.required-true { color: #1d7508; } +.required-false { color: #aaa; } +.properties { margin-top: 10px; margin-bottom: 10px; + border: 1px solid #eee; } +.properties li { padding: 5px 0; } +.properties li ul { border: 1px solid #eee; margin: 10px 10px 10px 40px; + background: white; } +.properties .properties { margin-left: 10px; } +.properties li:nth-child(even) { background: #fafafa; } +.properties p { padding: 10px 15px; } +.properties b { padding: 5px 10px; display: inline-block; + vertical-align: middle; } +.api-urls{ margin-top: 30px; margin-bottom: 30px; } + diff --git a/src/main/resources/web/favicon.ico b/src/main/resources/web/favicon.ico new file mode 100644 index 0000000..48060b1 Binary files /dev/null and b/src/main/resources/web/favicon.ico differ diff --git a/src/main/resources/web/index.html b/src/main/resources/web/index.html new file mode 100644 index 0000000..c7095ec --- /dev/null +++ b/src/main/resources/web/index.html @@ -0,0 +1,141 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> +<html> +<head> +<title>Onionoo — a Tor network status protocol</title> +<link href="css/style.css" type="text/css" rel="stylesheet"> +<meta http-equiv="content-type" content="text/html; charset=ISO-8859-1"> +<link href="favicon.ico" type="image/x-icon" rel="shortcut icon"> +</head> +<body> + +<div class="box"> + +<h1><a href="index.html">Onionoo</a> —</h1> +<h2>a Tor network status <a href="protocol.html">protocol</a></h2> + +Onionoo is a web-based protocol to learn about currently running +Tor relays and bridges. Onionoo itself was not designed as a service +for human beings—at least not directly. Onionoo provides the +data for other applications and websites which in turn present Tor +network status information to humans. The following Onionoo clients +are currently available: +<ul> +<li><a href="https://atlas.torproject.org/">Atlas</a> is a web +application to discover Tor relays. It provides useful +information on how relays are configured along with graphics about +their past.</li> +<li><a href="https://compass.torproject.org/">Compass</a> +is a Python script and website that extracts consensus weight +information of currently running relays and aggregates weights of +relays running in the same country or same autonomous system.</li> +<li><a href="http://tor2web.org/">Tor2web</a> is a web proxy to Tor +Hidden Services. It uses Onionoo to get the list of currently running +Tor Exits to detect if the client is a Tor user and if so redirect +them to the .onion address.</li> +<li>The <a href="https://nos-oignons.net/Services/index.en.html">Nos +oignons</a> website uses Onionoo to visualize bandwidth histories of +their relays.</li> +<li>The <a href="https://metrics.torproject.org/bubbles.html">metrics +website</a> visualizes diversity of the Tor network using bubble +graphs.</li> +<li><a href="https://github.com/woeisme/torchart">torchart</a> is a PHP +WordPress plugin which draws 3 day traffic charts using pChart currently +used for displaying metric charts on +<a href="http://icetor.is/wordpress/metrics/">icetor.is</a>.</li> +<li><a href="https://github.com/kloesing/challenger">challenger</a> +aggregates graph data from relays participating in +<a href="https://www.eff.org/torchallenge/">EFF's 2014 Tor Challenge</a> +to provide statistics like total bytes transferred.</li> +<li><a href="https://play.google.com/store/apps/details?id=com.networksaremadeofstring.a…">AnOnionooid</a> +is an Android app that helps find and explore Tor relays and bridges.</li> +<li><a href="https://oniontip.com/">OnionTip</a> uses Onionoo's data to +distribute donations to relays that have a bitcoin address in their +contact line.</li> +<li><a href="https://savannah.nongnu.org/projects/koninoo/">koninoo</a> is +a simple Java command line interface for querying Onionoo data.</li> +<li>The <a href="https://duckduckgo.com/">DuckDuckGo</a> search engine +displays Tor node details when being asked for "tor node" followed by a +search term.</li> +<li><a href="https://onionview.com/">OnionView</a> is a simple web service +which uses Tor relay data to plot the location of active Tor nodes onto an +interactive map of the world.</li> +</ul> + +The following library facilitates development of Onionoo clients: +<ul> +<li><a href="https://github.com/duk3luk3/onion-py">OnionPy</a> is a +comprehensive pure-Python (2.7+) Onionoo wrapper with caching +support.</li> +</ul> + +<h2>Developing Onionoo applications</h2> + +The project pages of the Onionoo clients listed above have further +information for contacting the authors and contributing ideas or code. +The authors will be happy to hear your thoughts! + +You don't find your favorite Onionoo client above? Want to +implement your own and tell us to add it to the list? The Onionoo +clients above are backed by a web-based +<a href="protocol.html">protocol</a>, which +facilitates developing new applications displaying Tor status +information. Here are a few ideas for new Onionoo clients: +<ul> +<li>Tor controller extension: Extend +<a href="https://www.torproject.org/projects/vidalia.html.en">Vidalia</a> and/or +<a href="https://www.torproject.org/projects/arm.html.en">arm</a> to look up details for the bridge +that the user is running and display what pool the bridge is contained +in.</li> +<li>Social network site plugin: Add a plugin to the social network +site of your choice to show your friends what Tor relays and bridges +you're running and how that helps users around the world.</li> +<li>Desktop tray icon: Write a tray icon for your favorite desktop +environment that tells you when your relay or bridge is down and that +displays some basic usage statistics.</li> +<li>E-mail notification service: Improve our e-mail notification +service <a href="https://weather.torproject.org/">Weather</a> by +implementing its own relay search or extending it to report when a +bridge drops off the network.</li> +<li>Command-line tool: Implement a command-line tool that quickly +searches a relay or bridge and prints out some status information to +help debug problems.</li> +<li>(Insert your idea here.)</li> +</ul> + +Want to help with developing the Onionoo server that provides +Tor status data, or want to run your own Onionoo server instance? The +Onionoo server is written in Java with a tiny portion of Java +Servlets. Instructions for setting up the Onionoo server to fetch the +required data from the Tor servers is described in the INSTALL file in +the sources. For more details see the +<a href="https://gitweb.torproject.org/onionoo.git">source code</a> and +<a href="https://trac.torproject.org/projects/tor/query?status=!closed&component=Oni…">issue +tracker</a>. + +<h2>Related projects</h2> + +TorStatus is the name of a nowadays +<a href="https://svn.torproject.org/svn/torstatus/trunk/">unmaintained</a> +website that displays Tor relay information similar to +<a href="http://atlas.torproject.org/">Atlas</a>. There are still a +few <a href="http://torstatus.blutmagie.de/">TorStatus</a> +<a href="https://torstatus.rueckgr.at/">websites</a> +<a href="http://tns.hermetix.org/">running</a>. + +There's another project from summer 2011 called TorStatus which is +a <a href="https://gitweb.torproject.org/torstatus.git">rewrite</a> of +the original TorStatus in Python/Django. Unfortunately, it's also +unmaintained. + +Finally, there's the +<a href="https://consensus-health.torproject.org/">consensus-health +page</a> which has the primary purpose of indicating problems with +creating a network status consensus. As a side-effect this page lists +all currently running relays and how the directory authorities voted +on them. + +</div>  + +</body> +</html> + diff --git a/src/main/resources/web/protocol.html b/src/main/resources/web/protocol.html new file mode 100644 index 0000000..c6cc534 --- /dev/null +++ b/src/main/resources/web/protocol.html @@ -0,0 +1,2338 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> +<html> +<head> +<title>Onionoo — protocol overview</title> +<link href="css/style.css" type="text/css" rel="stylesheet"> +<meta http-equiv="content-type" content="text/html; charset=ISO-8859-1"> +<link href="favicon.ico" type="image/x-icon" rel="shortcut icon"> +</head> +<body> + +<div class="box"> + +<h1><a href="index.html">Onionoo</a> —</h1> +<h2>protocol overview</h2> + + <h4>Table of contents</h4> + <ul> + <li><a href="#general">General</a></li> + <li><a href="#methods">Methods</a></li> + <li><a href="#summary">Summary documents</a></li> + <li><a href="#details">Details documents</a></li> + <li><a href="#history">History objects</a></li> + <li><a href="#bandwidth">Bandwidth documents</a></li> + <li><a href="#weights">Weights documents</a></li> + <li><a href="#clients">Clients documents</a></li> + <li><a href="#uptime">Uptime documents</a></li> + <li><a href="#examples">Example usage</a></li> + </ul> + +</div> +<div class="box"> + +<a name="general"></a> +<h3>General <a href="#general">#</a></h3> + + +The Onionoo service is designed as a RESTful web service. +Onionoo clients send HTTP GET requests to the Onionoo server which +responds with JSON-formatted replies. +Onionoo clients and their developers should follow a few rules: + + +<h4>Compression</h4> + +Clients should include an "Accept-Encoding: +gzip" header in their requests and handle gzip-compressed +responses. +Only requests starting at a certain size will be compressed by the +server. + + +<h4>Caching</h4> +Clients should make use of the "Last-Modified" +header of responses and include that timestamp in a +"If-Modified-Since" header of subsequent requests. + + +<h4>Response codes</h4> + +Clients should handle response codes by +distinguishing between client and server errors, and if there's a problem, +informing their users about the kind of problem. +The following response codes are used: + + +<ul class="properties"> + +<li> +200 OK + +The request was processed successfully. + +</li> + +<li> +304 Not Modified + +Server data has not changed since the +"If-Modified-Since" header included in the request. + +</li> + +<li> +400 Bad Request + +The request for a known resource could not be processed because of bad +syntax. +This is most likely a client problem. + +</li> + +<li> +404 Not Available + +The request could not be processed because the requested resource could +not be found. +This is most likely a client problem. + +</li> + +<li> +500 Internal Server Error + +There is an unspecific problem with +the server which the service operator may not yet be aware of. +Please check if there's already a bug report for this problem, and if not, +file one. + +</li> + +<li> +503 Service Unavailable + +The server is temporarily down for +maintenance, or there is a temporary problem with the server that the +service operator is already aware of. +Only file a bug report if this problem persists. + +</li> + +</ul> + +<h4>Protocol versions</h4> + +There are plenty of reasons why we may have +to change the protocol described here. +Clients should be able to handle all valid JSON responses, ignoring +unknown fields and not breaking horribly in case of missing fields. +Protocol changes will be announced here by writing deprecated API parts in +red and new parts in +blue. +Deprecated parts will be removed one month after their announcement. +If you want to be informed of upcoming protocol changes, subscribe to the +<a href="https://lists.torproject.org/cgi-bin/mailman/listinfo/onionoo-announce">onionoo-announce</a> +mailing list. + + +All responses contain a "version" string that +indicates whether clients can parse the document or not. +Version strings consist of a major and a minor version number. +The major version number is raised when previously required fields are +dropped or turned into optional fields, when request parameters or +response documents are removed, or when there are structural changes. +The minor version number is raised when new fields, request parameters, or +response documents are added or optional fields are dropped. +If clients support the same major version given in a response but only a +lower minor version, they should be able to parse the document but may not +understand all fields. +If clients support a lower major version, they should not attempt to parse +a document, because there may be backward-incompatible changes. + +Responses may also contain the optional +"next_major_version_scheduled" field that announces when +the next major version is scheduled to be deployed. +Clients should inform their users that they should upgrade to the next +major protocol version before the provided date. + +The following versions have been used in the past six months or are +scheduled to be deployed in the next months: + +<ul> +<li>1.0: First assigned version number on August 31, +2014.</li> +<li>1.1: Added optional "next_major_version_scheduled" +field on September 16, 2014.</li> +<li>1.2: Added qualified search terms to "search" +parameter on October 17, 2014.</li> +<li>2.0: Extended search parameter to base64-encoded +fingerprints on November 15, 2014.</li> +<li>2.1: Removed optional "advertised_bandwidth_fraction" +field from details documents and optional "advertised_bandwidth" and +"advertised_bandwidth_fraction" fields from weights documents on November +16, 2014.</li> +<li>2.2: Removed optional "pool_assignment" field and +added "transports" field to bridge details documents on December 8, +2014.</li> +<li>2.3: Added optional "flags" field to uptime +documents on March 22, 2015.</li> +<li>2.4: Added optional "effective_family" field to +details documents on July 3, 2015.</li> +<li>2.5: Added optional "measured" field to details +documents on August 13, 2015.</li> +<li>2.6: Added optional "alleged_family" and +"indirect_family" fields and deprecated optional "family" field in details +documents on August 25, 2015.</li> +<li>3.0: Extended search parameter to match any 4 hex +characters of a space-separated fingerprint on November 15, 2015.</li> +<li>3.1: Removed optional "family" field on January 18, +2016.</li> +</ul> + +</div>  + +<div class="box"> + +<a name="methods"></a> +<h3>Methods <a href="#methods">#</a></h3> + + +The following methods each return a single document containing zero or +more objects of relays and/or bridges. +By default, all relays and bridges are included that have been running in +the past week. + + +<ul class="api-urls"> + +<li class="api-request"> +GET +https://onionoo.torproject.org/summary +returns a <a href="#summary">summary +document</a> +</li> + +<li class="api-request"> +GET +https://onionoo.torproject.org/details +returns a <a href="#details">details +document</a> +</li> + +<li class="api-request"> +GET +https://onionoo.torproject.org/bandwidth +returns a <a href="#bandwidth">bandwidth +document</a> +</li> + +<li class="api-request"> +GET +https://onionoo.torproject.org/weights +returns a <a href="#weights">weights +document</a> +</li> + +<li class="api-request"> +GET +https://onionoo.torproject.org/clients +returns a <a href="#clients">clients +document</a> +</li> + +<li class="api-request"> +GET +https://onionoo.torproject.org/uptime +returns a <a href="#uptime">uptime +document</a> +</li> + +</ul> + +<h4>Parameters</h4> + +Each of the methods can be parameterized to select only a subset of relay +and/or bridge documents that are currently running or that have been +running in the past week. +(The fingerprint parameter is special here, because it +allows selecting a specific relay or bridge, regardless of whether it has +been running in the past week.) +If multiple parameters are specified, they are combined using a logical +AND operation, meaning that only the intersection of relays and bridges +matching all parameters is returned. +If the same parameter is specified more than once, only the first +parameter value is considered. + + +<ul class="properties"> + +<li> +type + +Return only relay (parameter value +relay) or only bridge documents (parameter value +bridge). +Parameter values are case-insensitive. + +</li> + +<li> +running + +Return only running (parameter value +true) or only non-running relays and/or bridges +(paramter value +false). +Parameter values are case-insensitive. + +</li> + +<li> +search + +Return only (1) relays with the parameter value matching (part of a) +nickname, (possibly $-prefixed) beginning of a hex-encoded fingerprint, +any 4 hex characters of a space-separated fingerprint, beginning of a +base64-encoded fingerprint without trailing equal signs, or beginning of +an IP address, (2) bridges with (part of a) nickname or (possibly +$-prefixed) beginning of a hashed hex-encoded fingerprint, and (3) relays +and/or bridges matching a given qualified search term. +Searches by relay IP address include all known addresses used for onion +routing and for exiting to the Internet. +Searches for beginnings of IP addresses are performed on textual +representations of canonical IP address forms, so that searches using CIDR +notation or non-canonical forms will return empty results. +Searches are case-insensitive, except for base64-encoded fingerprints. +If multiple search terms are given, separated by spaces, the intersection +of all relays and bridges matching all search terms will be returned. +Complete hex-encoded fingerprints should always be hashed using SHA-1, +regardless of searching for a relay or a bridge, in order to not +accidentally leak non-hashed bridge fingerprints in the URL. +Qualified search terms have the form "key:value" (without double quotes) +with "key" being one of the parameters listed here except for "search", +"fingerprint", "order", "limit", "offset", and "fields", and "value" being +the string that will internally be passed to that parameter. + +</li> + +<li> +lookup + +Return only the relay with the parameter +value matching the fingerprint or the bridge with the parameter value +matching the hashed fingerprint. +Fingerprints should always be hashed using SHA-1, regardless of looking up +a relay or a bridge, in order to not accidentally leak non-hashed bridge +fingerprints in the URL. +Lookups only work for full fingerprints or hashed fingerprints consisting +of 40 hex characters. +Lookups are case-insensitive. + +</li> + +<li> +fingerprint + +Return only the relay with the parameter value matching the fingerprint +or the bridge with the parameter value matching the hashed fingerprint. +Fingerprints must consist of 40 hex characters, case does not matter. +This parameter is quite similar to the lookup parameter +with two exceptions: +(1) the provided relay fingerprint or hashed bridge fingerprint must +not be hashed (again) using SHA-1; +(2) the response will contain any matching relay or bridge regardless of +whether they have been running in the past week. + +</li> + +<li> +country + +Return only relays which are located in the +given country as identified by a two-letter country code. +Filtering by country code is case-insensitive. + +</li> + +<li> +as + +Return only relays which are located in the +given autonomous system (AS) as identified by the AS number (with or +without preceding "AS" part). +Filtering by AS number is case-insensitive. + +</li> + +<li> +flag + +Return only relays which have the +given relay flag assigned by the directory authorities. +Note that if the flag parameter is specified more than once, only the +first parameter value will be considered. +Filtering by flag is case-insensitive. + +</li> + +<li> +first_seen_days + +Return only relays or bridges which +have first been seen during the given range of days ago. +A parameter value "x-y" with x <= y returns relays or bridges that have +first been seen at least x and at most y days ago. +Accepted short forms are "x", "x-", and "-y" which are interpreted as +"x-x", "x-infinity", and "0-y". + +</li> + +<li> +last_seen_days + +Return only relays or bridges which +have last been seen during the given range of days ago. +A parameter value "x-y" with x <= y returns relays or bridges that have +last been seen at least x and at most y days ago. +Accepted short forms are "x", "x-", and "-y" which are interpreted as +"x-x", "x-infinity", and "0-y". +Note that relays and bridges that haven't been running in the past week +are not included in results, so that setting x to 8 or higher will lead to +an empty result set. + +</li> + +<li> +contact + +Return only relays with the parameter value +matching (part of) the contact line. +If the parameter value contains spaces, only relays are returned which +contain all space-separated parts in their contact line. +Only printable ASCII characters are permitted in the parameter value, +some of which need to be percent-encoded (# as %23, % as %25, & as +%26, + as %2B, and / as %2F). +Comparisons are case-insensitive. + +</li> + +<li> +family + +Return only the relay whose fingerprint matches the parameter value and +all relays that this relay has listed in its family by fingerprint and +that in turn have listed this relay in their family by fingerprint. +If relays have listed other relays in their family by nickname, those +family relationships will not be considered, regardless of whether they +have the Named flag or not. +The provided relay fingerprint must consist of 40 hex characters where +case does not matter, and it must not be hashed using SHA-1. +Bridges are not contained in the result, regardless of whether they define +a family. + +</li> + +</ul> + + +Response documents can be reduced in size by requesting only a subset +of contained fields. + + +<ul class="properties"> + +<li> +fields + +Comma-separated list of fields that will be +included in the result. +So far, only top-level fields in relay or bridge objects of details +documents can be specified, e.g., +nickname,hashed_fingerprint. +If the fields parameter is provided, all other fields which are not +contained in the provided list will be removed from the result. +Field names are case-insensitive. + +</li> + +</ul> + + +Relay and/or bridge documents in the response can be ordered and +limited by providing further parameters. +If the same parameter is specified more than once, only the first +parameter value is considered. + + +<ul class="properties"> + +<li> +order + +Re-order results by a comma-separated list +of fields in ascending or descending order. +Results are first ordered by the first list element, then by the second, +and so on. +Possible fields for ordering are: consensus_weight. +Field names are case-insensitive. +Ascending order is the default; descending order is selected by prepending +fields with a minus sign (-). +Relays or bridges which don't have any value for a field to be ordered by +are always appended to the end, regardless or sorting order. +The ordering is defined independent of the requested document type and +does not require the ordering field to be contained in the document. +If no order parameter is given, ordering of results is +undefined. + +</li> + +<li> +offset + +Skip the given number of relays and/or +bridges. +Relays are skipped first, then bridges. +Non-positive offset values are treated as zero and don't +change the +result. + +</li> + +<li> +limit + +Limit result to the given number of +relays and/or bridges. +Relays are kept first, then bridges. +Non-positive limit values are treated as zero and lead +to an empty +result. +When used together with offset, the offsetting step +precedes the +limiting step. + +</li> + +</ul> + +</div>  + +<div class="box"> +<a name="summary"></a> +<h3>Summary documents <a href="#summary">#</a> + +<a href="summary?limit=4">example request</a> + +</h3> + +Summary documents contain short summaries of relays with nicknames, +fingerprints, IP addresses, and running information as well as bridges +with hashed fingerprints and running information. +Summary documents contain the following fields: + +<ul class="properties"> + +<li> +version +<code class="typeof">string</code> +required + +Onionoo protocol version string. + +</li> + +<li> +next_major_version_scheduled +<code class="typeof">string</code> +optional + +UTC date (YYYY-MM-DD) when the next major protocol version is scheduled to +be deployed. +Omitted if no major protocol changes are planned. + +</li> + +<li> +relays_published +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when the last known relay network +status consensus started being valid. +Indicates how recent the relay summaries in this document are. + +</li> + +<li> +relays +<code class="typeof">array of objects</code> +required + +Array of relay summary objects as specified below. + +</li> + +<li> +bridges_published +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when +the last known bridge network status was published. +Indicates how recent the bridge summaries in this document are. + +</li> + +<li> +bridges +<code class="typeof">array of objects</code> +required + +Array of bridge summary objects as specified below. + +</li> + +</ul> + +<h4>Relay summary objects</h4> + + +Relay summary objects contain the following key-value pairs: + + +<ul class="properties"> + +<li> +n +<code class="typeof">string</code> +optional + +Relay nickname consisting of 1–19 alphanumerical characters. +Omitted if the relay nickname is "Unnamed". + +</li> + +<li> +f +<code class="typeof">string</code> +required + +Relay fingerprint consisting of 40 upper-case hexadecimal characters. + +</li> + +<li> +a +<code class="typeof">array of strings</code> +required + +Array of IPv4 or IPv6 addresses where the relay accepts +onion-routing connections or which the relay used to exit to the Internet +in the past 24 hours. +The first address is the primary onion-routing address that the relay used +to register in the network, subsequent addresses are in arbitrary order. +IPv6 hex characters are all lower-case. + +</li> + +<li> +r +<code class="typeof">boolean</code> +required + +Boolean field saying whether this relay was listed as +running in the last relay network status consensus. + +</li> + +</ul> + +<h4>Bridge summary objects</h4> + + +Bridge summary objects contain the following key-value pairs: + + +<ul class="properties"> + +<li> +n +<code class="typeof">string</code> +optional + +Bridge nickname consisting of 1–19 alphanumerical characters. +Omitted if the bridge nickname is "Unnamed". + +</li> + +<li> +h +<code class="typeof">string</code> +required + +SHA-1 hash of the bridge fingerprint consisting of 40 +upper-case hexadecimal characters. + +</li> + +<li> +r +<code class="typeof">boolean</code> +required + +Boolean field saying whether this bridge was listed as +running in the last bridge network status. + +</li> + +</ul> + +</div>  + +<div class="box"> +<a name="details"></a> +<h3>Details documents <a href="#details">#</a> + +<a href="details?limit=4">example request</a> + +</h3> + + +Details documents are based on network statuses published by the Tor +directories, server descriptors published by relays and bridges, and data +published by Tor network services TorDNSEL and BridgeDB. +Details documents use the most recently published data from these sources, +which may lead to contradictions between fields based on different sources +in rare edge cases. +Details documents contain the following fields: + + +<ul class="properties"> + +<li> +version +<code class="typeof">string</code> +required + +Onionoo protocol version string. + +</li> + +<li> +next_major_version_scheduled +<code class="typeof">string</code> +optional + +UTC date (YYYY-MM-DD) when the next major protocol version is scheduled to +be deployed. +Omitted if no major protocol changes are planned. + +</li> + +<li> +relays_published +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when +the last known relay network status consensus started being valid. +Indicates how recent the relay details in this document are. + +</li> + +<li> +relays +<code class="typeof">array of objects</code> +required + +Array of relay details objects as specified below. + +</li> + +<li> +bridges_published +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when +the last known bridge network status was published. +Indicates how recent the bridge details in this document are. + +</li> + +<li> +bridges +<code class="typeof">array of objects</code> +required + +Array of bridge details objects as specified below. + + +</li> + +</ul> + +<h4>Relay details objects</h4> + + +Relay details objects contain the following key-value pairs: + + +<ul class="properties"> + +<li> +nickname +<code class="typeof">string</code> +optional + +Relay nickname consisting of 1–19 alphanumerical characters. +Omitted if the relay nickname is "Unnamed". + +</li> + +<li> +fingerprint +<code class="typeof">string</code> +required + +Relay fingerprint consisting of 40 upper-case +hexadecimal characters. + +</li> + +<li> +or_addresses +<code class="typeof">array of strings</code> +required + +Array of IPv4 or IPv6 addresses and TCP ports +or port lists where the relay accepts onion-routing connections. +The first address is the primary onion-routing address that the relay used +to register in the network, subsequent addresses are in arbitrary order. +IPv6 hex characters are all lower-case. + +</li> + +<li> +exit_addresses +<code class="typeof">array of strings</code> +optional + +Array of IPv4 or IPv6 addresses that the +relay used to exit to the Internet in the past 24 hours. +IPv6 hex characters are all lower-case. +Only those addresses are listed that are different from onion-routing +addresses. +Omitted if array is empty. + +</li> + +<li> +dir_address +<code class="typeof">string</code> +optional + +IPv4 address and TCP port where the relay +accepts directory connections. +Omitted if the relay does not accept directory connections. + +</li> + +<li> +last_seen +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when this +relay was last seen in a network status consensus. + +</li> + +<li> +last_changed_address_or_port +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD +hh:mm:ss) when this relay last stopped announcing an IPv4 or IPv6 address +or TCP port where it previously accepted onion-routing or directory +connections. +This timestamp can serve as indicator whether this relay would be a +suitable fallback directory. + +</li> + +<li> +first_seen +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when this +relay was first seen in a network status consensus. + +</li> + +<li> +running +<code class="typeof">boolean</code> +required + +Boolean field saying whether this relay was listed as +running in the last relay network status consensus. + +</li> + +<li> +hibernating +<code class="typeof">boolean</code> +optional + +Boolean field saying whether this relay indicated that it is hibernating +in its last known server descriptor. +This information may be helpful to decide whether a relay that is not +running anymore has reached its accounting limit and has not dropped out +of the network for another, unknown reason. +Omitted if either the relay is not hibernating, or if no information is +available about the hiberation status of the relay. + +</li> + +<li> +flags +<code class="typeof">array of strings</code> +optional + +Array of relay flags that the directory authorities +assigned to this relay. +Omitted if empty. + +</li> + +<li> +country +<code class="typeof">string</code> +optional + +Two-letter lower-case country code as found in a +GeoIP database by resolving the relay's first onion-routing IP address. +Omitted if the relay IP address could not be found in the GeoIP +database. + +</li> + +<li> +country_name +<code class="typeof">string</code> +optional + +Country name as found in a GeoIP database by +resolving the relay's first onion-routing IP address. +Omitted if the relay IP address could not be found in the GeoIP +database, or if the GeoIP database did not contain a country name. + +</li> + +<li> +region_name +<code class="typeof">string</code> +optional + +Region name as found in a GeoIP database by +resolving the relay's first onion-routing IP address. +Omitted if the relay IP address could not be found in the GeoIP +database, or if the GeoIP database did not contain a region name. + +</li> + +<li> +city_name +<code class="typeof">string</code> +optional + +City name as found in a +GeoIP database by resolving the relay's first onion-routing IP address. +Omitted if the relay IP address could not be found in the GeoIP +database, or if the GeoIP database did not contain a city name. + +</li> + +<li> +latitude +<code class="typeof">number</code> +optional + +Latitude as found in a GeoIP database by resolving +the relay's first onion-routing IP address. +Omitted if the relay IP address could not be found in the GeoIP +database. + +</li> + +<li> +longitude +<code class="typeof">number</code> +optional + +Longitude as found in a GeoIP database by +resolving the relay's first onion-routing IP address. +Omitted if the relay IP address could not be found in the GeoIP +database. + +</li> + +<li> +as_number +<code class="typeof">string</code> +optional + +AS number as found in an AS database by +resolving the relay's first onion-routing IP address. +AS number strings start with "AS", followed directly by the AS number. +Omitted if the relay IP address could not be found in the AS +database. + +</li> + +<li> +as_name +<code class="typeof">string</code> +optional + +AS name as found in an AS database by resolving the +relay's first onion-routing IP address. +Omitted if the relay IP address could not be found in the AS +database. + +</li> + +<li> +consensus_weight +<code class="typeof">number</code> +required + +Weight assigned to this relay by the +directory authorities that clients use in their path selection algorithm. +The unit is arbitrary; currently it's kilobytes per second, but that might +change in the future. + +</li> + +<li> +host_name +<code class="typeof">string</code> +optional + +Host name as found in a reverse DNS lookup of the +relay IP address. +This field is updated at most once in 12 hours, unless the relay IP +address changes. +Omitted if the relay IP address was not looked up or if no lookup request +was successful yet. + +</li> + +<li> +last_restarted +<code class="typeof">string</code> +optional + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when the +relay was last (re-)started. +Missing if router descriptor containing this information cannot be +found. + +</li> + +<li> +bandwidth_rate +<code class="typeof">number</code> +optional + +Average bandwidth +in bytes per second that this relay is willing to sustain over long +periods. +Missing if router descriptor containing this information cannot be +found. + +</li> + +<li> +bandwidth_burst +<code class="typeof">number</code> +optional + +Bandwidth in bytes +per second that this relay is willing to sustain in very short intervals. +Missing if router descriptor containing this information cannot be +found. + +</li> + +<li> +observed_bandwidth +<code class="typeof">number</code> +optional + +Bandwidth +estimate in bytes per second of the capacity this relay can handle. +The relay remembers the maximum bandwidth sustained output over any ten +second period in the past day, and another sustained input. +The "observed_bandwidth" value is the lesser of these two numbers. +Missing if router descriptor containing this information cannot be +found. + +</li> + +<li> +advertised_bandwidth +<code class="typeof">number</code> +optional + +Bandwidth in bytes per second that this +relay is willing and capable to provide. +This bandwidth value is the minimum of bandwidth_rate, +bandwidth_burst, and observed_bandwidth. +Missing if router descriptor containing this information cannot be +found. + +</li> + +<li> +exit_policy +<code class="typeof">array of strings</code> +optional + +Array of exit-policy lines. +Missing if router descriptor containing this information cannot be +found. +May contradict the "exit_policy_summary" field in a rare +edge case: this happens when the relay changes its exit policy after the +directory authorities summarized the previous exit policy. + +</li> + +<li> +exit_policy_summary +<code class="typeof">object</code> +optional + +Summary +version of the relay's exit policy containing a dictionary with either an +"accept" or a "reject" element. +If there is an "accept" ("reject") element, the relay accepts (rejects) +all TCP ports or port ranges in the given list for most IP addresses and +rejects (accepts) all other ports. +May contradict the "exit_policy" field in a rare edge +case: this happens when the relay changes its exit policy after the +directory authorities summarized the previous exit policy. + +</li> + +<li> +exit_policy_v6_summary +<code class="typeof">object</code> +optional + +Summary version of the relay's IPv6 exit policy containing a dictionary +with either an "accept" or a "reject" element. +If there is an "accept" ("reject") element, the relay accepts (rejects) +all TCP ports or port ranges in the given list for most IP addresses and +rejects (accepts) all other ports. +Missing if the relay rejects all connections to IPv6 addresses. +May contradict the "exit_policy_summary" field in a rare +edge case: this happens when the relay changes its exit policy after the +directory authorities summarized the previous exit policy. + +</li> + +<li> +contact +<code class="typeof">string</code> +optional + +Contact address of the relay operator. +Omitted if empty or if descriptor containing this information cannot be +found. + +</li> + +<li> +platform +<code class="typeof">string</code> +optional + +Platform string containing operating system and Tor +version details. +Omitted if empty or if descriptor containing this information cannot be +found. + +</li> + +<li> +recommended_version +<code class="typeof">boolean</code> +optional + +Boolean field saying whether the Tor software version of this relay is +recommended by the directory authorities or not. +Omitted if either the directory authorities did not recommend versions, or +the relay did not report which version it runs. + +</li> + +<li> +family +<code class="typeof">array of strings</code> +optional + +Array of fingerprints or nicknames of relays that this relay considers to +be part of its family. +There are no cross-checks whether the listed relays exist or consider this +relay part of their family, so that the effective family of this relay may +be smaller. +Omitted if empty or if descriptor containing this information cannot be +found. +Deprecated on August 25, 2015, removed on January 18, +2016. + +</li> + +<li> +effective_family +<code class="typeof">array of strings</code> +optional + +Array of $-prefixed fingerprints of relays that are in an effective, +mutual family relationship with this relay. +These relays are part of this relay's family and they consider this relay +to be part of their family. +Omitted if empty or if descriptor containing this information cannot be +found. + +</li> + +<li> +alleged_family +<code class="typeof">array of strings</code> +optional + +Array of $-prefixed fingerprints of relays that are not in an effective, +mutual family relationship with this relay. +These relays are part of this relay's family but they don't consider this +relay to be part of their family. +Omitted if empty or if descriptor containing this information cannot be +found. + +</li> + +<li> +indirect_family +<code class="typeof">array of strings</code> +optional + +Array of $-prefixed fingerprints of relays that are not in an effective, +mutual family relationship with this relay but that can be reached by +following effective, mutual family relationships starting at this relay. +Omitted if empty or if descriptor containing this information cannot be +found. + +</li> + +<li> +consensus_weight_fraction +<code class="typeof">number</code> +optional + +Fraction of this relay's consensus weight compared to the sum of all +consensus weights in the network. +This fraction is a very rough approximation of the probability of this +relay to be selected by clients. +Omitted if the relay is not running. + +</li> + +<li> +guard_probability +<code class="typeof">number</code> +optional + +Probability of this relay to be selected for the guard position. +This probability is calculated based on consensus weights, relay flags, +and bandwidth weights in the consensus. +Path selection depends on more factors, so that this probability can only +be an approximation. +Omitted if the relay is not running, or the consensus does not contain +bandwidth weights. + +</li> + +<li> +middle_probability +<code class="typeof">number</code> +optional + +Probability of this relay to be selected for the middle position. +This probability is calculated based on consensus weights, relay flags, +and bandwidth weights in the consensus. +Path selection depends on more factors, so that this probability can only +be an approximation. +Omitted if the relay is not running, or the consensus does not contain +bandwidth weights. + +</li> + +<li> +exit_probability +<code class="typeof">number</code> +optional + +Probability of this relay to be selected for the exit position. +This probability is calculated based on consensus weights, relay flags, +and bandwidth weights in the consensus. +Path selection depends on more factors, so that this probability can only +be an approximation. +Omitted if the relay is not running, or the consensus does not contain +bandwidth weights. + +</li> + +<li> +measured +<code class="typeof">boolean</code> +optional + +Boolean field saying whether the consensus weight of this relay is based +on a threshold of 3 or more measurements by Tor bandwidth authorities. +Omitted if the network status consensus containing this relay does not +contain measurement information. + +</li> + +</ul> + +<h4>Bridge details objects</h4> + + +Bridge details objects contain the following key-value pairs: + + +<ul class="properties"> + +<li> +nickname +<code class="typeof">string</code> +optional + +Bridge nickname consisting of 1–19 +alphanumerical characters. +Omitted if the bridge nickname is "Unnamed". + +</li> + +<li> +hashed_fingerprint +<code class="typeof">string</code> +required + +SHA-1 hash of the bridge fingerprint +consisting of 40 upper-case hexadecimal characters. + +</li> + +<li> +or_addresses +<code class="typeof">array of strings</code> +required + +Array of sanitized IPv4 or IPv6 addresses and +TCP ports or port lists where the bridge accepts onion-routing +connections. +The first address is the primary onion-routing address that the bridge +used to register in the network, subsequent addresses are in arbitrary +order. +IPv6 hex characters are all lower-case. +Sanitized IP addresses are always in 10/8 or +[fd9f:2e19:3bcf/48] IP networks and are only useful to +learn which +IP version the bridge uses and to detect whether the bridge changed its +address. +Sanitized IP addresses always change on the 1st of every month at 00:00:00 +UTC, regardless of the bridge actually changing its IP address. +TCP ports are not sanitized. + +</li> + +<li> +last_seen +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when this +bridge was last seen in a bridge network status. + +</li> + +<li> +first_seen +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when this +bridge was first seen in a bridge network status. + +</li> + +<li> +running +<code class="typeof">boolean</code> +required + +Boolean field saying whether this bridge was listed +as running in the last bridge network status. + +</li> + +<li> +flags +<code class="typeof">array of strings</code> +optional + +Array of relay flags that the bridge authority +assigned to this bridge. +Omitted if empty. + +</li> + +<li> +last_restarted +<code class="typeof">string</code> +optional + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when the +bridge was last (re-)started. +Missing if router descriptor containing this information cannot be +found. + +</li> + +<li> +advertised_bandwidth +<code class="typeof">number</code> +optional + +Bandwidth in bytes per second that this +bridge is willing and capable to provide. +This bandwidth value is the minimum of bandwidth_rate, +bandwidth_burst, and observed_bandwidth. +Missing if router descriptor containing this information cannot be +found. + +</li> + +<li> +platform +<code class="typeof">string</code> +optional + +Platform string containing operating system and Tor +version details. +Omitted if not provided by the bridge or if descriptor containing this +information cannot be found. + +</li> + +<li> +transports +<code class="typeof">array of strings</code> +optional + +Array of (pluggable) transport names supported by this bridge. + +</li> + +</ul> + +</div>  + +<div class="box"> +<a name="history"></a> +<h3>History objects <a href="#history">#</a></h3> + + +History objects are no documents by themselves, but are contained in +subsequent documents. + + +<h4>Graph history objects</h4> + + +Graph history objects contain the following fields: + + +<ul class="properties"> + +<li> +first +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) of the first data point, or more +specifically the interval midpoint of the first interval. + +</li> + +<li> +last +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) of the last data point, or more +specifically the interval midpoint of the last interval. + +</li> + +<li> +interval +<code class="typeof">number</code> +required + +Time interval between two data points in seconds. + +</li> + +<li> +factor +<code class="typeof">number</code> +required + +Factor by which subsequent data values need to be multiplied to obtain +original values. +The idea is to reduce document size while still providing sufficient +detail for very different data scales. + +</li> + +<li> +count +<code class="typeof">number</code> +optional + +Number of provided data points, included mostly for debugging purposes. +Can also be derived from the number of elements in the subsequent array. + +</li> + +<li> +values +<code class="typeof">array of numbers</code> +required + +Array of normalized values between 0 and 999. +May contain null values. +Contains at least two subsequent non-null values to enable drawing of line +graphs. + +</li> + +</ul> + +</div>  + +<div class="box"> +<a name="bandwidth"></a> +<h3>Bandwidth documents <a href="#bandwidth">#</a> + +<a href="bandwidth?limit=4">example request</a> + +</h3> + + +Bandwidth documents contain aggregate statistics of a relay's or +bridge's consumed bandwidth for different time intervals. +Bandwidth documents are only updated when a relay or bridge publishes a +new server descriptor, which may take up to 18 hours during normal +operation. +Bandwidth documents contain the following fields: + + +<ul class="properties"> + +<li> +version +<code class="typeof">string</code> +required + +Onionoo protocol version string. + +</li> + +<li> +next_major_version_scheduled +<code class="typeof">string</code> +optional + +UTC date (YYYY-MM-DD) when the next major protocol version is scheduled to +be deployed. +Omitted if no major protocol changes are planned. + +</li> + +<li> +relays_published +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when +the last known relay network status consensus started being valid. +Indicates how recent the relay bandwidth documents in this document are. + +</li> + +<li> +relays +<code class="typeof">array of objects</code> +required + +Array of relay bandwidth objects as specified below. + +</li> + +<li> +bridges_published +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when +the last known bridge network status was published. +Indicates how recent the bridge bandwidth documents in this document are. + +</li> + +<li> +bridges +<code class="typeof">array of objects</code> +required + +Array of bridge bandwidth objects as specified below. + +</li> + +</ul> + +<h4>Relay bandwidth objects</h4> + + +Relay bandwidth objects contain the following key-value pairs: + + +<ul class="properties"> + +<li> +fingerprint +<code class="typeof">string</code> +required + +Relay fingerprint consisting of 40 upper-case +hexadecimal characters. + +</li> + +<li> +write_history +<code class="typeof">object</code> +optional + +Object containing graph history objects with written bytes for different +time periods. +Keys are string representation of the time period covered by the graph +history object. +Keys are fixed strings "3_days", +"1_week", "1_month", +"3_months", "1_year", and +"5_years". +Keys refer to the last known bandwidth history of a relay, not to the time +when the bandwidth document was published. +A graph history object is only contained if the time period it covers is +not already contained in another graph history object with shorter time +period and higher data resolution. +Similarly, a graph history object is excluded if the relay did not provide +bandwidth histories on the required level of detail. +The unit is bytes per second. +Contained graph history objects may contain null values if the relay did +not provide any bandwidth data or only data for less than 20% of a given +time period. + +</li> + +<li> +read_history +<code class="typeof">object</code> +optional + +Object containing graph history objects with read bytes for different time +periods. +The specification of graph history objects is similar to those in the +write_history field. + +</li> + +</ul> + +<h4>Bridge bandwidth objects</h4> + + +Bridge bandwidth objects contain the following key-value pairs: + + +<ul class="properties"> + +<li> +fingerprint +<code class="typeof">string</code> +required + +SHA-1 hash of the bridge fingerprint consisting +of 40 upper-case hexadecimal characters. + +</li> + +<li> +write_history +<code class="typeof">object</code> +optional + +Object containing graph history objects with written bytes for different +time periods. +The specification of graph history objects is similar to those in the +write_history field of relays. + +</li> + +<li> +read_history +<code class="typeof">object</code> +optional + +Object containing graph history objects with read bytes for different time +periods. +The specification of graph history objects is similar to those in the +write_history field of relays. + +</li> + +</ul> + +</div>  + +<div class="box"> +<a name="weights"></a> +<h3>Weights documents <a href="#weights">#</a> + +<a href="weights?limit=4">example request</a> + +</h3> + + +Weights documents contain aggregate statistics of a relay's probability +to be selected by clients for building paths. +Weights documents contain different time intervals and are available for +relays only. +Weights documents contain the following fields: + + +<ul class="properties"> + +<li> +version +<code class="typeof">string</code> +required + +Onionoo protocol version string. + +</li> + +<li> +next_major_version_scheduled +<code class="typeof">string</code> +optional + +UTC date (YYYY-MM-DD) when the next major protocol version is scheduled to +be deployed. +Omitted if no major protocol changes are planned. + +</li> + +<li> +relays_published +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when +the last known relay network status consensus started being valid. +Indicates how recent the relay weights documents in this document are. + +</li> + +<li> +relays +<code class="typeof">array of objects</code> +required + +Array of relay weights objects as specified below. + +</li> + +<li> +bridges_published +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when +the last known bridge network status was published. +Only included for compatibility reasons with the other document types. + +</li> + +<li> +bridges +<code class="typeof">array of objects</code> +required + +Empty array of objects that would represent bridge weights documents. +Only included for compatibility reasons with the other document types. + +</li> + +</ul> + +<h4>Relay weights objects</h4> + + +Relay weights objects contain the following key-value pairs: + + +<ul class="properties"> + +<li> +fingerprint +<code class="typeof">string</code> +required + +Relay fingerprint consisting of 40 upper-case +hexadecimal characters. + +</li> + +<li> +consensus_weight_fraction +<code class="typeof">object</code> +optional + +History object containing the +fraction of this relay's consensus weight compared to the sum of all +consensus weights in the network. +This fraction is a very rough approximation of the probability of this +relay to be selected by clients. +Keys are string representation of the time period covered by the graph +history object. +Keys are fixed strings "1_week", +"1_month", "3_months", +"1_year", and "5_years". +Keys refer to the last known weights history of a relay, not to the time +when the weights document was published. +A graph history object is only contained if the time period it covers is +not already contained in another graph history object with shorter time +period and higher data resolution. +The unit is path-selection probability. +Contained graph history objects may contain null values if the relay was +running less than 20% of a given time period. + +</li> + +<li> +guard_probability +<code class="typeof">object</code> +optional + +History object containing the probability +of this relay to be selected for the guard position. +This probability is calculated based on consensus weights, relay flags, +and bandwidth weights in the consensus. +Path selection depends on more factors, so that this probability can only +be an approximation. +The specification of this history object is similar to that in the +consensus_weight_fraction field above. + +</li> + +<li> +middle_probability +<code class="typeof">object</code> +optional + +History object containing the probability +of this relay to be selected for the middle position. +This probability is calculated based on consensus weights, relay flags, +and bandwidth weights in the consensus. +Path selection depends on more factors, so that this probability can only +be an approximation. +The specification of this history object is similar to that in the +consensus_weight_fraction field above. + +</li> + +<li> +exit_probability +<code class="typeof">object</code> +optional + +History object containing the probability +of this relay to be selected for the exit position. +This probability is calculated based on consensus weights, relay flags, +and bandwidth weights in the consensus. +Path selection depends on more factors, so that this probability can only +be an approximation. +The specification of this history object is similar to that in the +consensus_weight_fraction field above. + +</li> + +<li> +consensus_weight +<code class="typeof">object</code> +optional + +History object containing the absolute consensus weight of this relay. +The specification of this history object is similar to that in the +consensus_weight_fraction field above. + +</li> + +</ul> + +</div>  + +<div class="box"> +<a name="clients"></a> +<h3>Clients documents <a href="#clients">#</a> + +<a href="clients?limit=4">example request</a> + +</h3> + + +Clients documents contain estimates of the average number of clients +connecting to a bridge every day. +There are no clients documents available for relays, just for bridges. +Clients documents contain different time intervals and are available for +bridges only. +Clients documents contain the following fields: + + +<ul class="properties"> + +<li> +version +<code class="typeof">string</code> +required + +Onionoo protocol version string. + +</li> + +<li> +next_major_version_scheduled +<code class="typeof">string</code> +optional + +UTC date (YYYY-MM-DD) when the next major protocol version is scheduled to +be deployed. +Omitted if no major protocol changes are planned. + +</li> + +<li> +relays_published +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when +the last known relay network status consensus started being valid. +Only included for compatibility reasons with the other document types. + +</li> + +<li> +relays +<code class="typeof">array of objects</code> +required + +Empty array of objects that would represent relay clients documents. +Only included for compatibility reasons with the other document types. + +</li> + +<li> +bridges_published +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when +the last known bridge network status was published. +Indicates how recent the bridge clients documents in this document are. + +</li> + +<li> +bridges +<code class="typeof">array of objects</code> +required + +Array of bridge clients objects as specified below. + + +</li> + +</ul> + +<h4>Bridge clients objects</h4> + + +Bridge clients objects contain the following key-value pairs: + + +<ul class="properties"> + +<li> +fingerprint +<code class="typeof">string</code> +required + +SHA-1 hash of the bridge fingerprint consisting +of 40 upper-case hexadecimal characters. + +</li> + +<li> +average_clients +<code class="typeof">object</code> +optional + +Object containing graph history objects with the average number of clients +connecting to this bridge. +Keys are string representation of the time period covered by the graph +history object. +Keys are fixed strings "1_week", +"1_month", "3_months", +"1_year", and "5_years". +Keys refer to the last known clients history of a bridge, not to the time +when the clients document was published. +A graph history object is only contained if the time period it covers +is not already contained in another clients graph object with shorter +time period and higher data resolution. +The unit is number of clients. +Contained graph history objects may contain null values if the bridge did +not report client statistics for at least 50% of a given time period. +Each graph history object contains the following additional key-value +pairs: + + +<ul class="properties"> + +<li> +countries +<code class="typeof">object</code> +optional + +Object containing fractions of clients by country in the considered time +period. +Keys are two-letter lower-case country codes as found in a GeoIP database. +Values are numbers between 0 and 1 standing for the fraction of clients by +country. +A country is only included if at least 1% of clients came from this +country. +Omitted if the bridge did not report client statistics by country. +BETA: This field breaks compatibility +with the history objects contained in other documents pretty badly. +It might be removed in the future without notice. + +</li> + +<li> +transports +<code class="typeof">object</code> +optional + +Object containing fractions of clients by transport in the considered time +period. +Keys are transport names, or "<OR>" for the default +onion-routing transport protocol. +Values are numbers between 0 and 1 standing for the fraction of clients by +transport. +Omitted if the bridge did not report client statistics by transport. +BETA: This field breaks compatibility +with the history objects contained in other documents pretty badly. +It might be removed in the future without notice. + +</li> + +<li> +versions +<code class="typeof">object</code> +optional + +Object containing fractions of clients by IP version in the considered +time period. +Keys are either "v4" for IPv4 or "v6" +for IPv6. +Values are numbers between 0 and 1 standing for the fraction of clients by +version. +Omitted if the bridge did not report client statistics by IP version. +BETA: This field breaks compatibility +with the history objects contained in other documents pretty badly. +It might be removed in the future without notice. + +</li> + +</ul> + +</li> + +</ul> + +</div>  + +<div class="box"> +<a name="uptime"></a> +<h3>Uptime documents <a href="#uptime">#</a> + +<a href="uptime?limit=4">example request</a> + +</h3> + + +Uptime documents contain fractional uptimes of relays and bridges. +Uptime documents contain different time intervals and are available for +relays and bridges. +Uptime documents contain the following fields: + + +<ul class="properties"> + +<li> +version +<code class="typeof">string</code> +required + +Onionoo protocol version string. + +</li> + +<li> +next_major_version_scheduled +<code class="typeof">string</code> +optional + +UTC date (YYYY-MM-DD) when the next major protocol version is scheduled to +be deployed. +Omitted if no major protocol changes are planned. + +</li> + +<li> +relays_published +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when +the last known relay network status consensus started being valid. +Indicates how recent the relay uptime documents in this document are. + +</li> + +<li> +relays +<code class="typeof">array of objects</code> +required + +Array of relay uptime objects as specified below. + +</li> + +<li> +bridges_published +<code class="typeof">string</code> +required + +UTC timestamp (YYYY-MM-DD hh:mm:ss) when +the last known bridge network status was published. +Indicates how recent the bridge uptime documents in this document are. + +</li> + +<li> +bridges +<code class="typeof">array of objects</code> +required + +Array of bridge uptime objects as specified below. + +</li> + +</ul> + +<h4>Relay uptime objects</h4> + + +Relay uptime objects contain the following key-value pairs: + + +<ul class="properties"> + +<li> +fingerprint +<code class="typeof">string</code> +required + +Relay fingerprint consisting of 40 upper-case +hexadecimal characters. + +</li> + +<li> +uptime +<code class="typeof">object</code> +optional + +Object containing graph history objects with the fractional uptime of this +relay. +Keys are string representation of the time period covered by the graph +history object. +Keys are fixed strings "1_week", +"1_month", "3_months", +"1_year", and "5_years". +Keys refer to the last known uptime history of a relay, not to the time +when the uptime document was published. +A graph history object is only contained if the time period it covers is +not already contained in another graph history object with shorter time +period and higher data resolution. +The unit is fractional uptime from 0 to 1. +Contained graph history objects may contain null values if less than 20% +of network statuses have been processed for a given time period. + +</li> + +<li> +flags +<code class="typeof">object</code> +optional + +Object containing fractional times of this relay having relay flags +assigned. +Keys are flag names like "Running" or +"Exit", values are objects similar to the +uptime field above, again with keys like +"1_week" etc. +If a relay never had a given relay flag assigned, no object is included +for that flag. + +</li> + +</ul> + +<h4>Bridge uptime objects</h4> + + +Bridge uptime objects contain the following key-value pairs: + + +<ul class="properties"> + +<li> +fingerprint +<code class="typeof">string</code> +required + +SHA-1 hash of the bridge fingerprint consisting +of 40 upper-case hexadecimal characters. + +</li> + +<li> +uptime +<code class="typeof">object</code> +optional + +Object containing uptime history objects for different time periods. +The specification of uptime history objects is similar to those in the +uptime field of relays. + +</li> + +</ul> + +</div>  + +<div class="box"> +<a name="examples"></a> +<h3>Example usage <a href="#examples">#</a> +</h3> + + +The following examples illustrate how to build requests for some trivial +and some more complex use cases. +While Onionoo is designed mainly for developers and not end users, there +may be cases when it's easier to quickly write a specific query Onionoo +rather than to find an Onionoo client that provides the desired +information. + + +<pre>https://onionoo.torproject.org/summary?limit=4</pre> + + +This first query returns the first four summary documents that Onionoo can +find. +The <code>limit</code> parameter should always be used while developing +new queries to avoid downloading huge responses. + + +<pre>https://onionoo.torproject.org/summary?limit=4&search=moria</pre> + + +The second query restricts results to relays and bridges containing the +string "moria" in one of a few searched fields. + + +<pre>https://onionoo.torproject.org/details?limit=4&search=moria</pre> + + +The third query switches from the short summary documents to the longer +details documents containing, well, more details. + + +<pre>https://onionoo.torproject.org/details?limit=4&search=moria&fields=nickname</pre> + + +The fourth query adds the <code>fields</code> parameter which removes all +fields except the specified ones from the result. +This parameter is only implemented for details documents. + + +<pre>https://onionoo.torproject.org/details?limit=4&search=moria&fields=nickname…</pre> + + +The fifth query sorts results by relay consensus weight from largest to +smallest. + + + +Obviously, this query can be made even more complex by adding more +parameters, and in some cases this is necessary and useful. +Please refer to the protocol specification for details. + + +</div>  + +</body> +</html> + diff --git a/src/main/resources/web/robots.txt b/src/main/resources/web/robots.txt new file mode 100644 index 0000000..9b19f8f --- /dev/null +++ b/src/main/resources/web/robots.txt @@ -0,0 +1,8 @@ +User-agent: * +Disallow: /summary +Disallow: /details +Disallow: /bandwidth +Disallow: /weights +Disallow: /clients +Disallow: /uptime + diff --git a/web/css/style.css b/web/css/style.css deleted file mode 100644 index e3caa4a..0000000 --- a/web/css/style.css +++ /dev/null @@ -1,36 +0,0 @@ -body { font-family: "Open Sans","lucida grande","Segoe UI",arial,verdana, - "lucida sans unicode",tahoma,sans-serif; background: #fafafa; - font-size: 13px; line-height: 22px; color: #222; } -h3 { color: #7D4698; position: relative } -a { color: #7D4698; text-decoration: none; font-weight: bold; } -p { margin: 0; padding: 10px; } -a[name] { padding: 0; margin: 0; } -.box { max-width: 850px; width: 100%; margin: 0 auto 30px auto; - padding-bottom: 30px; background: white; border: 1px solid #eee; } -.box > * { margin-left: 30px; margin-right: 30px; } -.box h3 a { visibility: hidden; } -.box:hover h3 a { visibility: visible; } -.api-request { border-bottom: 1px solid #eee; position: relative } -.request-url, .request-type, .request-response { padding: 8px 10px; - vertical-align: middle } -.request-type { color: #57145F; display: inline-block; } -.request-url { color: #333; font-size: 18px; } -.request-response { position: absolute; color: #666; right: 0; } -h3 .request-response { padding: 0 !important; } -.api-urls>li:last-child { border-bottom: 0; } -.required-true, .required-false, .typeof { display: inline-block; - vertical-align: middle; padding: 5px 10px; } -.required-true { color: #1d7508; } -.required-false { color: #aaa; } -.properties { margin-top: 10px; margin-bottom: 10px; - border: 1px solid #eee; } -.properties li { padding: 5px 0; } -.properties li ul { border: 1px solid #eee; margin: 10px 10px 10px 40px; - background: white; } -.properties .properties { margin-left: 10px; } -.properties li:nth-child(even) { background: #fafafa; } -.properties p { padding: 10px 15px; } -.properties b { padding: 5px 10px; display: inline-block; - vertical-align: middle; } -.api-urls{ margin-top: 30px; margin-bottom: 30px; } - diff --git a/web/favicon.ico b/web/favicon.ico deleted file mode 100644 index 48060b1..0000000 Binary files a/web/favicon.ico and /dev/null differ diff --git a/web/index.html b/web/index.html deleted file mode 100644 index c7095ec..0000000 --- a/web/index.html +++ /dev/null @@ -1,141 +0,0 @@ -<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> -<html> -<head> -<title>Onionoo — a Tor network status protocol</title> -<link href="css/style.css" type="text/css" rel="stylesheet"> -<meta http-equiv="content-type" content="text/html; charset=ISO-8859-1"> -<link href="favicon.ico" type="image/x-icon" rel="shortcut icon"> -</head> -<body> - -<div class="box"> - -<h1><a href="index.html">Onionoo</a> —</h1> -<h2>a Tor network status <a href="protocol.html">protocol</a></h2> - -Onionoo is a web-based protocol to learn about currently running -Tor relays and bridges. Onionoo itself was not designed as a service -for human beings—at least not directly. Onionoo provides the -data for other applications and websites which in turn present Tor -network status information to humans. The following Onionoo clients -are currently available: -<ul> -<li><a href="https://atlas.torproject.org/">Atlas</a> is a web -application to discover Tor relays. It provides useful -information on how relays are configured along with graphics about -their past.</li> -<li><a href="https://compass.torproject.org/">Compass</a> -is a Python script and website that extracts consensus weight -information of currently running relays and aggregates weights of -relays running in the same country or same autonomous system.</li> -<li><a href="http://tor2web.org/">Tor2web</a> is a web proxy to Tor -Hidden Services. It uses Onionoo to get the list of currently running -Tor Exits to detect if the client is a Tor user and if so redirect -them to the .onion address.</li> -<li>The <a href="https://nos-oignons.net/Services/index.en.html">Nos -oignons</a> website uses Onionoo to visualize bandwidth histories of -their relays.</li> -<li>The <a href="https://metrics.torproject.org/bubbles.html">metrics -website</a> visualizes diversity of the Tor network using bubble -graphs.</li> -<li><a href="https://github.com/woeisme/torchart">torchart</a> is a PHP -WordPress plugin which draws 3 day traffic charts using pChart currently -used for displaying metric charts on -<a href="http://icetor.is/wordpress/metrics/">icetor.is</a>.</li> -<li><a href="https://github.com/kloesing/challenger">challenger</a> -aggregates graph data from relays participating in -<a href="https://www.eff.org/torchallenge/">EFF's 2014 Tor Challenge</a> -to provide statistics like total bytes transferred.</li> -<li><a href="https://play.google.com/store/apps/details?id=com.networksaremadeofstring.a…">AnOnionooid</a> -is an Android app that helps find and explore Tor relays and bridges.</li> -<li><a href="https://oniontip.com/">OnionTip</a> uses Onionoo's data to -distribute donations to relays that have a bitcoin address in their -contact line.</li> -<li><a href="https://savannah.nongnu.org/projects/koninoo/">koninoo</a> is -a simple Java command line interface for querying Onionoo data.</li> -<li>The <a href="https://duckduckgo.com/">DuckDuckGo</a> search engine -displays Tor node details when being asked for "tor node" followed by a -search term.</li> -<li><a href="https://onionview.com/">OnionView</a> is a simple web service -which uses Tor relay data to plot the location of active Tor nodes onto an -interactive map of the world.</li> -</ul> - -The following library facilitates development of Onionoo clients: -<ul> -<li><a href="https://github.com/duk3luk3/onion-py">OnionPy</a> is a -comprehensive pure-Python (2.7+) Onionoo wrapper with caching -support.</li> -</ul> - -<h2>Developing Onionoo applications</h2> - -The project pages of the Onionoo clients listed above have further -information for contacting the authors and contributing ideas or code. -The authors will be happy to hear your thoughts! - -You don't find your favorite Onionoo client above? Want to -implement your own and tell us to add it to the list? The Onionoo -clients above are backed by a web-based -<a href="protocol.html">protocol</a>, which -facilitates developing new applications displaying Tor status -information. Here are a few ideas for new Onionoo clients: -<ul> -<li>Tor controller extension: Extend -<a href="https://www.torproject.org/projects/vidalia.html.en">Vidalia</a> and/or -<a href="https://www.torproject.org/projects/arm.html.en">arm</a> to look up details for the bridge -that the user is running and display what pool the bridge is contained -in.</li> -<li>Social network site plugin: Add a plugin to the social network -site of your choice to show your friends what Tor relays and bridges -you're running and how that helps users around the world.</li> -<li>Desktop tray icon: Write a tray icon for your favorite desktop -environment that tells you when your relay or bridge is down and that -displays some basic usage statistics.</li> -<li>E-mail notification service: Improve our e-mail notification -service <a href="https://weather.torproject.org/">Weather</a> by -implementing its own relay search or extending it to report when a -bridge drops off the network.</li> -<li>Command-line tool: Implement a command-line tool that quickly -searches a relay or bridge and prints out some status information to -help debug problems.</li> -<li>(Insert your idea here.)</li> -</ul> - -Want to help with developing the Onionoo server that provides -Tor status data, or want to run your own Onionoo server instance? The -Onionoo server is written in Java with a tiny portion of Java -Servlets. Instructions for setting up the Onionoo server to fetch the -required data from the Tor servers is described in the INSTALL file in -the sources. For more details see the -<a href="https://gitweb.torproject.org/onionoo.git">source code</a> and -<a href="https://trac.torproject.org/projects/tor/query?status=!closed&component=Oni…">issue -tracker</a>. - -<h2>Related projects</h2> - -TorStatus is the name of a nowadays -<a href="https://svn.torproject.org/svn/torstatus/trunk/">unmaintained</a> -website that displays Tor relay information similar to -<a href="http://atlas.torproject.org/">Atlas</a>. There are still a -few <a href="http://torstatus.blutmagie.de/">TorStatus</a> -<a href="https://torstatus.rueckgr.at/">websites</a> -<a href="http://tns.hermetix.org/">running</a>. - -There's another project from summer 2011 called TorStatus which is -a <a href="https://gitweb.torproject.org/torstatus.git">rewrite</a> of -the original TorStatus in Python/Django. Unfortunately, it's also -unmaintained. - -Finally, there's the -<a href="https://consensus-health.torproject.org/">consensus-health -page</a> which has the primary purpose of indicating problems with -creating a network status consensus. As a side-effect this page lists -all currently running relays and how the directory authorities voted -on them. - -</div>  - -</body> -</html> - diff --git a/web/protocol.html b/web/protocol.html deleted file mode 100644 index c6cc534..0000000 --- a/web/protocol.html +++ /dev/null @@ -1,2338 +0,0 @@ -<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> -<html> -<head> -<title>Onionoo — protocol overview</title> -<link href="css/style.css" type="text/css" rel="stylesheet"> -<meta http-equiv="content-type" content="text/html; charset=ISO-8859-1"> -<link href="favicon.ico" type="image/x-icon" rel="shortcut icon"> -</head> -<body> - -<div class="box"> - -<h1><a href="index.html">Onionoo</a> —</h1> -<h2>protocol overview</h2> - - <h4>Table of contents</h4> - <ul> - <li><a href="#general">General</a></li> - <li><a href="#methods">Methods</a></li> - <li><a href="#summary">Summary documents</a></li> - <li><a href="#details">Details documents</a></li> - <li><a href="#history">History objects</a></li> - <li><a href="#bandwidth">Bandwidth documents</a></li> - <li><a href="#weights">Weights documents</a></li> - <li><a href="#clients">Clients documents</a></li> - <li><a href="#uptime">Uptime documents</a></li> - <li><a href="#examples">Example usage</a></li> - </ul> - -</div> -<div class="box"> - -<a name="general"></a> -<h3>General <a href="#general">#</a></h3> - - -The Onionoo service is designed as a RESTful web service. -Onionoo clients send HTTP GET requests to the Onionoo server which -responds with JSON-formatted replies. -Onionoo clients and their developers should follow a few rules: - - -<h4>Compression</h4> - -Clients should include an "Accept-Encoding: -gzip" header in their requests and handle gzip-compressed -responses. -Only requests starting at a certain size will be compressed by the -server. - - -<h4>Caching</h4> -Clients should make use of the "Last-Modified" -header of responses and include that timestamp in a -"If-Modified-Since" header of subsequent requests. - - -<h4>Response codes</h4> - -Clients should handle response codes by -distinguishing between client and server errors, and if there's a problem, -informing their users about the kind of problem. -The following response codes are used: - - -<ul class="properties"> - -<li> -200 OK - -The request was processed successfully. - -</li> - -<li> -304 Not Modified - -Server data has not changed since the -"If-Modified-Since" header included in the request. - -</li> - -<li> -400 Bad Request - -The request for a known resource could not be processed because of bad -syntax. -This is most likely a client problem. - -</li> - -<li> -404 Not Available - -The request could not be processed because the requested resource could -not be found. -This is most likely a client problem. - -</li> - -<li> -500 Internal Server Error - -There is an unspecific problem with -the server which the service operator may not yet be aware of. -Please check if there's already a bug report for this problem, and if not, -file one. - -</li> - -<li> -503 Service Unavailable - -The server is temporarily down for -maintenance, or there is a temporary problem with the server that the -service operator is already aware of. -Only file a bug report if this problem persists. - -</li> - -</ul> - -<h4>Protocol versions</h4> - -There are plenty of reasons why we may have -to change the protocol described here. -Clients should be able to handle all valid JSON responses, ignoring -unknown fields and not breaking horribly in case of missing fields. -Protocol changes will be announced here by writing deprecated API parts in -red and new parts in -blue. -Deprecated parts will be removed one month after their announcement. -If you want to be informed of upcoming protocol changes, subscribe to the -<a href="https://lists.torproject.org/cgi-bin/mailman/listinfo/onionoo-announce">onionoo-announce</a> -mailing list. - - -All responses contain a "version" string that -indicates whether clients can parse the document or not. -Version strings consist of a major and a minor version number. -The major version number is raised when previously required fields are -dropped or turned into optional fields, when request parameters or -response documents are removed, or when there are structural changes. -The minor version number is raised when new fields, request parameters, or -response documents are added or optional fields are dropped. -If clients support the same major version given in a response but only a -lower minor version, they should be able to parse the document but may not -understand all fields. -If clients support a lower major version, they should not attempt to parse -a document, because there may be backward-incompatible changes. - -Responses may also contain the optional -"next_major_version_scheduled" field that announces when -the next major version is scheduled to be deployed. -Clients should inform their users that they should upgrade to the next -major protocol version before the provided date. - -The following versions have been used in the past six months or are -scheduled to be deployed in the next months: - -<ul> -<li>1.0: First assigned version number on August 31, -2014.</li> -<li>1.1: Added optional "next_major_version_scheduled" -field on September 16, 2014.</li> -<li>1.2: Added qualified search terms to "search" -parameter on October 17, 2014.</li> -<li>2.0: Extended search parameter to base64-encoded -fingerprints on November 15, 2014.</li> -<li>2.1: Removed optional "advertised_bandwidth_fraction" -field from details documents and optional "advertised_bandwidth" and -"advertised_bandwidth_fraction" fields from weights documents on November -16, 2014.</li> -<li>2.2: Removed optional "pool_assignment" field and -added "transports" field to bridge details documents on December 8, -2014.</li> -<li>2.3: Added optional "flags" field to uptime -documents on March 22, 2015.</li> -<li>2.4: Added optional "effective_family" field to -details documents on July 3, 2015.</li> -<li>2.5: Added optional "measured" field to details -documents on August 13, 2015.</li> -<li>2.6: Added optional "alleged_family" and -"indirect_family" fields and deprecated optional "family" field in details -documents on August 25, 2015.</li> -<li>3.0: Extended search parameter to match any 4 hex -characters of a space-separated fingerprint on November 15, 2015.</li> -<li>3.1: Removed optional "family" field on January 18, -2016.</li> -</ul> - -</div>  - -<div class="box"> - -<a name="methods"></a> -<h3>Methods <a href="#methods">#</a></h3> - - -The following methods each return a single document containing zero or -more objects of relays and/or bridges. -By default, all relays and bridges are included that have been running in -the past week. - - -<ul class="api-urls"> - -<li class="api-request"> -GET -https://onionoo.torproject.org/summary -returns a <a href="#summary">summary -document</a> -</li> - -<li class="api-request"> -GET -https://onionoo.torproject.org/details -returns a <a href="#details">details -document</a> -</li> - -<li class="api-request"> -GET -https://onionoo.torproject.org/bandwidth -returns a <a href="#bandwidth">bandwidth -document</a> -</li> - -<li class="api-request"> -GET -https://onionoo.torproject.org/weights -returns a <a href="#weights">weights -document</a> -</li> - -<li class="api-request"> -GET -https://onionoo.torproject.org/clients -returns a <a href="#clients">clients -document</a> -</li> - -<li class="api-request"> -GET -https://onionoo.torproject.org/uptime -returns a <a href="#uptime">uptime -document</a> -</li> - -</ul> - -<h4>Parameters</h4> - -Each of the methods can be parameterized to select only a subset of relay -and/or bridge documents that are currently running or that have been -running in the past week. -(The fingerprint parameter is special here, because it -allows selecting a specific relay or bridge, regardless of whether it has -been running in the past week.) -If multiple parameters are specified, they are combined using a logical -AND operation, meaning that only the intersection of relays and bridges -matching all parameters is returned. -If the same parameter is specified more than once, only the first -parameter value is considered. - - -<ul class="properties"> - -<li> -type - -Return only relay (parameter value -relay) or only bridge documents (parameter value -bridge). -Parameter values are case-insensitive. - -</li> - -<li> -running - -Return only running (parameter value -true) or only non-running relays and/or bridges -(paramter value -false). -Parameter values are case-insensitive. - -</li> - -<li> -search - -Return only (1) relays with the parameter value matching (part of a) -nickname, (possibly $-prefixed) beginning of a hex-encoded fingerprint, -any 4 hex characters of a space-separated fingerprint, beginning of a -base64-encoded fingerprint without trailing equal signs, or beginning of -an IP address, (2) bridges with (part of a) nickname or (possibly -$-prefixed) beginning of a hashed hex-encoded fingerprint, and (3) relays -and/or bridges matching a given qualified search term. -Searches by relay IP address include all known addresses used for onion -routing and for exiting to the Internet. -Searches for beginnings of IP addresses are performed on textual -representations of canonical IP address forms, so that searches using CIDR -notation or non-canonical forms will return empty results. -Searches are case-insensitive, except for base64-encoded fingerprints. -If multiple search terms are given, separated by spaces, the intersection -of all relays and bridges matching all search terms will be returned. -Complete hex-encoded fingerprints should always be hashed using SHA-1, -regardless of searching for a relay or a bridge, in order to not -accidentally leak non-hashed bridge fingerprints in the URL. -Qualified search terms have the form "key:value" (without double quotes) -with "key" being one of the parameters listed here except for "search", -"fingerprint", "order", "limit", "offset", and "fields", and "value" being -the string that will internally be passed to that parameter. - -</li> - -<li> -lookup - -Return only the relay with the parameter -value matching the fingerprint or the bridge with the parameter value -matching the hashed fingerprint. -Fingerprints should always be hashed using SHA-1, regardless of looking up -a relay or a bridge, in order to not accidentally leak non-hashed bridge -fingerprints in the URL. -Lookups only work for full fingerprints or hashed fingerprints consisting -of 40 hex characters. -Lookups are case-insensitive. - -</li> - -<li> -fingerprint - -Return only the relay with the parameter value matching the fingerprint -or the bridge with the parameter value matching the hashed fingerprint. -Fingerprints must consist of 40 hex characters, case does not matter. -This parameter is quite similar to the lookup parameter -with two exceptions: -(1) the provided relay fingerprint or hashed bridge fingerprint must -not be hashed (again) using SHA-1; -(2) the response will contain any matching relay or bridge regardless of -whether they have been running in the past week. - -</li> - -<li> -country - -Return only relays which are located in the -given country as identified by a two-letter country code. -Filtering by country code is case-insensitive. - -</li> - -<li> -as - -Return only relays which are located in the -given autonomous system (AS) as identified by the AS number (with or -without preceding "AS" part). -Filtering by AS number is case-insensitive. - -</li> - -<li> -flag - -Return only relays which have the -given relay flag assigned by the directory authorities. -Note that if the flag parameter is specified more than once, only the -first parameter value will be considered. -Filtering by flag is case-insensitive. - -</li> - -<li> -first_seen_days - -Return only relays or bridges which -have first been seen during the given range of days ago. -A parameter value "x-y" with x <= y returns relays or bridges that have -first been seen at least x and at most y days ago. -Accepted short forms are "x", "x-", and "-y" which are interpreted as -"x-x", "x-infinity", and "0-y". - -</li> - -<li> -last_seen_days - -Return only relays or bridges which -have last been seen during the given range of days ago. -A parameter value "x-y" with x <= y returns relays or bridges that have -last been seen at least x and at most y days ago. -Accepted short forms are "x", "x-", and "-y" which are interpreted as -"x-x", "x-infinity", and "0-y". -Note that relays and bridges that haven't been running in the past week -are not included in results, so that setting x to 8 or higher will lead to -an empty result set. - -</li> - -<li> -contact - -Return only relays with the parameter value -matching (part of) the contact line. -If the parameter value contains spaces, only relays are returned which -contain all space-separated parts in their contact line. -Only printable ASCII characters are permitted in the parameter value, -some of which need to be percent-encoded (# as %23, % as %25, & as -%26, + as %2B, and / as %2F). -Comparisons are case-insensitive. - -</li> - -<li> -family - -Return only the relay whose fingerprint matches the parameter value and -all relays that this relay has listed in its family by fingerprint and -that in turn have listed this relay in their family by fingerprint. -If relays have listed other relays in their family by nickname, those -family relationships will not be considered, regardless of whether they -have the Named flag or not. -The provided relay fingerprint must consist of 40 hex characters where -case does not matter, and it must not be hashed using SHA-1. -Bridges are not contained in the result, regardless of whether they define -a family. - -</li> - -</ul> - - -Response documents can be reduced in size by requesting only a subset -of contained fields. - - -<ul class="properties"> - -<li> -fields - -Comma-separated list of fields that will be -included in the result. -So far, only top-level fields in relay or bridge objects of details -documents can be specified, e.g., -nickname,hashed_fingerprint. -If the fields parameter is provided, all other fields which are not -contained in the provided list will be removed from the result. -Field names are case-insensitive. - -</li> - -</ul> - - -Relay and/or bridge documents in the response can be ordered and -limited by providing further parameters. -If the same parameter is specified more than once, only the first -parameter value is considered. - - -<ul class="properties"> - -<li> -order - -Re-order results by a comma-separated list -of fields in ascending or descending order. -Results are first ordered by the first list element, then by the second, -and so on. -Possible fields for ordering are: consensus_weight. -Field names are case-insensitive. -Ascending order is the default; descending order is selected by prepending -fields with a minus sign (-). -Relays or bridges which don't have any value for a field to be ordered by -are always appended to the end, regardless or sorting order. -The ordering is defined independent of the requested document type and -does not require the ordering field to be contained in the document. -If no order parameter is given, ordering of results is -undefined. - -</li> - -<li> -offset - -Skip the given number of relays and/or -bridges. -Relays are skipped first, then bridges. -Non-positive offset values are treated as zero and don't -change the -result. - -</li> - -<li> -limit - -Limit result to the given number of -relays and/or bridges. -Relays are kept first, then bridges. -Non-positive limit values are treated as zero and lead -to an empty -result. -When used together with offset, the offsetting step -precedes the -limiting step. - -</li> - -</ul> - -</div>  - -<div class="box"> -<a name="summary"></a> -<h3>Summary documents <a href="#summary">#</a> - -<a href="summary?limit=4">example request</a> - -</h3> - -Summary documents contain short summaries of relays with nicknames, -fingerprints, IP addresses, and running information as well as bridges -with hashed fingerprints and running information. -Summary documents contain the following fields: - -<ul class="properties"> - -<li> -version -<code class="typeof">string</code> -required - -Onionoo protocol version string. - -</li> - -<li> -next_major_version_scheduled -<code class="typeof">string</code> -optional - -UTC date (YYYY-MM-DD) when the next major protocol version is scheduled to -be deployed. -Omitted if no major protocol changes are planned. - -</li> - -<li> -relays_published -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when the last known relay network -status consensus started being valid. -Indicates how recent the relay summaries in this document are. - -</li> - -<li> -relays -<code class="typeof">array of objects</code> -required - -Array of relay summary objects as specified below. - -</li> - -<li> -bridges_published -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when -the last known bridge network status was published. -Indicates how recent the bridge summaries in this document are. - -</li> - -<li> -bridges -<code class="typeof">array of objects</code> -required - -Array of bridge summary objects as specified below. - -</li> - -</ul> - -<h4>Relay summary objects</h4> - - -Relay summary objects contain the following key-value pairs: - - -<ul class="properties"> - -<li> -n -<code class="typeof">string</code> -optional - -Relay nickname consisting of 1–19 alphanumerical characters. -Omitted if the relay nickname is "Unnamed". - -</li> - -<li> -f -<code class="typeof">string</code> -required - -Relay fingerprint consisting of 40 upper-case hexadecimal characters. - -</li> - -<li> -a -<code class="typeof">array of strings</code> -required - -Array of IPv4 or IPv6 addresses where the relay accepts -onion-routing connections or which the relay used to exit to the Internet -in the past 24 hours. -The first address is the primary onion-routing address that the relay used -to register in the network, subsequent addresses are in arbitrary order. -IPv6 hex characters are all lower-case. - -</li> - -<li> -r -<code class="typeof">boolean</code> -required - -Boolean field saying whether this relay was listed as -running in the last relay network status consensus. - -</li> - -</ul> - -<h4>Bridge summary objects</h4> - - -Bridge summary objects contain the following key-value pairs: - - -<ul class="properties"> - -<li> -n -<code class="typeof">string</code> -optional - -Bridge nickname consisting of 1–19 alphanumerical characters. -Omitted if the bridge nickname is "Unnamed". - -</li> - -<li> -h -<code class="typeof">string</code> -required - -SHA-1 hash of the bridge fingerprint consisting of 40 -upper-case hexadecimal characters. - -</li> - -<li> -r -<code class="typeof">boolean</code> -required - -Boolean field saying whether this bridge was listed as -running in the last bridge network status. - -</li> - -</ul> - -</div>  - -<div class="box"> -<a name="details"></a> -<h3>Details documents <a href="#details">#</a> - -<a href="details?limit=4">example request</a> - -</h3> - - -Details documents are based on network statuses published by the Tor -directories, server descriptors published by relays and bridges, and data -published by Tor network services TorDNSEL and BridgeDB. -Details documents use the most recently published data from these sources, -which may lead to contradictions between fields based on different sources -in rare edge cases. -Details documents contain the following fields: - - -<ul class="properties"> - -<li> -version -<code class="typeof">string</code> -required - -Onionoo protocol version string. - -</li> - -<li> -next_major_version_scheduled -<code class="typeof">string</code> -optional - -UTC date (YYYY-MM-DD) when the next major protocol version is scheduled to -be deployed. -Omitted if no major protocol changes are planned. - -</li> - -<li> -relays_published -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when -the last known relay network status consensus started being valid. -Indicates how recent the relay details in this document are. - -</li> - -<li> -relays -<code class="typeof">array of objects</code> -required - -Array of relay details objects as specified below. - -</li> - -<li> -bridges_published -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when -the last known bridge network status was published. -Indicates how recent the bridge details in this document are. - -</li> - -<li> -bridges -<code class="typeof">array of objects</code> -required - -Array of bridge details objects as specified below. - - -</li> - -</ul> - -<h4>Relay details objects</h4> - - -Relay details objects contain the following key-value pairs: - - -<ul class="properties"> - -<li> -nickname -<code class="typeof">string</code> -optional - -Relay nickname consisting of 1–19 alphanumerical characters. -Omitted if the relay nickname is "Unnamed". - -</li> - -<li> -fingerprint -<code class="typeof">string</code> -required - -Relay fingerprint consisting of 40 upper-case -hexadecimal characters. - -</li> - -<li> -or_addresses -<code class="typeof">array of strings</code> -required - -Array of IPv4 or IPv6 addresses and TCP ports -or port lists where the relay accepts onion-routing connections. -The first address is the primary onion-routing address that the relay used -to register in the network, subsequent addresses are in arbitrary order. -IPv6 hex characters are all lower-case. - -</li> - -<li> -exit_addresses -<code class="typeof">array of strings</code> -optional - -Array of IPv4 or IPv6 addresses that the -relay used to exit to the Internet in the past 24 hours. -IPv6 hex characters are all lower-case. -Only those addresses are listed that are different from onion-routing -addresses. -Omitted if array is empty. - -</li> - -<li> -dir_address -<code class="typeof">string</code> -optional - -IPv4 address and TCP port where the relay -accepts directory connections. -Omitted if the relay does not accept directory connections. - -</li> - -<li> -last_seen -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when this -relay was last seen in a network status consensus. - -</li> - -<li> -last_changed_address_or_port -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD -hh:mm:ss) when this relay last stopped announcing an IPv4 or IPv6 address -or TCP port where it previously accepted onion-routing or directory -connections. -This timestamp can serve as indicator whether this relay would be a -suitable fallback directory. - -</li> - -<li> -first_seen -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when this -relay was first seen in a network status consensus. - -</li> - -<li> -running -<code class="typeof">boolean</code> -required - -Boolean field saying whether this relay was listed as -running in the last relay network status consensus. - -</li> - -<li> -hibernating -<code class="typeof">boolean</code> -optional - -Boolean field saying whether this relay indicated that it is hibernating -in its last known server descriptor. -This information may be helpful to decide whether a relay that is not -running anymore has reached its accounting limit and has not dropped out -of the network for another, unknown reason. -Omitted if either the relay is not hibernating, or if no information is -available about the hiberation status of the relay. - -</li> - -<li> -flags -<code class="typeof">array of strings</code> -optional - -Array of relay flags that the directory authorities -assigned to this relay. -Omitted if empty. - -</li> - -<li> -country -<code class="typeof">string</code> -optional - -Two-letter lower-case country code as found in a -GeoIP database by resolving the relay's first onion-routing IP address. -Omitted if the relay IP address could not be found in the GeoIP -database. - -</li> - -<li> -country_name -<code class="typeof">string</code> -optional - -Country name as found in a GeoIP database by -resolving the relay's first onion-routing IP address. -Omitted if the relay IP address could not be found in the GeoIP -database, or if the GeoIP database did not contain a country name. - -</li> - -<li> -region_name -<code class="typeof">string</code> -optional - -Region name as found in a GeoIP database by -resolving the relay's first onion-routing IP address. -Omitted if the relay IP address could not be found in the GeoIP -database, or if the GeoIP database did not contain a region name. - -</li> - -<li> -city_name -<code class="typeof">string</code> -optional - -City name as found in a -GeoIP database by resolving the relay's first onion-routing IP address. -Omitted if the relay IP address could not be found in the GeoIP -database, or if the GeoIP database did not contain a city name. - -</li> - -<li> -latitude -<code class="typeof">number</code> -optional - -Latitude as found in a GeoIP database by resolving -the relay's first onion-routing IP address. -Omitted if the relay IP address could not be found in the GeoIP -database. - -</li> - -<li> -longitude -<code class="typeof">number</code> -optional - -Longitude as found in a GeoIP database by -resolving the relay's first onion-routing IP address. -Omitted if the relay IP address could not be found in the GeoIP -database. - -</li> - -<li> -as_number -<code class="typeof">string</code> -optional - -AS number as found in an AS database by -resolving the relay's first onion-routing IP address. -AS number strings start with "AS", followed directly by the AS number. -Omitted if the relay IP address could not be found in the AS -database. - -</li> - -<li> -as_name -<code class="typeof">string</code> -optional - -AS name as found in an AS database by resolving the -relay's first onion-routing IP address. -Omitted if the relay IP address could not be found in the AS -database. - -</li> - -<li> -consensus_weight -<code class="typeof">number</code> -required - -Weight assigned to this relay by the -directory authorities that clients use in their path selection algorithm. -The unit is arbitrary; currently it's kilobytes per second, but that might -change in the future. - -</li> - -<li> -host_name -<code class="typeof">string</code> -optional - -Host name as found in a reverse DNS lookup of the -relay IP address. -This field is updated at most once in 12 hours, unless the relay IP -address changes. -Omitted if the relay IP address was not looked up or if no lookup request -was successful yet. - -</li> - -<li> -last_restarted -<code class="typeof">string</code> -optional - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when the -relay was last (re-)started. -Missing if router descriptor containing this information cannot be -found. - -</li> - -<li> -bandwidth_rate -<code class="typeof">number</code> -optional - -Average bandwidth -in bytes per second that this relay is willing to sustain over long -periods. -Missing if router descriptor containing this information cannot be -found. - -</li> - -<li> -bandwidth_burst -<code class="typeof">number</code> -optional - -Bandwidth in bytes -per second that this relay is willing to sustain in very short intervals. -Missing if router descriptor containing this information cannot be -found. - -</li> - -<li> -observed_bandwidth -<code class="typeof">number</code> -optional - -Bandwidth -estimate in bytes per second of the capacity this relay can handle. -The relay remembers the maximum bandwidth sustained output over any ten -second period in the past day, and another sustained input. -The "observed_bandwidth" value is the lesser of these two numbers. -Missing if router descriptor containing this information cannot be -found. - -</li> - -<li> -advertised_bandwidth -<code class="typeof">number</code> -optional - -Bandwidth in bytes per second that this -relay is willing and capable to provide. -This bandwidth value is the minimum of bandwidth_rate, -bandwidth_burst, and observed_bandwidth. -Missing if router descriptor containing this information cannot be -found. - -</li> - -<li> -exit_policy -<code class="typeof">array of strings</code> -optional - -Array of exit-policy lines. -Missing if router descriptor containing this information cannot be -found. -May contradict the "exit_policy_summary" field in a rare -edge case: this happens when the relay changes its exit policy after the -directory authorities summarized the previous exit policy. - -</li> - -<li> -exit_policy_summary -<code class="typeof">object</code> -optional - -Summary -version of the relay's exit policy containing a dictionary with either an -"accept" or a "reject" element. -If there is an "accept" ("reject") element, the relay accepts (rejects) -all TCP ports or port ranges in the given list for most IP addresses and -rejects (accepts) all other ports. -May contradict the "exit_policy" field in a rare edge -case: this happens when the relay changes its exit policy after the -directory authorities summarized the previous exit policy. - -</li> - -<li> -exit_policy_v6_summary -<code class="typeof">object</code> -optional - -Summary version of the relay's IPv6 exit policy containing a dictionary -with either an "accept" or a "reject" element. -If there is an "accept" ("reject") element, the relay accepts (rejects) -all TCP ports or port ranges in the given list for most IP addresses and -rejects (accepts) all other ports. -Missing if the relay rejects all connections to IPv6 addresses. -May contradict the "exit_policy_summary" field in a rare -edge case: this happens when the relay changes its exit policy after the -directory authorities summarized the previous exit policy. - -</li> - -<li> -contact -<code class="typeof">string</code> -optional - -Contact address of the relay operator. -Omitted if empty or if descriptor containing this information cannot be -found. - -</li> - -<li> -platform -<code class="typeof">string</code> -optional - -Platform string containing operating system and Tor -version details. -Omitted if empty or if descriptor containing this information cannot be -found. - -</li> - -<li> -recommended_version -<code class="typeof">boolean</code> -optional - -Boolean field saying whether the Tor software version of this relay is -recommended by the directory authorities or not. -Omitted if either the directory authorities did not recommend versions, or -the relay did not report which version it runs. - -</li> - -<li> -family -<code class="typeof">array of strings</code> -optional - -Array of fingerprints or nicknames of relays that this relay considers to -be part of its family. -There are no cross-checks whether the listed relays exist or consider this -relay part of their family, so that the effective family of this relay may -be smaller. -Omitted if empty or if descriptor containing this information cannot be -found. -Deprecated on August 25, 2015, removed on January 18, -2016. - -</li> - -<li> -effective_family -<code class="typeof">array of strings</code> -optional - -Array of $-prefixed fingerprints of relays that are in an effective, -mutual family relationship with this relay. -These relays are part of this relay's family and they consider this relay -to be part of their family. -Omitted if empty or if descriptor containing this information cannot be -found. - -</li> - -<li> -alleged_family -<code class="typeof">array of strings</code> -optional - -Array of $-prefixed fingerprints of relays that are not in an effective, -mutual family relationship with this relay. -These relays are part of this relay's family but they don't consider this -relay to be part of their family. -Omitted if empty or if descriptor containing this information cannot be -found. - -</li> - -<li> -indirect_family -<code class="typeof">array of strings</code> -optional - -Array of $-prefixed fingerprints of relays that are not in an effective, -mutual family relationship with this relay but that can be reached by -following effective, mutual family relationships starting at this relay. -Omitted if empty or if descriptor containing this information cannot be -found. - -</li> - -<li> -consensus_weight_fraction -<code class="typeof">number</code> -optional - -Fraction of this relay's consensus weight compared to the sum of all -consensus weights in the network. -This fraction is a very rough approximation of the probability of this -relay to be selected by clients. -Omitted if the relay is not running. - -</li> - -<li> -guard_probability -<code class="typeof">number</code> -optional - -Probability of this relay to be selected for the guard position. -This probability is calculated based on consensus weights, relay flags, -and bandwidth weights in the consensus. -Path selection depends on more factors, so that this probability can only -be an approximation. -Omitted if the relay is not running, or the consensus does not contain -bandwidth weights. - -</li> - -<li> -middle_probability -<code class="typeof">number</code> -optional - -Probability of this relay to be selected for the middle position. -This probability is calculated based on consensus weights, relay flags, -and bandwidth weights in the consensus. -Path selection depends on more factors, so that this probability can only -be an approximation. -Omitted if the relay is not running, or the consensus does not contain -bandwidth weights. - -</li> - -<li> -exit_probability -<code class="typeof">number</code> -optional - -Probability of this relay to be selected for the exit position. -This probability is calculated based on consensus weights, relay flags, -and bandwidth weights in the consensus. -Path selection depends on more factors, so that this probability can only -be an approximation. -Omitted if the relay is not running, or the consensus does not contain -bandwidth weights. - -</li> - -<li> -measured -<code class="typeof">boolean</code> -optional - -Boolean field saying whether the consensus weight of this relay is based -on a threshold of 3 or more measurements by Tor bandwidth authorities. -Omitted if the network status consensus containing this relay does not -contain measurement information. - -</li> - -</ul> - -<h4>Bridge details objects</h4> - - -Bridge details objects contain the following key-value pairs: - - -<ul class="properties"> - -<li> -nickname -<code class="typeof">string</code> -optional - -Bridge nickname consisting of 1–19 -alphanumerical characters. -Omitted if the bridge nickname is "Unnamed". - -</li> - -<li> -hashed_fingerprint -<code class="typeof">string</code> -required - -SHA-1 hash of the bridge fingerprint -consisting of 40 upper-case hexadecimal characters. - -</li> - -<li> -or_addresses -<code class="typeof">array of strings</code> -required - -Array of sanitized IPv4 or IPv6 addresses and -TCP ports or port lists where the bridge accepts onion-routing -connections. -The first address is the primary onion-routing address that the bridge -used to register in the network, subsequent addresses are in arbitrary -order. -IPv6 hex characters are all lower-case. -Sanitized IP addresses are always in 10/8 or -[fd9f:2e19:3bcf/48] IP networks and are only useful to -learn which -IP version the bridge uses and to detect whether the bridge changed its -address. -Sanitized IP addresses always change on the 1st of every month at 00:00:00 -UTC, regardless of the bridge actually changing its IP address. -TCP ports are not sanitized. - -</li> - -<li> -last_seen -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when this -bridge was last seen in a bridge network status. - -</li> - -<li> -first_seen -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when this -bridge was first seen in a bridge network status. - -</li> - -<li> -running -<code class="typeof">boolean</code> -required - -Boolean field saying whether this bridge was listed -as running in the last bridge network status. - -</li> - -<li> -flags -<code class="typeof">array of strings</code> -optional - -Array of relay flags that the bridge authority -assigned to this bridge. -Omitted if empty. - -</li> - -<li> -last_restarted -<code class="typeof">string</code> -optional - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when the -bridge was last (re-)started. -Missing if router descriptor containing this information cannot be -found. - -</li> - -<li> -advertised_bandwidth -<code class="typeof">number</code> -optional - -Bandwidth in bytes per second that this -bridge is willing and capable to provide. -This bandwidth value is the minimum of bandwidth_rate, -bandwidth_burst, and observed_bandwidth. -Missing if router descriptor containing this information cannot be -found. - -</li> - -<li> -platform -<code class="typeof">string</code> -optional - -Platform string containing operating system and Tor -version details. -Omitted if not provided by the bridge or if descriptor containing this -information cannot be found. - -</li> - -<li> -transports -<code class="typeof">array of strings</code> -optional - -Array of (pluggable) transport names supported by this bridge. - -</li> - -</ul> - -</div>  - -<div class="box"> -<a name="history"></a> -<h3>History objects <a href="#history">#</a></h3> - - -History objects are no documents by themselves, but are contained in -subsequent documents. - - -<h4>Graph history objects</h4> - - -Graph history objects contain the following fields: - - -<ul class="properties"> - -<li> -first -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) of the first data point, or more -specifically the interval midpoint of the first interval. - -</li> - -<li> -last -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) of the last data point, or more -specifically the interval midpoint of the last interval. - -</li> - -<li> -interval -<code class="typeof">number</code> -required - -Time interval between two data points in seconds. - -</li> - -<li> -factor -<code class="typeof">number</code> -required - -Factor by which subsequent data values need to be multiplied to obtain -original values. -The idea is to reduce document size while still providing sufficient -detail for very different data scales. - -</li> - -<li> -count -<code class="typeof">number</code> -optional - -Number of provided data points, included mostly for debugging purposes. -Can also be derived from the number of elements in the subsequent array. - -</li> - -<li> -values -<code class="typeof">array of numbers</code> -required - -Array of normalized values between 0 and 999. -May contain null values. -Contains at least two subsequent non-null values to enable drawing of line -graphs. - -</li> - -</ul> - -</div>  - -<div class="box"> -<a name="bandwidth"></a> -<h3>Bandwidth documents <a href="#bandwidth">#</a> - -<a href="bandwidth?limit=4">example request</a> - -</h3> - - -Bandwidth documents contain aggregate statistics of a relay's or -bridge's consumed bandwidth for different time intervals. -Bandwidth documents are only updated when a relay or bridge publishes a -new server descriptor, which may take up to 18 hours during normal -operation. -Bandwidth documents contain the following fields: - - -<ul class="properties"> - -<li> -version -<code class="typeof">string</code> -required - -Onionoo protocol version string. - -</li> - -<li> -next_major_version_scheduled -<code class="typeof">string</code> -optional - -UTC date (YYYY-MM-DD) when the next major protocol version is scheduled to -be deployed. -Omitted if no major protocol changes are planned. - -</li> - -<li> -relays_published -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when -the last known relay network status consensus started being valid. -Indicates how recent the relay bandwidth documents in this document are. - -</li> - -<li> -relays -<code class="typeof">array of objects</code> -required - -Array of relay bandwidth objects as specified below. - -</li> - -<li> -bridges_published -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when -the last known bridge network status was published. -Indicates how recent the bridge bandwidth documents in this document are. - -</li> - -<li> -bridges -<code class="typeof">array of objects</code> -required - -Array of bridge bandwidth objects as specified below. - -</li> - -</ul> - -<h4>Relay bandwidth objects</h4> - - -Relay bandwidth objects contain the following key-value pairs: - - -<ul class="properties"> - -<li> -fingerprint -<code class="typeof">string</code> -required - -Relay fingerprint consisting of 40 upper-case -hexadecimal characters. - -</li> - -<li> -write_history -<code class="typeof">object</code> -optional - -Object containing graph history objects with written bytes for different -time periods. -Keys are string representation of the time period covered by the graph -history object. -Keys are fixed strings "3_days", -"1_week", "1_month", -"3_months", "1_year", and -"5_years". -Keys refer to the last known bandwidth history of a relay, not to the time -when the bandwidth document was published. -A graph history object is only contained if the time period it covers is -not already contained in another graph history object with shorter time -period and higher data resolution. -Similarly, a graph history object is excluded if the relay did not provide -bandwidth histories on the required level of detail. -The unit is bytes per second. -Contained graph history objects may contain null values if the relay did -not provide any bandwidth data or only data for less than 20% of a given -time period. - -</li> - -<li> -read_history -<code class="typeof">object</code> -optional - -Object containing graph history objects with read bytes for different time -periods. -The specification of graph history objects is similar to those in the -write_history field. - -</li> - -</ul> - -<h4>Bridge bandwidth objects</h4> - - -Bridge bandwidth objects contain the following key-value pairs: - - -<ul class="properties"> - -<li> -fingerprint -<code class="typeof">string</code> -required - -SHA-1 hash of the bridge fingerprint consisting -of 40 upper-case hexadecimal characters. - -</li> - -<li> -write_history -<code class="typeof">object</code> -optional - -Object containing graph history objects with written bytes for different -time periods. -The specification of graph history objects is similar to those in the -write_history field of relays. - -</li> - -<li> -read_history -<code class="typeof">object</code> -optional - -Object containing graph history objects with read bytes for different time -periods. -The specification of graph history objects is similar to those in the -write_history field of relays. - -</li> - -</ul> - -</div>  - -<div class="box"> -<a name="weights"></a> -<h3>Weights documents <a href="#weights">#</a> - -<a href="weights?limit=4">example request</a> - -</h3> - - -Weights documents contain aggregate statistics of a relay's probability -to be selected by clients for building paths. -Weights documents contain different time intervals and are available for -relays only. -Weights documents contain the following fields: - - -<ul class="properties"> - -<li> -version -<code class="typeof">string</code> -required - -Onionoo protocol version string. - -</li> - -<li> -next_major_version_scheduled -<code class="typeof">string</code> -optional - -UTC date (YYYY-MM-DD) when the next major protocol version is scheduled to -be deployed. -Omitted if no major protocol changes are planned. - -</li> - -<li> -relays_published -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when -the last known relay network status consensus started being valid. -Indicates how recent the relay weights documents in this document are. - -</li> - -<li> -relays -<code class="typeof">array of objects</code> -required - -Array of relay weights objects as specified below. - -</li> - -<li> -bridges_published -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when -the last known bridge network status was published. -Only included for compatibility reasons with the other document types. - -</li> - -<li> -bridges -<code class="typeof">array of objects</code> -required - -Empty array of objects that would represent bridge weights documents. -Only included for compatibility reasons with the other document types. - -</li> - -</ul> - -<h4>Relay weights objects</h4> - - -Relay weights objects contain the following key-value pairs: - - -<ul class="properties"> - -<li> -fingerprint -<code class="typeof">string</code> -required - -Relay fingerprint consisting of 40 upper-case -hexadecimal characters. - -</li> - -<li> -consensus_weight_fraction -<code class="typeof">object</code> -optional - -History object containing the -fraction of this relay's consensus weight compared to the sum of all -consensus weights in the network. -This fraction is a very rough approximation of the probability of this -relay to be selected by clients. -Keys are string representation of the time period covered by the graph -history object. -Keys are fixed strings "1_week", -"1_month", "3_months", -"1_year", and "5_years". -Keys refer to the last known weights history of a relay, not to the time -when the weights document was published. -A graph history object is only contained if the time period it covers is -not already contained in another graph history object with shorter time -period and higher data resolution. -The unit is path-selection probability. -Contained graph history objects may contain null values if the relay was -running less than 20% of a given time period. - -</li> - -<li> -guard_probability -<code class="typeof">object</code> -optional - -History object containing the probability -of this relay to be selected for the guard position. -This probability is calculated based on consensus weights, relay flags, -and bandwidth weights in the consensus. -Path selection depends on more factors, so that this probability can only -be an approximation. -The specification of this history object is similar to that in the -consensus_weight_fraction field above. - -</li> - -<li> -middle_probability -<code class="typeof">object</code> -optional - -History object containing the probability -of this relay to be selected for the middle position. -This probability is calculated based on consensus weights, relay flags, -and bandwidth weights in the consensus. -Path selection depends on more factors, so that this probability can only -be an approximation. -The specification of this history object is similar to that in the -consensus_weight_fraction field above. - -</li> - -<li> -exit_probability -<code class="typeof">object</code> -optional - -History object containing the probability -of this relay to be selected for the exit position. -This probability is calculated based on consensus weights, relay flags, -and bandwidth weights in the consensus. -Path selection depends on more factors, so that this probability can only -be an approximation. -The specification of this history object is similar to that in the -consensus_weight_fraction field above. - -</li> - -<li> -consensus_weight -<code class="typeof">object</code> -optional - -History object containing the absolute consensus weight of this relay. -The specification of this history object is similar to that in the -consensus_weight_fraction field above. - -</li> - -</ul> - -</div>  - -<div class="box"> -<a name="clients"></a> -<h3>Clients documents <a href="#clients">#</a> - -<a href="clients?limit=4">example request</a> - -</h3> - - -Clients documents contain estimates of the average number of clients -connecting to a bridge every day. -There are no clients documents available for relays, just for bridges. -Clients documents contain different time intervals and are available for -bridges only. -Clients documents contain the following fields: - - -<ul class="properties"> - -<li> -version -<code class="typeof">string</code> -required - -Onionoo protocol version string. - -</li> - -<li> -next_major_version_scheduled -<code class="typeof">string</code> -optional - -UTC date (YYYY-MM-DD) when the next major protocol version is scheduled to -be deployed. -Omitted if no major protocol changes are planned. - -</li> - -<li> -relays_published -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when -the last known relay network status consensus started being valid. -Only included for compatibility reasons with the other document types. - -</li> - -<li> -relays -<code class="typeof">array of objects</code> -required - -Empty array of objects that would represent relay clients documents. -Only included for compatibility reasons with the other document types. - -</li> - -<li> -bridges_published -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when -the last known bridge network status was published. -Indicates how recent the bridge clients documents in this document are. - -</li> - -<li> -bridges -<code class="typeof">array of objects</code> -required - -Array of bridge clients objects as specified below. - - -</li> - -</ul> - -<h4>Bridge clients objects</h4> - - -Bridge clients objects contain the following key-value pairs: - - -<ul class="properties"> - -<li> -fingerprint -<code class="typeof">string</code> -required - -SHA-1 hash of the bridge fingerprint consisting -of 40 upper-case hexadecimal characters. - -</li> - -<li> -average_clients -<code class="typeof">object</code> -optional - -Object containing graph history objects with the average number of clients -connecting to this bridge. -Keys are string representation of the time period covered by the graph -history object. -Keys are fixed strings "1_week", -"1_month", "3_months", -"1_year", and "5_years". -Keys refer to the last known clients history of a bridge, not to the time -when the clients document was published. -A graph history object is only contained if the time period it covers -is not already contained in another clients graph object with shorter -time period and higher data resolution. -The unit is number of clients. -Contained graph history objects may contain null values if the bridge did -not report client statistics for at least 50% of a given time period. -Each graph history object contains the following additional key-value -pairs: - - -<ul class="properties"> - -<li> -countries -<code class="typeof">object</code> -optional - -Object containing fractions of clients by country in the considered time -period. -Keys are two-letter lower-case country codes as found in a GeoIP database. -Values are numbers between 0 and 1 standing for the fraction of clients by -country. -A country is only included if at least 1% of clients came from this -country. -Omitted if the bridge did not report client statistics by country. -BETA: This field breaks compatibility -with the history objects contained in other documents pretty badly. -It might be removed in the future without notice. - -</li> - -<li> -transports -<code class="typeof">object</code> -optional - -Object containing fractions of clients by transport in the considered time -period. -Keys are transport names, or "<OR>" for the default -onion-routing transport protocol. -Values are numbers between 0 and 1 standing for the fraction of clients by -transport. -Omitted if the bridge did not report client statistics by transport. -BETA: This field breaks compatibility -with the history objects contained in other documents pretty badly. -It might be removed in the future without notice. - -</li> - -<li> -versions -<code class="typeof">object</code> -optional - -Object containing fractions of clients by IP version in the considered -time period. -Keys are either "v4" for IPv4 or "v6" -for IPv6. -Values are numbers between 0 and 1 standing for the fraction of clients by -version. -Omitted if the bridge did not report client statistics by IP version. -BETA: This field breaks compatibility -with the history objects contained in other documents pretty badly. -It might be removed in the future without notice. - -</li> - -</ul> - -</li> - -</ul> - -</div>  - -<div class="box"> -<a name="uptime"></a> -<h3>Uptime documents <a href="#uptime">#</a> - -<a href="uptime?limit=4">example request</a> - -</h3> - - -Uptime documents contain fractional uptimes of relays and bridges. -Uptime documents contain different time intervals and are available for -relays and bridges. -Uptime documents contain the following fields: - - -<ul class="properties"> - -<li> -version -<code class="typeof">string</code> -required - -Onionoo protocol version string. - -</li> - -<li> -next_major_version_scheduled -<code class="typeof">string</code> -optional - -UTC date (YYYY-MM-DD) when the next major protocol version is scheduled to -be deployed. -Omitted if no major protocol changes are planned. - -</li> - -<li> -relays_published -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when -the last known relay network status consensus started being valid. -Indicates how recent the relay uptime documents in this document are. - -</li> - -<li> -relays -<code class="typeof">array of objects</code> -required - -Array of relay uptime objects as specified below. - -</li> - -<li> -bridges_published -<code class="typeof">string</code> -required - -UTC timestamp (YYYY-MM-DD hh:mm:ss) when -the last known bridge network status was published. -Indicates how recent the bridge uptime documents in this document are. - -</li> - -<li> -bridges -<code class="typeof">array of objects</code> -required - -Array of bridge uptime objects as specified below. - -</li> - -</ul> - -<h4>Relay uptime objects</h4> - - -Relay uptime objects contain the following key-value pairs: - - -<ul class="properties"> - -<li> -fingerprint -<code class="typeof">string</code> -required - -Relay fingerprint consisting of 40 upper-case -hexadecimal characters. - -</li> - -<li> -uptime -<code class="typeof">object</code> -optional - -Object containing graph history objects with the fractional uptime of this -relay. -Keys are string representation of the time period covered by the graph -history object. -Keys are fixed strings "1_week", -"1_month", "3_months", -"1_year", and "5_years". -Keys refer to the last known uptime history of a relay, not to the time -when the uptime document was published. -A graph history object is only contained if the time period it covers is -not already contained in another graph history object with shorter time -period and higher data resolution. -The unit is fractional uptime from 0 to 1. -Contained graph history objects may contain null values if less than 20% -of network statuses have been processed for a given time period. - -</li> - -<li> -flags -<code class="typeof">object</code> -optional - -Object containing fractional times of this relay having relay flags -assigned. -Keys are flag names like "Running" or -"Exit", values are objects similar to the -uptime field above, again with keys like -"1_week" etc. -If a relay never had a given relay flag assigned, no object is included -for that flag. - -</li> - -</ul> - -<h4>Bridge uptime objects</h4> - - -Bridge uptime objects contain the following key-value pairs: - - -<ul class="properties"> - -<li> -fingerprint -<code class="typeof">string</code> -required - -SHA-1 hash of the bridge fingerprint consisting -of 40 upper-case hexadecimal characters. - -</li> - -<li> -uptime -<code class="typeof">object</code> -optional - -Object containing uptime history objects for different time periods. -The specification of uptime history objects is similar to those in the -uptime field of relays. - -</li> - -</ul> - -</div>  - -<div class="box"> -<a name="examples"></a> -<h3>Example usage <a href="#examples">#</a> -</h3> - - -The following examples illustrate how to build requests for some trivial -and some more complex use cases. -While Onionoo is designed mainly for developers and not end users, there -may be cases when it's easier to quickly write a specific query Onionoo -rather than to find an Onionoo client that provides the desired -information. - - -<pre>https://onionoo.torproject.org/summary?limit=4</pre> - - -This first query returns the first four summary documents that Onionoo can -find. -The <code>limit</code> parameter should always be used while developing -new queries to avoid downloading huge responses. - - -<pre>https://onionoo.torproject.org/summary?limit=4&search=moria</pre> - - -The second query restricts results to relays and bridges containing the -string "moria" in one of a few searched fields. - - -<pre>https://onionoo.torproject.org/details?limit=4&search=moria</pre> - - -The third query switches from the short summary documents to the longer -details documents containing, well, more details. - - -<pre>https://onionoo.torproject.org/details?limit=4&search=moria&fields=nickname</pre> - - -The fourth query adds the <code>fields</code> parameter which removes all -fields except the specified ones from the result. -This parameter is only implemented for details documents. - - -<pre>https://onionoo.torproject.org/details?limit=4&search=moria&fields=nickname…</pre> - - -The fifth query sorts results by relay consensus weight from largest to -smallest. - - - -Obviously, this query can be made even more complex by adding more -parameters, and in some cases this is necessary and useful. -Please refer to the protocol specification for details. - - -</div>  - -</body> -</html> - diff --git a/web/robots.txt b/web/robots.txt deleted file mode 100644 index 9b19f8f..0000000 --- a/web/robots.txt +++ /dev/null @@ -1,8 +0,0 @@ -User-agent: * -Disallow: /summary -Disallow: /details -Disallow: /bandwidth -Disallow: /weights -Disallow: /clients -Disallow: /uptime -

1 0

Jump to page: