July 2016 - tor-commits - lists.torproject.org

[onionoo/master] Resolve all remaining Javadoc-related checkstyle issues.
by karsten＠torproject.org 25 Jul '16

25 Jul '16

commit c51516bd5044b4b59c24c3cfe6bddc8d9882e018 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jul 19 10:11:02 2016 +0200 Resolve all remaining Javadoc-related checkstyle issues. Resolve very few of these warnings by suppressing them, in particular getters and setters that happen to contain more than 2 lines of code, because we should either document all or none of them. Implements more of #19613. --- build.xml | 13 ++++++++++++ .../java/org/torproject/onionoo/cron/Main.java | 3 +++ .../torproject/onionoo/docs/BandwidthStatus.java | 2 ++ .../torproject/onionoo/docs/ClientsHistory.java | 12 +++++++++++ .../org/torproject/onionoo/docs/ClientsStatus.java | 6 ++++++ .../torproject/onionoo/docs/DateTimeHelper.java | 6 ++++++ .../torproject/onionoo/docs/DetailsDocument.java | 8 ++++++++ .../org/torproject/onionoo/docs/DetailsStatus.java | 3 +++ .../org/torproject/onionoo/docs/DocumentStore.java | 13 ++++++++++++ .../onionoo/docs/DocumentStoreFactory.java | 5 +++++ .../org/torproject/onionoo/docs/NodeStatus.java | 23 ++++++++++++++++++++++ .../torproject/onionoo/docs/SummaryDocument.java | 13 ++++++++++++ .../org/torproject/onionoo/docs/UpdateStatus.java | 2 ++ .../org/torproject/onionoo/docs/UptimeHistory.java | 12 +++++++++++ .../org/torproject/onionoo/docs/UptimeStatus.java | 6 ++++++ .../org/torproject/onionoo/docs/WeightsStatus.java | 6 ++++++ .../org/torproject/onionoo/server/NodeIndexer.java | 12 +++++++++++ .../onionoo/server/NodeIndexerFactory.java | 5 +++++ .../onionoo/server/PerformanceMetrics.java | 6 ++++++ .../torproject/onionoo/server/RequestHandler.java | 4 ++++ .../torproject/onionoo/server/ResourceServlet.java | 6 ++++++ .../org/torproject/onionoo/server/ServerMain.java | 1 + .../onionoo/updater/BandwidthStatusUpdater.java | 6 ++++++ .../onionoo/updater/ClientsStatusUpdater.java | 6 ++++++ .../onionoo/updater/DescriptorSource.java | 10 ++++++++++ .../onionoo/updater/DescriptorSourceFactory.java | 5 +++++ .../torproject/onionoo/updater/LookupService.java | 5 +++++ .../onionoo/updater/NodeDetailsStatusUpdater.java | 6 ++++++ .../onionoo/updater/RdnsLookupRequest.java | 1 + .../onionoo/updater/RdnsLookupWorker.java | 1 + .../onionoo/updater/ReverseDomainNameResolver.java | 9 +++++++++ .../onionoo/updater/StatusUpdateRunner.java | 4 ++++ .../onionoo/updater/UptimeStatusUpdater.java | 6 ++++++ .../onionoo/updater/WeightsStatusUpdater.java | 6 ++++++ .../torproject/onionoo/util/FormattingUtils.java | 5 +++++ .../org/torproject/onionoo/util/TimeFactory.java | 5 +++++ .../onionoo/writer/BandwidthDocumentWriter.java | 2 ++ .../onionoo/writer/ClientsDocumentWriter.java | 2 ++ .../onionoo/writer/DetailsDocumentWriter.java | 2 ++ .../onionoo/writer/DocumentWriterRunner.java | 4 ++++ .../onionoo/writer/SummaryDocumentWriter.java | 2 ++ .../onionoo/writer/UptimeDocumentWriter.java | 2 ++ .../onionoo/writer/WeightsDocumentWriter.java | 2 ++ src/test/resources/metrics_checks.xml | 2 ++ 44 files changed, 260 insertions(+) diff --git a/build.xml b/build.xml index 1688207..f7f6bc0 100644 --- a/build.xml +++ b/build.xml @@ -10,6 +10,7 @@ <property name="classes" value="classes"/> <property name="testresources" value="src/test/resources/"/> <property name="dist" value="dist"/> + <property name="docs" value="${generated}/javadoc/"/> <property name="libs" value="lib"/> <property name="config" value="etc"/> <property name="webxmlfile" value="${config}/web.xml"/> @@ -85,6 +86,7 @@ <copy file="${contextxmltemplate}" tofile="${contextxml}"/> <copy file="${webxmltemplate}" tofile="${webxml}"/> <mkdir dir="${classes}"/> + <mkdir dir="${docs}"/> <mkdir dir="${dist}"/> <mkdir dir="${generated}"/> </target> @@ -111,6 +113,17 @@ </javac> </target> + <target name="docs" depends="init"> + <javadoc destdir="${docs}" + footer="&copy; 2016 The Tor Project" + doctitle="Onionoo Documentation" + use="true" + windowtitle="Onionoo Documentation"> + <classpath refid="classpath"/> + <fileset dir="${javasources}/" includes="**/*.java" /> + </javadoc> + </target> + <target name="test" depends="compile"> <javac destdir="${classes}" srcdir="${tests}" diff --git a/src/main/java/org/torproject/onionoo/cron/Main.java b/src/main/java/org/torproject/onionoo/cron/Main.java index 4a57f66..04b6955 100644 --- a/src/main/java/org/torproject/onionoo/cron/Main.java +++ b/src/main/java/org/torproject/onionoo/cron/Main.java @@ -28,6 +28,8 @@ public class Main implements Runnable { private Logger log = LoggerFactory.getLogger(Main.class); + /** Executes a single update run or partial update run, or initiates + * hourly executions, depending on the given command-line arguments. */ public static void main(String[] args) { Main main = new Main(); main.parseArgsOrExit(args); @@ -121,6 +123,7 @@ public class Main implements Runnable { TimeUnit.MINUTES); } + @Override public void run() { this.acquireLockOrExit(); this.initialize(); diff --git a/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java b/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java index 65d4cf5..ac7085f 100644 --- a/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java @@ -51,6 +51,7 @@ public class BandwidthStatus extends Document { return this.readHistory; } + @Override public void setFromDocumentString(String documentString) { try (Scanner s = new Scanner(documentString)) { while (s.hasNextLine()) { @@ -170,6 +171,7 @@ public class BandwidthStatus extends Document { } } + @Override public String toDocumentString() { StringBuilder sb = new StringBuilder(); for (long[] v : writeHistory.values()) { diff --git a/src/main/java/org/torproject/onionoo/docs/ClientsHistory.java b/src/main/java/org/torproject/onionoo/docs/ClientsHistory.java index 0f4c148..e99c3ef 100644 --- a/src/main/java/org/torproject/onionoo/docs/ClientsHistory.java +++ b/src/main/java/org/torproject/onionoo/docs/ClientsHistory.java @@ -51,6 +51,9 @@ public class ClientsHistory implements Comparable<ClientsHistory> { return this.responsesByVersion; } + /** Instantiates a new clients history object with given interval start + * and end, total responses, and responses by country, transport, and + * version. */ public ClientsHistory(long startMillis, long endMillis, double totalResponses, SortedMap<String, Double> responsesByCountry, @@ -64,6 +67,8 @@ public class ClientsHistory implements Comparable<ClientsHistory> { this.responsesByVersion = responsesByVersion; } + /** Instantiates a new clients history object from the given string that + * may have been produced by {@link #toString()}. */ public static ClientsHistory fromString( String responseHistoryString) { String[] parts = responseHistoryString.split(" ", 8); @@ -133,6 +138,7 @@ public class ClientsHistory implements Comparable<ClientsHistory> { return responses; } + @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append(DateTimeHelper.format(startMillis)); @@ -154,6 +160,9 @@ public class ClientsHistory implements Comparable<ClientsHistory> { } } + /** Adds responses from another clients history object to this one by + * summing up response numbers and extending interval start and/or + * end. */ public void addResponses(ClientsHistory other) { this.totalResponses += other.totalResponses; this.addResponsesByCategory(this.responsesByCountry, @@ -183,16 +192,19 @@ public class ClientsHistory implements Comparable<ClientsHistory> { } } + @Override public int compareTo(ClientsHistory other) { return this.startMillis < other.startMillis ? -1 : this.startMillis > other.startMillis ? 1 : 0; } + @Override public boolean equals(Object other) { return other instanceof ClientsHistory && this.startMillis == ((ClientsHistory) other).startMillis; } + @Override public int hashCode() { return (int) this.startMillis; } diff --git a/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java b/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java index 4ef998e..45466ae 100644 --- a/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java @@ -38,6 +38,7 @@ public class ClientsStatus extends Document { return this.history; } + @Override public void setFromDocumentString(String documentString) { try (Scanner s = new Scanner(documentString)) { while (s.hasNextLine()) { @@ -53,6 +54,8 @@ public class ClientsStatus extends Document { } } + /** Adds all given clients history objects that don't overlap with + * existing clients history objects. */ public void addToHistory(SortedSet<ClientsHistory> newIntervals) { for (ClientsHistory interval : newIntervals) { if ((this.history.headSet(interval).isEmpty() @@ -67,6 +70,8 @@ public class ClientsStatus extends Document { } } + /** Compresses the history of clients objects by merging adjacent + * intervals, depending on how far back in the past they lie. */ public void compressHistory() { SortedSet<ClientsHistory> uncompressedHistory = new TreeSet<ClientsHistory>(this.history); @@ -107,6 +112,7 @@ public class ClientsStatus extends Document { } } + @Override public String toDocumentString() { StringBuilder sb = new StringBuilder(); for (ClientsHistory interval : this.history) { diff --git a/src/main/java/org/torproject/onionoo/docs/DateTimeHelper.java b/src/main/java/org/torproject/onionoo/docs/DateTimeHelper.java index 4e422e9..1dc40c3 100644 --- a/src/main/java/org/torproject/onionoo/docs/DateTimeHelper.java +++ b/src/main/java/org/torproject/onionoo/docs/DateTimeHelper.java @@ -109,6 +109,9 @@ public class DateTimeHelper { return format(millis, ISO_DATETIME_FORMAT); } + /** Parses the given string using the given format and return the time + * in milliseconds since the epoch or {@link #NO_TIME_AVAILABLE} if the + * string cannot be parsed. */ public static long parse(String string, String format) { if (null == string) { log.warn("Date String was null."); @@ -122,6 +125,9 @@ public class DateTimeHelper { } } + /** Parses the given string using {@link #ISO_DATETIME_FORMAT} as format + * and return the time in milliseconds since the epoch or + * {@link #NO_TIME_AVAILABLE} if the string cannot be parsed. */ public static long parse(String string) { return parse(string, ISO_DATETIME_FORMAT); } diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java index 8ffb358..85c0154 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java @@ -377,6 +377,8 @@ public class DetailsDocument extends Document { private Float consensus_weight_fraction; + /** Sets the consensus weight fraction to the given value, but only if + * that value is neither null nor negative. */ public void setConsensusWeightFraction(Float consensusWeightFraction) { if (consensusWeightFraction == null || consensusWeightFraction >= 0.0) { @@ -390,6 +392,8 @@ public class DetailsDocument extends Document { private Float guard_probability; + /** Sets the guard probability to the given value, but only if that + * value is neither null nor negative. */ public void setGuardProbability(Float guardProbability) { if (guardProbability == null || guardProbability >= 0.0) { this.guard_probability = guardProbability; @@ -402,6 +406,8 @@ public class DetailsDocument extends Document { private Float middle_probability; + /** Sets the middle probability to the given value, but only if that + * value is neither null nor negative. */ public void setMiddleProbability(Float middleProbability) { if (middleProbability == null || middleProbability >= 0.0) { this.middle_probability = middleProbability; @@ -414,6 +420,8 @@ public class DetailsDocument extends Document { private Float exit_probability; + /** Sets the exit probability to the given value, but only if that + * value is neither null nor negative. */ public void setExitProbability(Float exitProbability) { if (exitProbability == null || exitProbability >= 0.0) { this.exit_probability = exitProbability; diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java index dcc6fcb..1a3c05d 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java @@ -250,6 +250,9 @@ public class DetailsStatus extends Document { this.or_addresses_and_ports; } + /** Returns all addresses used for the onion-routing protocol which + * includes the primary address and all additionally configured + * onion-routing addresses. */ public SortedSet<String> getOrAddresses() { SortedSet<String> orAddresses = new TreeSet<String>(); if (this.address != null) { diff --git a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java index 600115c..42c75aa 100644 --- a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java +++ b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java @@ -97,6 +97,8 @@ public class DocumentStore { return this.list(documentType, 0L); } + /** Returns all fingerprints of documents of the given type that have + * been updated after the given time in milliseconds since the epoch. */ public <T extends Document> SortedSet<String> list( Class<T> documentType, long updatedAfter) { if (documentType.equals(NodeStatus.class)) { @@ -258,6 +260,8 @@ public class DocumentStore { return this.store(document, null); } + /** Stores the given document using the given fingerprint as + * identifier. */ public <T extends Document> boolean store(T document, String fingerprint) { if (document instanceof NodeStatus) { @@ -372,6 +376,8 @@ public class DocumentStore { return this.retrieve(documentType, parse, null); } + /** Retrieves the document with given type and identified by the given + * fingerprint, and either parses it or returns it unparsed. */ public <T extends Document> T retrieve(Class<T> documentType, boolean parse, String fingerprint) { if (documentType.equals(NodeStatus.class)) { @@ -576,6 +582,8 @@ public class DocumentStore { return this.remove(documentType, null); } + /** Removes the document with given type and identified by the given + * fingerprint. */ public <T extends Document> boolean remove(Class<T> documentType, String fingerprint) { if (documentType.equals(NodeStatus.class)) { @@ -681,6 +689,8 @@ public class DocumentStore { return documentFile; } + /** Writes cached node statuses, cached summary documents, and then the + * update file to disk. */ public void flushDocumentCache() { /* Write cached node statuses to disk, and write update file * containing current time. It's important to write the update file @@ -698,6 +708,8 @@ public class DocumentStore { } } + /** Invalidates the document cache, so that it will be freshly populated + * during the next execution. */ public void invalidateDocumentCache() { this.cachedNodeStatuses = null; this.cachedSummaryDocuments = null; @@ -810,6 +822,7 @@ public class DocumentStore { this.store(updateStatus); } + /** Returns a string with statistics on document storage operations. */ public String getStatsString() { StringBuilder sb = new StringBuilder(); sb.append(" " + FormattingUtils.formatDecimalNumber(listOperations) diff --git a/src/main/java/org/torproject/onionoo/docs/DocumentStoreFactory.java b/src/main/java/org/torproject/onionoo/docs/DocumentStoreFactory.java index 3fa28fc..455a27f 100644 --- a/src/main/java/org/torproject/onionoo/docs/DocumentStoreFactory.java +++ b/src/main/java/org/torproject/onionoo/docs/DocumentStoreFactory.java @@ -7,10 +7,15 @@ public class DocumentStoreFactory { private static DocumentStore documentStoreInstance; + /** Sets a custom singleton document store instance that will be + * returned by {@link #getDocumentStore()} rather than creating an + * instance upon first invocation. */ public static void setDocumentStore(DocumentStore documentStore) { documentStoreInstance = documentStore; } + /** Returns the singleton document store instance that gets created upon + * first invocation of this method. */ public static DocumentStore getDocumentStore() { if (documentStoreInstance == null) { documentStoreInstance = new DocumentStore(); diff --git a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java index 630171b..afdd6c6 100644 --- a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java @@ -29,6 +29,9 @@ public class NodeStatus extends Document { private String contact; + /** Sets the contact to a lower-cased variant of the given string with + * all non-printable characters outside of ASCII code 32 (space) to 126 + * (dash) replaced with spaces. */ public void setContact(String contact) { if (contact == null) { this.contact = null; @@ -130,6 +133,9 @@ public class NodeStatus extends Document { : this.orAddressesAndPorts; } + /** Returns all addresses used for the onion-routing protocol which + * includes the primary address and all additionally configured + * onion-routing addresses. */ public SortedSet<String> getOrAddresses() { SortedSet<String> orAddresses = new TreeSet<String>(); if (this.address != null) { @@ -195,6 +201,7 @@ public class NodeStatus extends Document { private BitSet relayFlags; + @SuppressWarnings("checkstyle:javadocmethod") public void setRelayFlags(SortedSet<String> relayFlags) { BitSet newRelayFlags = new BitSet(relayFlagIndexes.size()); for (String relayFlag : relayFlags) { @@ -207,6 +214,7 @@ public class NodeStatus extends Document { this.relayFlags = newRelayFlags; } + @SuppressWarnings("checkstyle:javadocmethod") public SortedSet<String> getRelayFlags() { SortedSet<String> result = new TreeSet<String>(); if (this.relayFlags != null) { @@ -255,6 +263,9 @@ public class NodeStatus extends Document { return new TreeMap<Long, Set<String>>(this.lastAddresses); } + /** Adds addresses and ports together with the time in milliseconds + * since the epoch when they were last seen to the history of last seen + * addresses and ports. */ public void addLastAddresses(long lastSeenMillis, String address, int orPort, int dirPort, SortedSet<String> orAddressesAndPorts) { Set<String> addressesAndPorts = new HashSet<String>(); @@ -270,6 +281,8 @@ public class NodeStatus extends Document { } } + /** Returns the time in milliseconds since the epoch when addresses or + * ports were last changed. */ public long getLastChangedOrAddressOrPort() { long lastChangedAddressesMillis = -1L; if (this.lastAddresses != null) { @@ -369,6 +382,9 @@ public class NodeStatus extends Document { return stringArrayToSortedSet(this.extendedFamily); } + /** Returns the alleged family consisting of all relays in this relay's + * declared family that are not in a mutual family relationship with + * this relay. */ public SortedSet<String> getAllegedFamily() { SortedSet<String> allegedFamily = new TreeSet<String>( stringArrayToSortedSet(this.declaredFamily)); @@ -376,6 +392,9 @@ public class NodeStatus extends Document { return allegedFamily; } + /** Returns the indirect family consisting of all relays that can be + * reached via mutual family relationships except for those that can be + * reached directly via such a relationship. */ public SortedSet<String> getIndirectFamily() { SortedSet<String> indirectFamily = new TreeSet<String>( stringArrayToSortedSet(this.extendedFamily)); @@ -385,10 +404,13 @@ public class NodeStatus extends Document { /* Constructor and (de-)serialization methods: */ + /** Instantiates a new node status object from the given fingerprint. */ public NodeStatus(String fingerprint) { this.fingerprint = fingerprint; } + /** Instantiates a new node status object from the given string that may + * have been produced by {@link #toString()}. */ public static NodeStatus fromString(String documentString) { try { String[] parts = documentString.trim().split("\t"); @@ -529,6 +551,7 @@ public class NodeStatus extends Document { } } + @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append(this.isRelay ? "r" : "b"); diff --git a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java index cebcf5e..6a520c8 100644 --- a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java @@ -30,6 +30,9 @@ public class SummaryDocument extends Document { private String f; + /** Sets the fingerprint to the given 40 hex characters and clears + * SHA1-hashed and base64 fingerprints, so that they are re-computed at + * next request. */ public void setFingerprint(String fingerprint) { if (fingerprint != null) { Pattern fingerprintPattern = Pattern.compile("^[0-9a-fA-F]{40}$"); @@ -49,6 +52,8 @@ public class SummaryDocument extends Document { private transient String hashedFingerprint = null; + /** Returns the SHA1-hashed fingerprint, or <code>null</code> if no + * fingerprint is set. */ public String getHashedFingerprint() { if (this.hashedFingerprint == null && this.f != null) { try { @@ -63,6 +68,8 @@ public class SummaryDocument extends Document { private transient String base64Fingerprint = null; + /** Returns the base64-encoded fingerprint, or <code>null</code> if no + * fingerprint is set. */ public String getBase64Fingerprint() { if (this.base64Fingerprint == null && this.f != null) { try { @@ -77,6 +84,9 @@ public class SummaryDocument extends Document { private transient String[] fingerprintSortedHexBlocks = null; + /** Returns a sorted array containing blocks of 4 upper-case hex + * characters from the fingerprint, or <code>null</code> if no + * fingerprint is set. */ public String[] getFingerprintSortedHexBlocks() { if (this.fingerprintSortedHexBlocks == null && this.f != null) { String fingerprint = this.f.toUpperCase(); @@ -94,6 +104,7 @@ public class SummaryDocument extends Document { private String n; + @SuppressWarnings("checkstyle:javadocmethod") public void setNickname(String nickname) { if (nickname == null || nickname.equals("Unnamed")) { this.n = null; @@ -219,6 +230,7 @@ public class SummaryDocument extends Document { private String c; + @SuppressWarnings("checkstyle:javadocmethod") public void setContact(String contact) { if (contact != null && contact.length() == 0) { this.c = null; @@ -257,6 +269,7 @@ public class SummaryDocument extends Document { /* The familyFingerprints parameter can go away after September 8, 2015. * See above. */ + /** Instantiates a summary document with all given properties. */ public SummaryDocument(boolean isRelay, String nickname, String fingerprint, List<String> addresses, long lastSeenMillis, boolean running, SortedSet<String> relayFlags, long consensusWeight, diff --git a/src/main/java/org/torproject/onionoo/docs/UpdateStatus.java b/src/main/java/org/torproject/onionoo/docs/UpdateStatus.java index 07d5d20..4223337 100644 --- a/src/main/java/org/torproject/onionoo/docs/UpdateStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/UpdateStatus.java @@ -20,6 +20,7 @@ public class UpdateStatus extends Document { return this.updatedMillis; } + @Override public void setFromDocumentString(String documentString) { try { this.updatedMillis = Long.parseLong(documentString.trim()); @@ -30,6 +31,7 @@ public class UpdateStatus extends Document { } } + @Override public String toDocumentString() { return String.valueOf(this.updatedMillis); } diff --git a/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java b/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java index 6f1cb2e..3cd0a0e 100644 --- a/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java +++ b/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java @@ -38,6 +38,8 @@ public class UptimeHistory implements Comparable<UptimeHistory> { return this.flags; } + /** Instantiates a new uptime history object for a relay or bridge with + * the given interval start, uptime hours, and relay flags. */ UptimeHistory(boolean relay, long startMillis, int uptimeHours, SortedSet<String> flags) { this.relay = relay; @@ -46,6 +48,8 @@ public class UptimeHistory implements Comparable<UptimeHistory> { this.flags = flags; } + /** Instantiates a new uptime history object from the given string that + * may have been produced by {@link #toString()}. */ public static UptimeHistory fromString(String uptimeHistoryString) { String[] parts = uptimeHistoryString.split(" ", -1); if (parts.length < 3) { @@ -87,6 +91,7 @@ public class UptimeHistory implements Comparable<UptimeHistory> { return new UptimeHistory(relay, startMillis, uptimeHours, flags); } + @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append(this.relay ? (this.flags == null ? "r" : "R") : "b"); @@ -101,6 +106,10 @@ public class UptimeHistory implements Comparable<UptimeHistory> { return sb.toString(); } + /** Adds uptime hours from another uptime history object, which is + * assumed to either start right after this one or which ends right + * before it, and sets the interval start to the earlier interval + * start. */ public void addUptime(UptimeHistory other) { this.uptimeHours += other.uptimeHours; if (this.startMillis > other.startMillis) { @@ -108,6 +117,7 @@ public class UptimeHistory implements Comparable<UptimeHistory> { } } + @Override public int compareTo(UptimeHistory other) { if (this.relay && !other.relay) { return -1; @@ -118,12 +128,14 @@ public class UptimeHistory implements Comparable<UptimeHistory> { : this.startMillis > other.startMillis ? 1 : 0; } + @Override public boolean equals(Object other) { return other instanceof UptimeHistory && this.relay == ((UptimeHistory) other).relay && this.startMillis == ((UptimeHistory) other).startMillis; } + @Override public int hashCode() { return (int) this.startMillis + (this.relay ? 1 : 0); } diff --git a/src/main/java/org/torproject/onionoo/docs/UptimeStatus.java b/src/main/java/org/torproject/onionoo/docs/UptimeStatus.java index ce3a4d1..b725acc 100644 --- a/src/main/java/org/torproject/onionoo/docs/UptimeStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/UptimeStatus.java @@ -40,6 +40,7 @@ public class UptimeStatus extends Document { return this.bridgeHistory; } + @Override public void setFromDocumentString(String documentString) { try (Scanner s = new Scanner(documentString)) { while (s.hasNextLine()) { @@ -59,6 +60,8 @@ public class UptimeStatus extends Document { } } + /** Adds all given uptime history objects that don't overlap with + * existing uptime history objects. */ public void addToHistory(boolean relay, long startMillis, SortedSet<String> flags) { SortedSet<UptimeHistory> history = relay ? this.relayHistory @@ -105,6 +108,8 @@ public class UptimeStatus extends Document { this.isDirty = true; } + /** Compresses the history of uptime objects by merging adjacent + * intervals. */ public void compressHistory() { this.compressHistory(this.relayHistory); this.compressHistory(this.bridgeHistory); @@ -137,6 +142,7 @@ public class UptimeStatus extends Document { } } + @Override public String toDocumentString() { StringBuilder sb = new StringBuilder(); for (UptimeHistory interval : this.relayHistory) { diff --git a/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java b/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java index 47eaed9..efb7c25 100644 --- a/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java @@ -46,6 +46,7 @@ public class WeightsStatus extends Document { return this.history; } + @Override public void setFromDocumentString(String documentString) { try (Scanner s = new Scanner(documentString)) { while (s.hasNextLine()) { @@ -90,6 +91,8 @@ public class WeightsStatus extends Document { } } + /** Adds all given weights history objects that don't overlap with + * existing weights history objects. */ public void addToHistory(long validAfterMillis, long freshUntilMillis, double[] weights) { long[] interval = new long[] { validAfterMillis, freshUntilMillis }; @@ -104,6 +107,8 @@ public class WeightsStatus extends Document { } } + /** Compresses the history of weights objects by merging adjacent + * intervals, depending on how far back in the past they lie. */ public void compressHistory() { SortedMap<long[], double[]> uncompressedHistory = new TreeMap<long[], double[]>(this.history); @@ -176,6 +181,7 @@ public class WeightsStatus extends Document { } } + @Override public String toDocumentString() { StringBuilder sb = new StringBuilder(); for (Map.Entry<long[], double[]> e : history.entrySet()) { diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java index 30a6f0e..93b5af7 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java @@ -34,6 +34,7 @@ public class NodeIndexer implements ServletContextListener, Runnable { private static final Logger log = LoggerFactory.getLogger( NodeIndexer.class); + @Override public void contextInitialized(ServletContextEvent contextEvent) { ServletContext servletContext = contextEvent.getServletContext(); File outDir = new File(servletContext.getInitParameter("outDir")); @@ -50,6 +51,7 @@ public class NodeIndexer implements ServletContextListener, Runnable { this.startIndexing(); } + @Override public void contextDestroyed(ServletContextEvent contextEvent) { this.stopIndexing(); } @@ -60,6 +62,9 @@ public class NodeIndexer implements ServletContextListener, Runnable { private Thread nodeIndexerThread = null; + /** Returns the creation time of the last known node index in + * milliseconds since the epoch, or <code>-1</code> if no node index + * could be retrieved within <code>timeoutMillis</code> milliseconds. */ public synchronized long getLastIndexed(long timeoutMillis) { if (this.lastIndexed == -1L && this.nodeIndexerThread != null && timeoutMillis > 0L) { @@ -71,6 +76,8 @@ public class NodeIndexer implements ServletContextListener, Runnable { return this.lastIndexed; } + /** Returns the last known node index, or null if no node index could be + * retrieved within <code>timeoutMillis</code> milliseconds. */ public synchronized NodeIndex getLatestNodeIndex(long timeoutMillis) { if (this.latestNodeIndex == null && this.nodeIndexerThread != null && timeoutMillis > 0L) { @@ -82,6 +89,8 @@ public class NodeIndexer implements ServletContextListener, Runnable { return this.latestNodeIndex; } + /** Start reading the node index into memory periodically in a + * background thread. */ public synchronized void startIndexing() { if (this.nodeIndexerThread == null) { this.nodeIndexerThread = new Thread(this); @@ -94,6 +103,7 @@ public class NodeIndexer implements ServletContextListener, Runnable { private static final long ONE_DAY = 24L * 60L * ONE_MINUTE; + @Override public void run() { while (this.nodeIndexerThread != null) { this.indexNodeStatuses(); @@ -104,6 +114,8 @@ public class NodeIndexer implements ServletContextListener, Runnable { } } + /** Stop the background process that is periodically reading the node + * index. */ public synchronized void stopIndexing() { Thread indexerThread = this.nodeIndexerThread; this.nodeIndexerThread = null; diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexerFactory.java b/src/main/java/org/torproject/onionoo/server/NodeIndexerFactory.java index 1dfa859..b938c9e 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndexerFactory.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndexerFactory.java @@ -7,10 +7,15 @@ public class NodeIndexerFactory { private static NodeIndexer nodeIndexerInstance; + /** Sets a custom singleton node indexer instance that will be returned + * by {@link #getNodeIndexer()} rather than creating an instance upon + * first invocation. */ public static void setNodeIndexer(NodeIndexer nodeIndexer) { nodeIndexerInstance = nodeIndexer; } + /** Returns the singleton node indexer instance that gets created upon + * first invocation of this method. */ public static NodeIndexer getNodeIndexer() { if (nodeIndexerInstance == null) { nodeIndexerInstance = new NodeIndexer(); diff --git a/src/main/java/org/torproject/onionoo/server/PerformanceMetrics.java b/src/main/java/org/torproject/onionoo/server/PerformanceMetrics.java index a12e5c7..7adad76 100644 --- a/src/main/java/org/torproject/onionoo/server/PerformanceMetrics.java +++ b/src/main/java/org/torproject/onionoo/server/PerformanceMetrics.java @@ -30,6 +30,7 @@ class Counter { this.value++; } + @Override public String toString() { return String.valueOf(this.value); } @@ -52,6 +53,7 @@ class MostFrequentString { } } + @Override public String toString() { SortedMap<Integer, SortedSet<String>> sortedFrequencies = new TreeMap<Integer, SortedSet<String>>( @@ -98,6 +100,7 @@ class IntegerDistribution { logValues[64 - Long.numberOfLeadingZeros(value)]++; } + @Override public String toString() { StringBuilder sb = new StringBuilder(); int totalValues = 0; @@ -172,6 +175,9 @@ public class PerformanceMetrics { private static IntegerDistribution buildResponseMillis = new IntegerDistribution(); + /** Collects aggregate statistics on a given request for periodic + * request statistics, and logs requests taking longer than expected to + * process. */ public static void logStatistics(long receivedRequestMillis, String resourceType, Collection<String> parameterKeys, long parsedRequestMillis, int relayDocumentsWritten, diff --git a/src/main/java/org/torproject/onionoo/server/RequestHandler.java b/src/main/java/org/torproject/onionoo/server/RequestHandler.java index 85a6ff1..eaa4fe2 100644 --- a/src/main/java/org/torproject/onionoo/server/RequestHandler.java +++ b/src/main/java/org/torproject/onionoo/server/RequestHandler.java @@ -111,6 +111,7 @@ public class RequestHandler { private int[] firstSeenDays; + @SuppressWarnings("checkstyle:javadocmethod") public void setFirstSeenDays(int[] firstSeenDays) { this.firstSeenDays = new int[firstSeenDays.length]; System.arraycopy(firstSeenDays, 0, this.firstSeenDays, 0, @@ -119,6 +120,7 @@ public class RequestHandler { private int[] lastSeenDays; + @SuppressWarnings("checkstyle:javadocmethod") public void setLastSeenDays(int[] lastSeenDays) { this.lastSeenDays = new int[lastSeenDays.length]; System.arraycopy(lastSeenDays, 0, this.lastSeenDays, 0, @@ -137,6 +139,8 @@ public class RequestHandler { private Map<String, SummaryDocument> filteredBridges = new HashMap<String, SummaryDocument>(); + /** Handles this request by filtering by all given parameters and then + * possibly ordering, offsetting, and limiting results. */ public void handleRequest() { this.filteredRelays.putAll( this.nodeIndex.getRelayFingerprintSummaryLines()); diff --git a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java index 4fd38b7..5b3ab69 100644 --- a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java +++ b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java @@ -31,6 +31,7 @@ public class ResourceServlet extends HttpServlet { private boolean maintenanceMode = false; /* Called by servlet container, not by test class. */ + @Override public void init(ServletConfig config) throws ServletException { super.init(config); this.maintenanceMode = config.getInitParameter("maintenance") != null @@ -39,6 +40,7 @@ public class ResourceServlet extends HttpServlet { private static final long INDEX_WAITING_TIME = 10L * 1000L; + @Override public long getLastModified(HttpServletRequest request) { if (this.maintenanceMode) { return super.getLastModified(request); @@ -48,6 +50,7 @@ public class ResourceServlet extends HttpServlet { } } + @Override public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { HttpServletRequestWrapper requestWrapper = @@ -72,6 +75,9 @@ public class ResourceServlet extends HttpServlet { new HashSet<String>(Arrays.asList(("search,fingerprint,order,limit," + "offset,fields").split(","))); + /** Handles the HTTP GET request in the wrapped <code>request</code> by + * writing an HTTP GET response to the likewise <code>response</code>, + * both of which are wrapped to facilitate testing. */ public void doGet(HttpServletRequestWrapper request, HttpServletResponseWrapper response) throws IOException { diff --git a/src/main/java/org/torproject/onionoo/server/ServerMain.java b/src/main/java/org/torproject/onionoo/server/ServerMain.java index 22e315b..fd4dc90 100644 --- a/src/main/java/org/torproject/onionoo/server/ServerMain.java +++ b/src/main/java/org/torproject/onionoo/server/ServerMain.java @@ -14,6 +14,7 @@ public class ServerMain { private static final Logger log = LoggerFactory.getLogger( ServerMain.class); + /** Starts the web server listening for incoming client connections. */ public static void main(String[] args) { try { Resource onionooXml = Resource.newSystemResource("jetty.xml"); diff --git a/src/main/java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java index 1c0c181..3aa3dd1 100644 --- a/src/main/java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java @@ -16,6 +16,9 @@ public class BandwidthStatusUpdater implements DescriptorListener, private DocumentStore documentStore; + /** Initializes a new status updater, obtains references to all relevant + * singleton instances, and registers as listener at the (singleton) + * descriptor source. */ public BandwidthStatusUpdater() { this.descriptorSource = DescriptorSourceFactory.getDescriptorSource(); this.documentStore = DocumentStoreFactory.getDocumentStore(); @@ -29,12 +32,14 @@ public class BandwidthStatusUpdater implements DescriptorListener, DescriptorType.BRIDGE_EXTRA_INFOS); } + @Override public void processDescriptor(Descriptor descriptor, boolean relay) { if (descriptor instanceof ExtraInfoDescriptor) { this.parseDescriptor((ExtraInfoDescriptor) descriptor); } } + @Override public void updateStatuses() { /* Status files are already updated while processing descriptors. */ } @@ -59,6 +64,7 @@ public class BandwidthStatusUpdater implements DescriptorListener, } } + @Override public String getStatsString() { /* TODO Add statistics string. */ return null; diff --git a/src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java index 492b67b..28b8ea7 100644 --- a/src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java @@ -43,6 +43,9 @@ public class ClientsStatusUpdater implements DescriptorListener, private DocumentStore documentStore; + /** Initializes a new status updater, obtains references to all relevant + * singleton instances, and registers as listener at the (singleton) + * descriptor source. */ public ClientsStatusUpdater() { this.descriptorSource = DescriptorSourceFactory.getDescriptorSource(); this.documentStore = DocumentStoreFactory.getDocumentStore(); @@ -54,6 +57,7 @@ public class ClientsStatusUpdater implements DescriptorListener, DescriptorType.BRIDGE_EXTRA_INFOS); } + @Override public void processDescriptor(Descriptor descriptor, boolean relay) { if (descriptor instanceof ExtraInfoDescriptor && !relay) { this.processBridgeExtraInfoDescriptor( @@ -143,6 +147,7 @@ public class ClientsStatusUpdater implements DescriptorListener, return weightedResponses; } + @Override public void updateStatuses() { for (Map.Entry<String, SortedSet<ClientsHistory>> e : this.newResponses.entrySet()) { @@ -161,6 +166,7 @@ public class ClientsStatusUpdater implements DescriptorListener, } } + @Override public String getStatsString() { int newIntervals = 0; for (SortedSet<ClientsHistory> hist : this.newResponses.values()) { diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java index b9d07b4..176a17c 100644 --- a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java +++ b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java @@ -32,6 +32,7 @@ public class DescriptorSource { private DescriptorQueue archiveDescriptorQueue; + /** Instantiates a new descriptor source. */ public DescriptorSource() { this.descriptorQueues = new ArrayList<DescriptorQueue>(); this.descriptorListeners = @@ -53,6 +54,7 @@ public class DescriptorSource { private Map<DescriptorType, Set<DescriptorListener>> descriptorListeners; + /** Registers a descriptor listener for a given descriptor type. */ public void registerDescriptorListener(DescriptorListener listener, DescriptorType descriptorType) { if (!this.descriptorListeners.containsKey(descriptorType)) { @@ -62,6 +64,7 @@ public class DescriptorSource { this.descriptorListeners.get(descriptorType).add(listener); } + /** Downloads descriptors from CollecTor. */ public void downloadDescriptors() { for (DescriptorType descriptorType : DescriptorType.values()) { log.info("Loading: " + descriptorType); @@ -87,6 +90,8 @@ public class DescriptorSource { this.deletedLocalFiles += descriptorDownloader.deleteOldLocalFiles(); } + /** Reads archived and recent descriptors from disk and feeds them into + * any registered listeners. */ public void readDescriptors() { this.readArchivedDescriptors(); log.debug("Reading recent " + DescriptorType.RELAY_SERVER_DESCRIPTORS @@ -154,6 +159,8 @@ public class DescriptorSource { } } + /** Reads archived descriptors from disk and feeds them into any + * registered listeners. */ public void readArchivedDescriptors() { if (!this.inArchiveDir.exists()) { return; @@ -208,6 +215,7 @@ public class DescriptorSource { log.info("Read archived descriptors"); } + /** Writes parse histories for recent descriptors to disk. */ public void writeHistoryFiles() { log.debug("Writing parse histories for recent descriptors..."); for (DescriptorQueue descriptorQueue : this.descriptorQueues) { @@ -215,6 +223,8 @@ public class DescriptorSource { } } + /** Returns a string with statistics on the number of processed + * descriptors during the current execution. */ public String getStatsString() { StringBuilder sb = new StringBuilder(); sb.append(" " + this.localFilesBefore + " recent descriptor files " diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorSourceFactory.java b/src/main/java/org/torproject/onionoo/updater/DescriptorSourceFactory.java index 65c5622..4974488 100644 --- a/src/main/java/org/torproject/onionoo/updater/DescriptorSourceFactory.java +++ b/src/main/java/org/torproject/onionoo/updater/DescriptorSourceFactory.java @@ -7,11 +7,16 @@ public class DescriptorSourceFactory { private static DescriptorSource descriptorSourceInstance; + /** Sets a custom singleton descriptor source instance that will be + * returned by {@link #getDescriptorSource()} rather than creating an + * instance upon first invocation. */ public static void setDescriptorSource( DescriptorSource descriptorSource) { descriptorSourceInstance = descriptorSource; } + /** Returns the singleton descriptor source instance that gets created + * upon first invocation of this method. */ public static DescriptorSource getDescriptorSource() { if (descriptorSourceInstance == null) { descriptorSourceInstance = new DescriptorSource(); diff --git a/src/main/java/org/torproject/onionoo/updater/LookupService.java b/src/main/java/org/torproject/onionoo/updater/LookupService.java index 85d56e8..2b0993f 100644 --- a/src/main/java/org/torproject/onionoo/updater/LookupService.java +++ b/src/main/java/org/torproject/onionoo/updater/LookupService.java @@ -97,6 +97,9 @@ public class LookupService { return addressNumber; } + /** Looks up address strings in the configured + * <code>GeoLite2-City-*.csv</code> and <code>GeoIPASNum2.csv</code> + * files and returns all lookup results. */ public SortedMap<String, LookupResult> lookup( SortedSet<String> addressStrings) { @@ -363,6 +366,8 @@ public class LookupService { private int addressesResolved = 0; + /** Returns a string with the number of addresses looked up and + * resolved. */ public String getStatsString() { StringBuilder sb = new StringBuilder(); sb.append(" " + FormattingUtils.formatDecimalNumber( diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java index 9bceaa1..d873072 100644 --- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java @@ -95,6 +95,9 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, private int bridgeStatusesProcessed = 0; + /** Initializes a new status updater, obtains references to all relevant + * singleton instances, and registers as listener at the (singleton) + * descriptor source. */ public NodeDetailsStatusUpdater( ReverseDomainNameResolver reverseDomainNameResolver, LookupService lookupService) { @@ -125,6 +128,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, private SortedSet<String> updatedNodes = new TreeSet<String>(); + @Override public void processDescriptor(Descriptor descriptor, boolean relay) { if (descriptor instanceof ServerDescriptor && relay) { this.processRelayServerDescriptor((ServerDescriptor) descriptor); @@ -385,6 +389,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, this.bridgeStatusesProcessed++; } + @Override public void updateStatuses() { this.readNodeStatuses(); log.info("Read node statuses"); @@ -927,6 +932,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, } } + @Override public String getStatsString() { StringBuilder sb = new StringBuilder(); sb.append(" " + FormattingUtils.formatDecimalNumber( diff --git a/src/main/java/org/torproject/onionoo/updater/RdnsLookupRequest.java b/src/main/java/org/torproject/onionoo/updater/RdnsLookupRequest.java index e2ee5e1..75ad315 100644 --- a/src/main/java/org/torproject/onionoo/updater/RdnsLookupRequest.java +++ b/src/main/java/org/torproject/onionoo/updater/RdnsLookupRequest.java @@ -28,6 +28,7 @@ class RdnsLookupRequest extends Thread { this.address = address; } + @Override public void run() { this.lookupStartedMillis = this.reverseDomainNameResolver.time.currentTimeMillis(); diff --git a/src/main/java/org/torproject/onionoo/updater/RdnsLookupWorker.java b/src/main/java/org/torproject/onionoo/updater/RdnsLookupWorker.java index 0c48be5..165507b 100644 --- a/src/main/java/org/torproject/onionoo/updater/RdnsLookupWorker.java +++ b/src/main/java/org/torproject/onionoo/updater/RdnsLookupWorker.java @@ -11,6 +11,7 @@ class RdnsLookupWorker extends Thread { this.reverseDomainNameResolver = reverseDomainNameResolver; } + @Override public void run() { while (this.reverseDomainNameResolver.time.currentTimeMillis() - ReverseDomainNameResolver.RDNS_LOOKUP_MAX_DURATION_MILLIS diff --git a/src/main/java/org/torproject/onionoo/updater/ReverseDomainNameResolver.java b/src/main/java/org/torproject/onionoo/updater/ReverseDomainNameResolver.java index 0f3caad..3214c74 100644 --- a/src/main/java/org/torproject/onionoo/updater/ReverseDomainNameResolver.java +++ b/src/main/java/org/torproject/onionoo/updater/ReverseDomainNameResolver.java @@ -48,6 +48,8 @@ public class ReverseDomainNameResolver { this.addressLastLookupTimes = addressLastLookupTimes; } + /** Starts reverse domain name lookups in one or more background + * threads and returns immediately. */ public void startReverseDomainNameLookups() { this.startedRdnsLookups = this.time.currentTimeMillis(); this.rdnsLookupJobs = new HashSet<String>(); @@ -69,6 +71,8 @@ public class ReverseDomainNameResolver { } } + /** Joins all background threads performing reverse domain name lookups + * and returns as soon as they have all finished. */ public void finishReverseDomainNameLookups() { for (RdnsLookupWorker rdnsLookupWorker : this.rdnsLookupWorkers) { try { @@ -80,16 +84,21 @@ public class ReverseDomainNameResolver { } } + /** Returns reverse domain name lookup results. */ public Map<String, String> getLookupResults() { synchronized (this.rdnsLookupResults) { return new HashMap<String, String>(this.rdnsLookupResults); } } + /** Returns the time in milliseconds since the epoch when reverse domain + * lookups have been started. */ public long getLookupStartMillis() { return this.startedRdnsLookups; } + /** Returns a string with the number of performed reverse domain name + * lookups and some simple statistics on lookup time. */ public String getStatsString() { StringBuilder sb = new StringBuilder(); sb.append(" " + FormattingUtils.formatDecimalNumber( diff --git a/src/main/java/org/torproject/onionoo/updater/StatusUpdateRunner.java b/src/main/java/org/torproject/onionoo/updater/StatusUpdateRunner.java index 2ba8401..7aa1a5e 100644 --- a/src/main/java/org/torproject/onionoo/updater/StatusUpdateRunner.java +++ b/src/main/java/org/torproject/onionoo/updater/StatusUpdateRunner.java @@ -19,6 +19,8 @@ public class StatusUpdateRunner { private StatusUpdater[] statusUpdaters; + /** Instantiates a new status update runner with newly created instances + * of all known status updater implementations. */ public StatusUpdateRunner() { this.ls = new LookupService(new File("geoip")); this.rdnr = new ReverseDomainNameResolver(); @@ -32,6 +34,7 @@ public class StatusUpdateRunner { usu }; } + /** Lets each configured status updater update its status files. */ public void updateStatuses() { for (StatusUpdater su : this.statusUpdaters) { log.debug("Begin update of " + su.getClass().getSimpleName()); @@ -41,6 +44,7 @@ public class StatusUpdateRunner { } } + /** Logs statistics of all configured status updaters. */ public void logStatistics() { for (StatusUpdater su : this.statusUpdaters) { String statsString = su.getStatsString(); diff --git a/src/main/java/org/torproject/onionoo/updater/UptimeStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/UptimeStatusUpdater.java index f2d7dc1..d4951b7 100644 --- a/src/main/java/org/torproject/onionoo/updater/UptimeStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/UptimeStatusUpdater.java @@ -28,6 +28,9 @@ public class UptimeStatusUpdater implements DescriptorListener, private DocumentStore documentStore; + /** Initializes a new status updater, obtains references to all relevant + * singleton instances, and registers as listener at the (singleton) + * descriptor source. */ public UptimeStatusUpdater() { this.descriptorSource = DescriptorSourceFactory.getDescriptorSource(); this.documentStore = DocumentStoreFactory.getDocumentStore(); @@ -41,6 +44,7 @@ public class UptimeStatusUpdater implements DescriptorListener, DescriptorType.BRIDGE_STATUSES); } + @Override public void processDescriptor(Descriptor descriptor, boolean relay) { if (descriptor instanceof RelayNetworkStatusConsensus) { this.processRelayNetworkStatusConsensus( @@ -132,6 +136,7 @@ public class UptimeStatusUpdater implements DescriptorListener, } } + @Override public void updateStatuses() { for (Map.Entry<String, SortedMap<Long, Flags>> e : this.newRunningRelays.entrySet()) { @@ -179,6 +184,7 @@ public class UptimeStatusUpdater implements DescriptorListener, } } + @Override public String getStatsString() { StringBuilder sb = new StringBuilder(); sb.append(" " + FormattingUtils.formatDecimalNumber( diff --git a/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java index 14e425b..2442a00 100644 --- a/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java @@ -24,6 +24,9 @@ public class WeightsStatusUpdater implements DescriptorListener, private DocumentStore documentStore; + /** Initializes a new status updater, obtains references to all relevant + * singleton instances, and registers as listener at the (singleton) + * descriptor source. */ public WeightsStatusUpdater() { this.descriptorSource = DescriptorSourceFactory.getDescriptorSource(); this.documentStore = DocumentStoreFactory.getDocumentStore(); @@ -35,6 +38,7 @@ public class WeightsStatusUpdater implements DescriptorListener, DescriptorType.RELAY_CONSENSUSES); } + @Override public void processDescriptor(Descriptor descriptor, boolean relay) { if (descriptor instanceof RelayNetworkStatusConsensus) { this.processRelayNetworkConsensus( @@ -42,6 +46,7 @@ public class WeightsStatusUpdater implements DescriptorListener, } } + @Override public void updateStatuses() { /* Nothing to do. */ } @@ -193,6 +198,7 @@ public class WeightsStatusUpdater implements DescriptorListener, return pathSelectionProbabilities; } + @Override public String getStatsString() { /* TODO Add statistics string. */ return null; diff --git a/src/main/java/org/torproject/onionoo/util/FormattingUtils.java b/src/main/java/org/torproject/onionoo/util/FormattingUtils.java index 36fb838..fd83cf8 100644 --- a/src/main/java/org/torproject/onionoo/util/FormattingUtils.java +++ b/src/main/java/org/torproject/onionoo/util/FormattingUtils.java @@ -12,11 +12,14 @@ public class FormattingUtils { private static final long ONE_MINUTE = 60L * ONE_SECOND; + /** Formats the given number of milliseconds using the format + * <code>"${minutes}:${seconds}.{milliseconds} minutes"</code>. */ public static String formatMillis(long millis) { return String.format("%02d:%02d.%03d minutes", millis / ONE_MINUTE, (millis % ONE_MINUTE) / ONE_SECOND, millis % ONE_SECOND); } + /** Formats the given number of bytes as B, KiB, MiB, GiB, etc. */ public static String formatBytes(long bytes) { if (bytes < 1024) { return bytes + " B"; @@ -27,6 +30,8 @@ public class FormattingUtils { } } + /** Formats the given decimal number with a comma as thousands + * separator. */ public static String formatDecimalNumber(long decimalNumber) { return String.format("%,d", decimalNumber); } diff --git a/src/main/java/org/torproject/onionoo/util/TimeFactory.java b/src/main/java/org/torproject/onionoo/util/TimeFactory.java index e3067a4..608b595 100644 --- a/src/main/java/org/torproject/onionoo/util/TimeFactory.java +++ b/src/main/java/org/torproject/onionoo/util/TimeFactory.java @@ -7,10 +7,15 @@ public class TimeFactory { private static Time timeInstance; + /** Sets a custom singleton time instance that will be returned by + * {@link #getTime} rather than creating an instance upon first + * invocation. */ public static void setTime(Time time) { timeInstance = time; } + /** Returns the singleton node indexer instance that gets created upon + * first invocation of this method. */ public static Time getTime() { if (timeInstance == null) { timeInstance = new Time(); diff --git a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java index a844f7a..7238c1b 100644 --- a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java @@ -36,6 +36,7 @@ public class BandwidthDocumentWriter implements DocumentWriter { this.now = TimeFactory.getTime().currentTimeMillis(); } + @Override public void writeDocuments() { UpdateStatus updateStatus = this.documentStore.retrieve( UpdateStatus.class, true); @@ -196,6 +197,7 @@ public class BandwidthDocumentWriter implements DocumentWriter { return graphs; } + @Override public String getStatsString() { /* TODO Add statistics string. */ return null; diff --git a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java index 6cbd2e0..00389d0 100644 --- a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java @@ -64,6 +64,7 @@ public class ClientsDocumentWriter implements DocumentWriter { private int writtenDocuments = 0; + @Override public void writeDocuments() { UpdateStatus updateStatus = this.documentStore.retrieve( UpdateStatus.class, true); @@ -286,6 +287,7 @@ public class ClientsDocumentWriter implements DocumentWriter { } } + @Override public String getStatsString() { StringBuilder sb = new StringBuilder(); sb.append(" " + FormattingUtils.formatDecimalNumber( diff --git a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java index 44c675f..c167152 100644 --- a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java @@ -31,6 +31,7 @@ public class DetailsDocumentWriter implements DocumentWriter { this.documentStore = DocumentStoreFactory.getDocumentStore(); } + @Override public void writeDocuments() { UpdateStatus updateStatus = this.documentStore.retrieve( UpdateStatus.class, true); @@ -186,6 +187,7 @@ public class DetailsDocumentWriter implements DocumentWriter { this.documentStore.store(detailsDocument, fingerprint); } + @Override public String getStatsString() { /* TODO Add statistics string. */ return null; diff --git a/src/main/java/org/torproject/onionoo/writer/DocumentWriterRunner.java b/src/main/java/org/torproject/onionoo/writer/DocumentWriterRunner.java index 4809ed9..d9d1c47 100644 --- a/src/main/java/org/torproject/onionoo/writer/DocumentWriterRunner.java +++ b/src/main/java/org/torproject/onionoo/writer/DocumentWriterRunner.java @@ -13,6 +13,8 @@ public class DocumentWriterRunner { private DocumentWriter[] documentWriters; + /** Instantiates a new document writer runner with newly created + * instances of all known document writer implementations. */ public DocumentWriterRunner() { SummaryDocumentWriter sdw = new SummaryDocumentWriter(); DetailsDocumentWriter ddw = new DetailsDocumentWriter(); @@ -24,6 +26,7 @@ public class DocumentWriterRunner { udw }; } + /** Lets each configured document writer write its documents. */ public void writeDocuments() { for (DocumentWriter dw : this.documentWriters) { log.debug("Writing " + dw.getClass().getSimpleName()); @@ -31,6 +34,7 @@ public class DocumentWriterRunner { } } + /** Logs statistics of all configured document writers. */ public void logStatistics() { for (DocumentWriter dw : this.documentWriters) { String statsString = dw.getStatsString(); diff --git a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java index f0d7fe0..f941ee3 100644 --- a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java @@ -32,6 +32,7 @@ public class SummaryDocumentWriter implements DocumentWriter { private int deletedDocuments = 0; + @Override public void writeDocuments() { long relaysLastValidAfterMillis = -1L; long bridgesLastPublishedMillis = -1L; @@ -101,6 +102,7 @@ public class SummaryDocumentWriter implements DocumentWriter { log.info("Wrote summary document files"); } + @Override public String getStatsString() { StringBuilder sb = new StringBuilder(); sb.append(" " + FormattingUtils.formatDecimalNumber( diff --git a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java index e93627b..502c351 100644 --- a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java @@ -40,6 +40,7 @@ public class UptimeDocumentWriter implements DocumentWriter { this.now = TimeFactory.getTime().currentTimeMillis(); } + @Override public void writeDocuments() { UptimeStatus uptimeStatus = this.documentStore.retrieve( UptimeStatus.class, true); @@ -318,6 +319,7 @@ public class UptimeDocumentWriter implements DocumentWriter { } } + @Override public String getStatsString() { StringBuilder sb = new StringBuilder(); sb.append(" " + FormattingUtils.formatDecimalNumber( diff --git a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java index cabe964..326fbda 100644 --- a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java @@ -36,6 +36,7 @@ public class WeightsDocumentWriter implements DocumentWriter { this.now = TimeFactory.getTime().currentTimeMillis(); } + @Override public void writeDocuments() { UpdateStatus updateStatus = this.documentStore.retrieve( UpdateStatus.class, true); @@ -210,6 +211,7 @@ public class WeightsDocumentWriter implements DocumentWriter { } } + @Override public String getStatsString() { /* TODO Add statistics string. */ return null; diff --git a/src/test/resources/metrics_checks.xml b/src/test/resources/metrics_checks.xml index 0777894..a4af08a 100644 --- a/src/test/resources/metrics_checks.xml +++ b/src/test/resources/metrics_checks.xml @@ -34,6 +34,7 @@ <property name="eachLine" value="true"/> </module> + <module name="SuppressWarningsFilter" /> <module name="TreeWalker"> <module name="OuterTypeFilename"/> <module name="IllegalTokenText"> @@ -213,5 +214,6 @@ <property name="exceptionVariableName" value="expected"/> </module> <module name="CommentsIndentation"/> + <module name="SuppressWarningsHolder" /> </module> </module>

1 0

[tor/master] Also ship compat_time.h in release tarballs. Fixes bug #19746
by nickm＠torproject.org 25 Jul '16

25 Jul '16

commit 518c8fe0ec375fe852e350ef4455ec8defd8c6b0 Author: Peter Palfrader <peter(a)palfrader.org> Date: Mon Jul 25 09:07:29 2016 +0200 Also ship compat_time.h in release tarballs. Fixes bug #19746 --- src/common/include.am | 1 + 1 file changed, 1 insertion(+) diff --git a/src/common/include.am b/src/common/include.am index b022680..40c463c 100644 --- a/src/common/include.am +++ b/src/common/include.am @@ -142,6 +142,7 @@ COMMONHEADERS = \ src/common/compat_libevent.h \ src/common/compat_openssl.h \ src/common/compat_threads.h \ + src/common/compat_time.h \ src/common/container.h \ src/common/crypto.h \ src/common/crypto_curve25519.h \

1 0

[stem/master] Add encoding() method to stem.util.term
by atagar＠torproject.org 24 Jul '16

24 Jul '16

commit 50d884e9c9a732210825bd9079c8303ef65d10aa Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jul 24 09:22:25 2016 -0700 Add encoding() method to stem.util.term Breaking up our format() method a little by adding a helper to get ANSI escape sequences. This is something I need for nyx right now anyway... Oddly our stem.util.term module didn't have any direct test coverage. It was tested tangentially by our testing harness and interpreter tests but strange I missed unit test coverage for this one. It's an easy module to include. Oh well - threw in some basic tests. --- docs/change_log.rst | 1 + stem/util/term.py | 57 +++++++++++++++++++++++++++++++++----------------- test/settings.cfg | 1 + test/unit/util/term.py | 30 ++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 19 deletions(-) diff --git a/docs/change_log.rst b/docs/change_log.rst index 98dc18c..839c7c9 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -94,6 +94,7 @@ The following are only available within Stem's `git repository * Added an **is_ipv6** value to :class:`~stem.util.connection.Connection` instances * Added :func:`~stem.util.system.pids_by_user` * Added :func:`~stem.util.connection.address_to_int` + * Added :func:`~stem.util.term.encoding` * Added :func:`~stem.util.__init__.datetime_to_unix` * **Interpreter** diff --git a/stem/util/term.py b/stem/util/term.py index 3287bb1..a32fe93 100644 --- a/stem/util/term.py +++ b/stem/util/term.py @@ -8,6 +8,7 @@ Utilities for working with the terminal. :: + encoding - provides the ANSI escape sequence for a terminal attribute format - wrap text with ANSI for the given colors or attributes .. data:: Color (enum) @@ -65,6 +66,41 @@ CSI = '\x1B[%sm' RESET = CSI % '0' +def encoding(*attrs): + """ + Provides the ANSI escape sequence for these terminal color or attributes. + + .. versionadded:: 1.5.0 + + :param list attr: :data:`~stem.util.terminal.Color`, + :data:`~stem.util.terminal.BgColor`, or :data:`~stem.util.terminal.Attr` to + provide an ecoding for + + :return: **str** of the ANSI escape sequence, **None** no attributes are + recognized + """ + + term_encodings = [] + + for attr in attrs: + # TODO: Account for an earlier misspelled attribute. This should be dropped + # in Stem. 2.0.x. + + if attr == 'HILIGHT': + attr = 'HIGHLIGHT' + + attr = stem.util.str_tools._to_camel_case(attr) + term_encoding = FG_ENCODING.get(attr, None) + term_encoding = BG_ENCODING.get(attr, term_encoding) + term_encoding = ATTR_ENCODING.get(attr, term_encoding) + + if term_encoding: + term_encodings.append(term_encoding) + + if term_encodings: + return CSI % ';'.join(term_encodings) + + def format(msg, *attr): """ Simple terminal text formatting using `ANSI escape sequences @@ -93,26 +129,9 @@ def format(msg, *attr): if RESET in msg: return ''.join([format(comp, *attr) for comp in msg.split(RESET)]) - encodings = [] - - for text_attr in attr: - # TODO: Account for an earlier misspelled attribute. This should be dropped - # in Stem. 2.0.x. - - if text_attr == 'HILIGHT': - text_attr = 'HIGHLIGHT' - - text_attr, encoding = stem.util.str_tools._to_camel_case(text_attr), None - encoding = FG_ENCODING.get(text_attr, encoding) - encoding = BG_ENCODING.get(text_attr, encoding) - encoding = ATTR_ENCODING.get(text_attr, encoding) - - if encoding: - encodings.append(encoding) - - if encodings: - prefix, suffix = CSI % ';'.join(encodings), RESET + prefix, suffix = encoding(*attr), RESET + if prefix: if Attr.READLINE_ESCAPE in attr: prefix = '\001%s\002' % prefix suffix = '\001%s\002' % suffix diff --git a/test/settings.cfg b/test/settings.cfg index fd199e1..10e2ccf 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -167,6 +167,7 @@ test.unit_tests |test.unit.util.proc.TestProc |test.unit.util.str_tools.TestStrTools |test.unit.util.system.TestSystem +|test.unit.util.term.TestTerminal |test.unit.util.tor_tools.TestTorTools |test.unit.util.__init__.TestBaseUtil |test.unit.descriptor.export.TestExport diff --git a/test/unit/util/term.py b/test/unit/util/term.py new file mode 100644 index 0000000..758db22 --- /dev/null +++ b/test/unit/util/term.py @@ -0,0 +1,30 @@ +""" +Unit tests for the stem.util.term functions. +""" + +import unittest + +import stem.util.term + +from stem.util.term import Color, Attr + + +class TestTerminal(unittest.TestCase): + def test_encoding(self): + """ + Exercises our encoding function. + """ + + self.assertEqual(None, stem.util.term.encoding()) + self.assertEqual('\x1b[31m', stem.util.term.encoding(Color.RED)) + self.assertEqual('\x1b[31;1m', stem.util.term.encoding(Color.RED, Attr.BOLD)) + + def test_format(self): + """ + Exercises our format function. + """ + + self.assertEqual('hi!', stem.util.term.format('hi!')) + self.assertEqual('\x1b[31mhi!\x1b[0m', stem.util.term.format('hi!', Color.RED)) + self.assertEqual('\x1b[31;1mhi!\x1b[0m', stem.util.term.format('hi!', Color.RED, Attr.BOLD)) + self.assertEqual('\001\x1b[31m\002hi!\001\x1b[0m\002', stem.util.term.format('hi!', Color.RED, Attr.READLINE_ESCAPE))

1 0

[tor-browser-bundle/maint-6.0] Release preparations for 6.0.3
by gk＠torproject.org 22 Jul '16

22 Jul '16

commit 033fe948a692f545d50132ebe2046704be5dff7d Author: Georg Koppen <gk(a)torproject.org> Date: Fri Jul 22 16:40:03 2016 +0000 Release preparations for 6.0.3 --- Bundle-Data/Docs/ChangeLog.txt | 18 ++++++++++++++++++ gitian/versions | 2 +- tools/update-responses/config.yml | 13 ++++++------- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/Bundle-Data/Docs/ChangeLog.txt b/Bundle-Data/Docs/ChangeLog.txt index 6325c23..2abd47e 100644 --- a/Bundle-Data/Docs/ChangeLog.txt +++ b/Bundle-Data/Docs/ChangeLog.txt @@ -1,3 +1,21 @@ +Tor Browser 6.0.3 -- August 2 + * All Platforms + * Update Firefox to 45.3.0esr + * Update Torbutton to 1.9.5.6 + * Bug 19417: Disable asmjs for now + * Bug 19689: Use proper parent window for plugin prompt + * Update HTTPS-Everywhere to 5.2.1 + * Bug 19417: Disable asmjs for now + * Bug 19715: Disable the meek-google pluggable transport option + * Bug 19714: Remove mercurius4 obfs4 bridge + * Bug 19585: Fix regression test for keyboard layout fingerprinting + * Bug 19515: Tor Browser is crashing in graphics code + * Bug 18513: Favicon requests can bypass New Identity + * OS X + * Bug 19269: Icon doesn't appear in Applications folder or Dock + * Android + * Bug 19484: Avoid compilation error when MOZ_UPDATER is not defined + Tor Browser 6.0.2 -- June 21 * All Platforms * Update Torbutton to 1.9.5.5 diff --git a/gitian/versions b/gitian/versions index 6e54edd..c46c735 100755 --- a/gitian/versions +++ b/gitian/versions @@ -18,7 +18,7 @@ TORBROWSER_TAG=tor-browser-${FIREFOX_VERSION}-6.0-1-build2 TOR_TAG=tor-0.2.7.6 TORLAUNCHER_TAG=0.2.9.3 TORBUTTON_TAG=1.9.5.5 -HTTPSE_TAG=5.1.9 +HTTPSE_TAG=5.2.1 NSIS_TAG=v0.3.1 ZLIB_TAG=v1.2.8 LIBEVENT_TAG=release-2.0.22-stable diff --git a/tools/update-responses/config.yml b/tools/update-responses/config.yml index 4e89b8a..16843df 100644 --- a/tools/update-responses/config.yml +++ b/tools/update-responses/config.yml @@ -13,15 +13,14 @@ build_targets: osx64: Darwin_x86_64-gcc3 channels: alpha: 6.0a5 - release: 6.0.2 + release: 6.0.3 versions: - 6.0.2: - platformVersion: 45.2.0 - detailsURL: https://blog.torproject.org/blog/tor-browser-602-released - download_url: https://cdn.torproject.org/aus1/torbrowser/6.0.2 + 6.0.3: + platformVersion: 45.3.0 + detailsURL: https://blog.torproject.org/blog/tor-browser-603-released + download_url: https://cdn.torproject.org/aus1/torbrowser/6.0.3 incremental_from: - - 6.0 - - 6.0.1 + - 6.0.2 migrate_archs: osx32: osx64 osx32:

1 0

[torbutton/maint-1.9.5] Release preparations for 1.9.5.6
by gk＠torproject.org 22 Jul '16

22 Jul '16

commit 7b983fe74fca4bc29d68e0a4ed3c2eeae2ba1113 Author: Georg Koppen <gk(a)torproject.org> Date: Fri Jul 22 14:33:41 2016 +0000 Release preparations for 1.9.5.6 --- src/CHANGELOG | 4 ++++ src/install.rdf | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/CHANGELOG b/src/CHANGELOG index 77ba290..28cff54 100644 --- a/src/CHANGELOG +++ b/src/CHANGELOG @@ -1,3 +1,7 @@ +1.9.5.6 + * Bug 19417: Disable asmjs for now + * Bug 19689: Use proper parent windows for plugin prompt + 1.9.5.5 * Bug 19417: Clear asmjscache diff --git a/src/install.rdf b/src/install.rdf index ac45b1d..9098991 100644 --- a/src/install.rdf +++ b/src/install.rdf @@ -6,7 +6,7 @@ <em:name>Torbutton</em:name> <em:creator>Mike Perry</em:creator> <em:id>torbutton(a)torproject.org</em:id> - <em:version>1.9.5.5</em:version> + <em:version>1.9.5.6</em:version> <em:homepageURL>https://www.torproject.org/projects/torbrowser.html.en</em:homepageURL> <em:optionsURL>chrome://torbutton/content/preferences.xul</em:optionsURL> <em:iconURL>chrome://torbutton/skin/tor.png</em:iconURL>

1 0

[collector/master] Implements task-19727: make exitlist url configurable and correct default setting.
by karsten＠torproject.org 22 Jul '16

22 Jul '16

commit 7dc17f8e14b3e87f26bd34e1d7c4649546e3476a Author: iwakeh <iwakeh(a)torproject.org> Date: Fri Jul 22 09:13:28 2016 +0200 Implements task-19727: make exitlist url configurable and correct default setting. --- .../org/torproject/collector/conf/Configuration.java | 16 ++++++++++++++++ src/main/java/org/torproject/collector/conf/Key.java | 2 ++ .../collector/exitlists/ExitListDownloader.java | 4 +--- src/main/resources/collector.properties | 1 + .../org/torproject/collector/conf/ConfigurationTest.java | 9 ++++++++- 5 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/torproject/collector/conf/Configuration.java b/src/main/java/org/torproject/collector/conf/Configuration.java index 4e05ad4..9295811 100644 --- a/src/main/java/org/torproject/collector/conf/Configuration.java +++ b/src/main/java/org/torproject/collector/conf/Configuration.java @@ -3,6 +3,8 @@ package org.torproject.collector.conf; +import java.net.MalformedURLException; +import java.net.URL; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Properties; @@ -113,4 +115,18 @@ public class Configuration extends Properties { } } + /** + * Returns a {@code URL} property, e.g. + * {@code urlProperty = https://my.url.here}. + */ + public URL getUrl(Key key) throws ConfigurationException { + try { + checkClass(key, URL.class); + return new URL(getProperty(key.name())); + } catch (MalformedURLException mue) { + throw new ConfigurationException("Corrupt property: " + key + + " reason: " + mue.getMessage(), mue); + } + } + } diff --git a/src/main/java/org/torproject/collector/conf/Key.java b/src/main/java/org/torproject/collector/conf/Key.java index 4f5df64..b4119b6 100644 --- a/src/main/java/org/torproject/collector/conf/Key.java +++ b/src/main/java/org/torproject/collector/conf/Key.java @@ -1,5 +1,6 @@ package org.torproject.collector.conf; +import java.net.URL; import java.nio.file.Path; /** @@ -9,6 +10,7 @@ import java.nio.file.Path; public enum Key { ExitlistOutputDirectory(Path.class), + ExitlistUrl(URL.class), InstanceBaseUrl(String.class), LockFilePath(Path.class), ArchivePath(Path.class), diff --git a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java index 70de74f..79fe19f 100644 --- a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java +++ b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java @@ -84,9 +84,7 @@ public class ExitListDownloader extends Thread { sb.append("@type tordnsel 1.0\n"); sb.append("Downloaded " + dateTimeFormat.format(downloadedDate) + "\n"); - String exitAddressesUrl = - "http://exitlist.torproject.org/exit-addresses"; - URL url = new URL(exitAddressesUrl); + URL url = config.getUrl(Key.ExitlistUrl); HttpURLConnection huc = (HttpURLConnection) url.openConnection(); huc.setRequestMethod("GET"); huc.connect(); diff --git a/src/main/resources/collector.properties b/src/main/resources/collector.properties index fb9e1b4..76ae6bd 100644 --- a/src/main/resources/collector.properties +++ b/src/main/resources/collector.properties @@ -98,6 +98,7 @@ SanitizedBridgesWriteDirectory = out/bridge-descriptors/ # ## ExitlistOutputDirectory = out/exit-lists/ +ExitlistUrl = https://check.torproject.org/exit-addresses ######## Torperf downloader ######## # diff --git a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java index 57dda75..ac9fd4f 100644 --- a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java +++ b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java @@ -22,7 +22,7 @@ public class ConfigurationTest { public void testKeyCount() throws Exception { assertEquals("The number of properties keys in enum Key changed." + "\n This test class should be adapted.", - 32, Key.values().length); + 33, Key.values().length); } @Test() @@ -129,6 +129,13 @@ public class ConfigurationTest { } @Test( expected = ConfigurationException.class) + public void testUrlValueException() throws Exception { + Configuration conf = new Configuration(); + conf.load(new ByteArrayInputStream("ExitlistUrl = xxx://y.y.y".getBytes())); + conf.getUrl(Key.ExitlistUrl); + } + + @Test( expected = ConfigurationException.class) public void testIntValueException() throws Exception { Configuration conf = new Configuration(); conf.load(new ByteArrayInputStream("BridgeDescriptorMappingsLimit = y7".getBytes()));

1 0

[torspec/master] Remove the prop#216 "Curve25519 group" sentence from NewHope proposal.
by isis＠torproject.org 22 Jul '16

22 Jul '16

commit d04f771f8b8bcb7b5b6c27d1df352061282ac2ac Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun May 8 16:08:16 2016 +0000 Remove the prop#216 "Curve25519 group" sentence from NewHope proposal. --- proposals/XXX-newhope-hybrid-handshake.txt | 8 -------- 1 file changed, 8 deletions(-) diff --git a/proposals/XXX-newhope-hybrid-handshake.txt b/proposals/XXX-newhope-hybrid-handshake.txt index 607b533..6c234cb 100644 --- a/proposals/XXX-newhope-hybrid-handshake.txt +++ b/proposals/XXX-newhope-hybrid-handshake.txt @@ -218,14 +218,6 @@ Depends: prop#220 prop#249 prop#264 if AUTH == H(auth_input, T_MAC) return NTOR_KEY - Both parties check that none of the EXP() operations produced the point at - infinity. [NOTE: This is an adequate replacement for checking Y for group - membership, if the group is Curve25519.] - - [XXX: This doesn't sound exactly right. You need the scalar tweaking of - X25519 for this to work and also, the point at infinity is obviously an - element of the group --isis, peter] - Both parties now have a shared value for NTOR_KEY. They expand this into the keys needed for the Tor relay protocol.

1 0

[torspec/master] Add NewHope + X25519 handshake proposal draft.
by isis＠torproject.org 22 Jul '16

22 Jul '16

commit 0f5ddf6ca863eaa748733557a3a96c44f3361085 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Apr 22 19:23:43 2016 +0000 Add NewHope + X25519 handshake proposal draft. --- proposals/XXX-newhope-hybrid-handshake.txt | 768 +++++++++++++++++++++++++++++ 1 file changed, 768 insertions(+) diff --git a/proposals/XXX-newhope-hybrid-handshake.txt b/proposals/XXX-newhope-hybrid-handshake.txt new file mode 100644 index 0000000..2a5e076 --- /dev/null +++ b/proposals/XXX-newhope-hybrid-handshake.txt @@ -0,0 +1,768 @@ +Filename: XXX-newhope-hybrid-handshake.txt +Title: Post-Quantum Secure Hybrid Handshake Based on NewHope +Author: Isis Lovecruft, Peter Schwabe +Created: 16 Apr 2016 +Updated: 4 May 2016 +Status: Draft +Depends: prop#220 prop#249 prop#264 + +§0. Introduction + + NewHope is a post-quantum-secure lattice-based key-exchange protocol based + on the ring-learning-with-errors (Ring-LWE) problem. We propose a hybrid + handshake for Tor, based on a combination of Tor's current NTor handshake + and a shared key derived through a NewHope ephemeral key exchange. + + For further details on the NewHope key exchange, the reader is referred to + "Post-quantum key exchange - a new hope" by Alkim, Ducas, Pöppelmann, and + Schwabe [0][1]. + + For the purposes of brevity, we consider that NTor is currently the only + handshake protocol in Tor; the older TAP protocol is ignored completely, due + to the fact that it is currently deprecated and nearly entirely unused. + + +§1. Motivation + + An attacker currently monitoring and storing circuit-layer NTor handshakes + who later has the ability to run Shor's algorithm on a quantum computer will + be able to break Tor's current handshake protocol and decrypt previous + communications. + + It is unclear if and when such attackers equipped with large quantum + computers will exist, but various estimates by researchers in quantum + physics and quantum engineering give estimates of only 1 to 2 decades. + Clearly, the security requirements of many Tor users include secrecy of + their messages beyond this time span, which means that Tor needs to update + the key exchange to protect against such attackers as soon as possible. + + +§2. Design + + An initiator and responder, in parallel, conduct two handshakes: + + - An X25519 key exchange, as described in the description of the NTor + handshake in Tor proposal #216. + - A NewHope key exchange. + + The shared keys derived from these two handshakes are then concatenated and + used as input to the SHAKE-256 extendable output function (XOF), as decribed + in FIPS-PUB-202 [2], in order to produce a shared key of the desired length. + The testvectors in §C assume that this key has a length of 32 bytes, but the + use of a XOF allows arbitrary lengths to easily support future updates of + the symmetric primitives using the key. See also §3.3.1. + + +§3. Specification + +§3.1. Notation + + Let `a || b` be the concatenation of a with b. + + Let `a^b` denote the exponentiation of a to the bth power. + + Let `a == b` denote the equality of a with b, and vice versa. + + Let `a := b` be the assignment of the value of b to the variable a. + + Let `H(x)` be 32-bytes of output of the SHAKE-256 XOF (as described in + FIPS-PUB-202) applied to message x. + + Let X25519 refer to the curve25519-based key agreement protocol described + in RFC7748 §6.1. [3] + + Let `EXP(a, b) == X25519(., b, a)` with `g == 9`. Let X25519_KEYGEN() do + the appropriate manipulations when generating the secret key (clearing the + low bits, twidding the high bits). + + [XXX match RFC7748 notation more. --isis] + + Let `X25519_KEYID(B) == B` where B is a valid X25519 public key. + + When representing an element of the Curve25519 subgroup as a byte string, + use the standard (32-byte, little-endian, x-coordinate-only) representation + for Curve25519 points. + + Let `ID` be a router's identity key taken from the router microdescriptor. + In the case for relays possessing Ed25519 identity keys (c.f. Tor proposal + #220), this is a 32-byte string representing the public Ed25519 identity key. + For backwards and forwards compatibility with routers which do not possess + Ed25519 identity keys, this is a 32-byte string created via the output of + H(ID). + + We refer to the router as the handshake "responder", and the client (which + may be an OR or an OP) as the "initiator". + + + ID_LENGTH [32 bytes] + H_LENGTH [32 bytes] + G_LENGTH [32 bytes] + + PROTOID := "pqtor-x25519-newhope-shake256-1" + T_MAC := PROTOID || ":mac" + T_KEY := PROTOID || ":key_extract" + T_VERIFY := PROTOID || ":verify" + + (X25519_SK, X25519_PK) := X25519_KEYGEN() + + +§3.2. Protocol + + ======================================================================================== + | | + | Fig. 1: The NewHope-X25519 Hybrid Handshake. | + | | + | Before the handshake the Initiator is assumed to know Z, a public X25519 key for | + | the Responder, as well as the Responder's ID. | + ---------------------------------------------------------------------------------------- + | | + | Initiator Responder | + | | + | SEED := H(randombytes(32)) | + | x, X := X25519_KEYGEN() | + | a, A := NEWHOPE_KEYGEN(SEED) | + | CLIENT_HDATA := ID || Z || X || A | + | | + | --- CLIENT_HDATA ---> | + | | + | y, Y := X25519_KEYGEN() | + | NTOR_KEY, AUTH := NTOR_SHAREDB(X,y,Y,z,Z,ID,B) | + | M, NEWHOPE_KEY := NEWHOPE_SHAREDB(A) | + | SERVER_HDATA := Y || AUTH || M | + | sk := SHAKE-256(NTOR_KEY || NEWHOPE_KEY) | + | | + | <-- SERVER_HDATA ---- | + | | + | NTOR_KEY := NTOR_SHAREDA(x, X, Y, Z, ID, AUTH) | + | NEWHOPE_KEY := NEWHOPE_SHAREDA(M, a) | + | sk := SHAKE-256(NTOR_KEY, NEWHOPE_KEY) | + | | + ======================================================================================== + + +§3.2.1. The NTor Handshake + +§3.2.1.1. Prologue + + Take a router with identity ID. As setup, the router generates a secret key z, + and a public onion key Z with: + + z, Z := X25519_KEYGEN() + + The router publishes Z in its server descriptor in the "ntor-onion-key" entry. + Henceforward, we refer to this router as the "responder". + + +§3.2.1.2. Initiator + + To send a create cell, the initiator generates a keypair: + + x, X := X25519_KEYGEN() + + and creates the NTor portion of a CREATE2V cell's HDATA section: + + CLIENT_NTOR := ID || Z || X [96 bytes] + + The initiator includes the responder's ID and Z in the CLIENT_NTOR so that, in + the event the responder OR has recently rotated keys, the responder can + determine which keypair to use. + + The initiator then concatenates CLIENT_NTOR with CLIENT_NEWHOPE (see §3.2.2), + to create CLIENT_HDATA, and creates and sends a CREATE2V cell (see §A.1) + to the responder. + + CLIENT_NEWHOPE [1824 bytes] (see §3.2.2) + CLIENT_HDATA := CLIENT_NTOR || CLIENT_NEWHOPE [1920 bytes] + + If the responder does not respond with a CREATED2V cell, the initiator SHOULD + NOT attempt to extend the circuit through the responder by sending fragmented + EXTEND2 cells, since the responder's lack of support for CREATE2V cells is + assumed to imply the responder also lacks support for fragmented EXTEND2 + cells. Alternatively, for initiators with a sufficiently late consensus + method, the initiator MUST check that "proto" line in the responder's + descriptor (c.f. Tor proposal #264) advertises support for the "Relay" + subprotocol version 3 (see §5). + + +§3.2.1.3. Responder + + The responder generates a keypair of y, Y = X25519_KEYGEN(), and does + NTOR_SHAREDB() as follows: + + (NTOR_KEY, AUTH) ← NTOR_SHAREDB(X, y, Y, z, Z, ID, B): + secret_input := EXP(X, y) || EXP(X, z) || ID || B || Z || Y || PROTOID + NTOR_KEY := H(secret_input, T_KEY) + verify := H(secret_input, T_VERIFY) + auth_input := verify || ID || Z || Y || X || PROTOID || "Server" + AUTH := H(auth_input, T_MAC) + + The responder sends a CREATED2V cell containing: + + SERVER_NTOR := Y || AUTH [64 bytes] + SERVER_NEWHOPE [2048 bytes] (see §3.2.2) + SERVER_HDATA := SERVER_NTOR || SERVER_NEWHOPE [2112 bytes] + + and sends this to the initiator. + + +§3.2.1.4. Finalisation + + The initiator then checks Y is in G^* [see NOTE below], and does + NTOR_SHAREDA() as follows: + + (NTOR_KEY) ← NTOR_SHAREDA(x, X, Y, Z, ID, AUTH) + secret_input := EXP(Y, x) || EXP(Z, x) || ID || Z || X || Y || PROTOID + NTOR_KEY := H(secret_input, T_KEY) + verify := H(secret_input, T_VERIFY) + auth_input := verify || ID || Z || Y || X || PROTOID || "Server" + if AUTH == H(auth_input, T_MAC) + return NTOR_KEY + + Both parties check that none of the EXP() operations produced the point at + infinity. [NOTE: This is an adequate replacement for checking Y for group + membership, if the group is Curve25519.] + + [XXX: This doesn't sound exactly right. You need the scalar tweaking of + X25519 for this to work and also, the point at infinity is obviously an + element of the group --isis, peter] + + Both parties now have a shared value for NTOR_KEY. They expand this into + the keys needed for the Tor relay protocol. + + [XXX We think we want to omit the final hashing in the production of NTOR_KEY + here, and instead put all the inputs through SHAKE-256. --isis, peter] + + [XXX We probably want to remove ID and B from the input to the shared key + material, since they serve for authentication but, as pre-established + "prologue" material to the handshake, they should not be used in attempts to + strengthen the cryptographic suitability of the shared key. Also, their + inclusion is implicit in the DH exponentiations. I should probably ask Ian + about the reasoning for the original design choice. --isis] + + +§3.2.2. The NewHope Handshake + +§3.2.2.1. Parameters & Mathematical Structures + + Let ℤ be the ring of rational integers. Let ℤq, for q ≥ 1, denote the quotient + ring ℤ/qℤ. We define R = ℤ[X]/((X^n)+1) as the ring of integer polynomials + modulo ((X^n)+1), and Rq = ℤq[X]/((X^n)+1) as the ring of integer polynomials + modulo ((X^n)+1) where each coefficient is reduced modulo q. When we refer to + a polynomial, we mean an element of Rq. + + n := 1024 + q := 12289 + + SEED [32 Bytes] + NEWHOPE_POLY [1792 Bytes] + NEWHOPE_REC [256 Bytes] + NEWHOPE_KEY [32 Bytes] + + NEWHOPE_MSGA := (NEWHOPE_POLY || SEED) + NEWHOPE_MSGB := (NEWHOPE_POLY || NEWHOPE_REC) + + +§3.2.2.2. High-level Description of Newhope API Functions + + For a description of internal functions, see §B. + + (NEWHOPE_POLY, NEWHOPE_MSGA) ← NEWHOPE_KEYGEN(SEED): + â := gen_a(seed) + s := poly_getnoise() + e := poly_getnoise() + ŝ := poly_ntt(s) + ê := poly_ntt(e) + b̂ := pointwise(â, ŝ) + ê + sp := poly_tobytes(ŝ) + bp := poly_tobytes(b̂) + return (sp, (bp || seed)) + + (NEWHOPE_MSGB, NEWHOPE_KEY) ← NEWHOPE_SHAREDB(NEWHOPE_MSGA): + s' := poly_getnoise() + e' := poly_getnoise() + e" := poly_getnoise() + b̂ := poly_frombytes(bp) + â := gen_a(seed) + ŝ' := poly_ntt(s') + ê' := poly_ntt(e') + û := poly_pointwise(â, ŝ') + ê' + v := poly_invntt(poly_pointwise(b̂,ŝ')) + e" + r := helprec(v) + up := poly_tobytes(û) + k := rec(v, r) + return ((up || r), k) + + NEWHOPE_KEY ← NEWHOPE_SHAREDA(NEWHOPE_MSGB, NEWHOPE_POLY): + û := poly_frombytes(up) + ŝ := poly_frombytes(sp) + v' := poly_invntt(poly_pointwise(û, ŝ)) + k := rec(v', r) + return k + + When a client uses a SEED within a CREATE2V cell, the client SHOULD NOT use + that SEED in any other CREATE2V or EXTEND2 cells. See §4 for further + discussion. + + +§3.3. Key Expansion + + The client and server derive a shared key, SHARED, by: + + HKDFID := "THESE ARENT THE DROIDS YOURE LOOKING FOR" + SHARED := SHAKE_256(HKDFID || NTorKey || NewHopeKey) + + +§3.3.1. Note on the Design Choice + + The reader may wonder why one would use SHAKE-256 to produce a 256-bit + output, since the security strength in bits for SHAKE-256 is min(d/2,256) + for collision resistance and min(d,256) for first- and second-order + preimages, where d is the output length. + + The reasoning is that we should be aiming for 256-bit security for all of + our symmetric cryptography. One could then argue that we should just use + SHA3-256 for the KDF. We choose SHAKE-256 instead in order to provide an + easy way to derive longer shared secrets in the future without requiring a + new handshake. The construction is odd, but the future is bright. + As we are already using SHAKE-256 for the 32-byte output hash, we are also + using it for all other 32-byte hashes involved in the protocol. Note that + the only difference between SHA3-256 and SHAKE-256 with 32-byte output is + one domain-separation byte. + + [XXX why would you want 256-bit security for the symmetric side? Are you + talking pre- or post-quantum security? --peter] + + +§4. Security & Anonymity Implications + + This handshake protocol is one-way authenticated. That is, the server is + authenticated, while the client remains anonymous. + + The client MUST NOT cache and reuse SEED. Doing so gives non-trivial + adversarial advantages w.r.t. all-for-the-price-of-one attacks during the + caching period. More importantly, if the SEED used to generate NEWHOPE_MSGA + is reused for handshakes along the same circuit or multiple different + circuits, an adversary conducting a sybil attack somewhere along the path(s) + will be able to correlate the identity of the client across circuits or + hops. + + +§5. Compatibility + + Because our proposal requires both the client and server to send more than + the 505 bytes possible within a CREATE2 cell's HDATA section, it depends + upon the implementation of a mechanism for allowing larger CREATE cells + (c.f. Tor proposal #249). + + We reserve the following handshake type for use in CREATE2V/CREATED2V and + EXTEND2V/EXTENDED2V cells: + + 0x0003 [NEWHOPE + X25519 HYBRID HANDSHAKE] + + We introduce a new sub-protocol number, "Relay=3", (c.f. Tor proposal #264 + §5.3) to signify support this handshake, and hence for the CREATE2V and + fragmented EXTEND2 cells which it requires. + + There are no additional entries or changes required within either router + descriptors or microdescriptors to support this handshake method, due to the + NewHope keys being ephemeral and derived on-the-fly, and due to the NTor X25519 + public keys already being in included within the "ntor-onion-key" entry. + + Add a "UseNewHopeKEX" configuration option and a corresponding consensus + parameter to control whether clients prefer using this NewHope hybrid + handshake or some previous handshake protocol. If the configuration option + is "auto", clients SHOULD obey the consensus parameter. The default + configuration SHOULD be "auto" and the consensus value SHOULD initially be "0". + + +§6. Implementation + + The paper by Alkim, Ducas, Pöppelmann and Schwabe describes two software + implementations of NewHope, one C reference implementation and an optimized + implementation using AVX2 vector instructions. Those implementations are + available at [1]. + + Additionally, there are implementations in Go by Yawning Angel, available + from [4] and in Rust by Isis Lovecruft, available from [5]. + + The software used to generate the test vectors in §C is based on the C + reference implementation and available from: + + https://code.ciph.re/isis/newhope-tor-testvectors + https://github.com/isislovecruft/newhope-tor-testvectors + + +§7. Performance & Scalability + + The computationally expensive part in the current NTor handshake is the + X25519 key-pair generation and the X25519 shared-key computation. The + current implementation in Tor is a wrapper to support various highly optimized + implementations on different architectures. On Intel Haswell processors, the + fastest implementation of X25519, as reported by the eBACS benchmarking + project [6], takes 169920 cycles for key-pair generation and 161648 cycles + for shared-key computation; these add up to a total of 331568 cycles on each + side (initiator and responder). + + The C reference implementation of NewHope, also benchmarked on Intel + Haswell, takes 358234 cycles for the initiator and 402058 cycles for the + Responder. The core computation of the proposed combination of NewHope and + X25519 will thus mean a slowdown of about a factor of 2.1 for the Initiator + and a slowdown by a factor of 2.2 for the Responder compared to the current + NTor handshake. These numbers assume a fully optimized implementation of the + NTor handshake and a C reference implementation of NewHope. With optimized + implementations of NewHope, such as the one for Intel Haswell described in + [0], the computational slowdown will be considerably smaller than a factor + of 2. + + +§8. References + +[0]: https://cryptojedi.org/papers/newhope-20160328.pdf +[1]: https://cryptojedi.org/crypto/#newhope +[2]: http://www.nist.gov/customcf/get_pdf.cfm?pub_id=919061 +[3]: https://tools.ietf.org/html/rfc7748#section-6.1 +[4]: https://github.com/Yawning/newhope +[5]: https://code.ciph.re/isis/newhopers +[6]: http://bench.cr.yp.to + + +§A. Cell Formats + +§A.1. CREATE2V Cells + + The client portion of the handshake should send CLIENT_HDATA, formatted + into a CREATE2V cell as follows: + + CREATE2V { [2114 bytes] + HTYPE := 0x0003 [2 bytes] + HLEN := 0x0780 [2 bytes] + HDATA := CLIENT_HDATA [1920 bytes] + IGNORED := 0x00 [194 bytes] + } + + [XXX do we really want to pad with IGNORED to make CLIENT_HDATA the + same number of bytes as SERVER_HDATA? --isis] + +§A.2. CREATED2V Cells + + The server responds to the client's CREATE2V cell with SERVER_HDATA, + formatted into a CREATED2V cell as follows: + + CREATED2V { [2114 bytes] + HLEN := 0x0800 [2 bytes] + HDATA := SERVER_HDATA [2112 bytes] + IGNORED := 0x00 [0 bytes] + } + +§A.3. Fragmented EXTEND2 Cells + + When the client wishes to extend a circuit, the client should fragment + CLIENT_HDATA into four EXTEND2 cells: + + EXTEND2 { + NSPEC := 0x02 { [1 byte] + LINK_ID_SERVER [22 bytes] XXX + LINK_ADDRESS_SERVER [8 bytes] XXX + } + HTYPE := 0x0003 [2 bytes] + HLEN := 0x0780 [2 bytes] + HDATA := CLIENT_HDATA[0,461] [462 bytes] + } + EXTEND2 { + NSPEC := 0x00 [1 byte] + HTYPE := 0xFFFF [2 bytes] + HLEN := 0x0000 [2 bytes] + HDATA := CLIENT_HDATA[462,954] [492 bytes] + } + EXTEND2 { + NSPEC := 0x00 [1 byte] + HTYPE := 0xFFFF [2 bytes] + HLEN := 0x0000 [2 bytes] + HDATA := CLIENT_HDATA[955,1447] [492 bytes] + } + EXTEND2 { + NSPEC := 0x00 [1 byte] + HTYPE := 0xFFFF [2 bytes] + HLEN := 0x0000 [2 bytes] + HDATA := CLIENT_HDATA[1448,1919] || 0x00[20] [492 bytes] + } + EXTEND2 { + NSPEC := 0x00 [1 byte] + HTYPE := 0xFFFF [2 bytes] + HLEN := 0x0000 [2 bytes] + HDATA := 0x00[172] [172 bytes] + } + + The client sends this to the server to extend the circuit from, and that + server should format the fragmented EXTEND2 cells into a CREATE2V cell, as + described in §A.1. + +§A.4. Fragmented EXTENDED2 Cells + + EXTENDED2 { + NSPEC := 0x02 { [1 byte] + LINK_ID_SERVER [22 bytes] XXX + LINK_ADDRESS_SERVER [8 bytes] XXX + } + HTYPE := 0x0003 [2 bytes] + HLEN := 0x0800 [2 bytes] + HDATA := SERVER_HDATA[0,461] [462 bytes] + } + EXTENDED2 { + NSPEC := 0x00 [1 byte] + HTYPE := 0xFFFF [2 bytes] + HLEN := 0x0000 [2 bytes] + HDATA := SERVER_HDATA[462,954] [492 bytes] + } + EXTEND2 { + NSPEC := 0x00 [1 byte] + HTYPE := 0xFFFF [2 bytes] + HLEN := 0x0000 [2 bytes] + HDATA := SERVER_HDATA[955,1447] [492 bytes] + } + EXTEND2 { + NSPEC := 0x00 [1 byte] + HTYPE := 0xFFFF [2 bytes] + HLEN := 0x0000 [2 bytes] + HDATA := SERVER_HDATA[1448,1939] [492 bytes] + } + EXTEND2 { + NSPEC := 0x00 [1 byte] + HTYPE := 0xFFFF [2 bytes] + HLEN := 0x0000 [2 bytes] + HDATA := SERVER_HDATA[1940,2112] [172 bytes] + } + + +§B. NewHope Internal Functions + + gen_a(SEED): returns a uniformly random poly + poly_getnoise(): returns a poly sampled from a centered binomial + poly_ntt(poly): number-theoretic transform; returns a poly + poly_invntt(poly): inverse number-theoretic transform; returns a poly + poly_pointwise(poly, poly): pointwise multiplication; returns a poly + poly_tobytes(poly): packs a poly to a NEWHOPE_POLY byte array + poly_frombytes(NEWHOPE_POLY): unpacks a NEWHOPE_POLY byte array to a poly + + helprec(poly): returns a NEWHOPE_REC byte array + rec(poly, NEWHOPE_REC): returns a NEWHOPE_KEY + + + --- Description of the Newhope internal functions --- + + gen_a(SEED seed) receives as input a 32-byte (public) seed. It expands + this seed through SHAKE-128 from the FIPS202 standard. The output of SHAKE-128 + is considered a sequence of 16-bit little-endian integers. This sequence is + used to initialize the coefficients of the returned polynomial from the least + significant (coefficient of X^0) to the most significant (coefficient of + X^1023) coefficient. For each of the 16-bit integers first eliminate the + highest two bits (to make it a 14-bit integer) and then use it as the next + coefficient if it is smaller than q=12289. + Note that the amount of output required from SHAKE to initialize all 1024 + coefficients of the polynomial varies depending on the input seed. + Note further that this function does not process any secret data and thus does + not need any timing-attack protection. + + + poly_getnoise() first generates 4096 Bytes of uniformly random data. This can + be done by reading these bytes from the system's RNG; efficient + implementations will typically only read a 32-byte seed from the system's RNG + and expand it through some fast PRNG (for example, ChaCha20 or AES-256 in CTR + mode). The output of the PRG is considered an array of 2048 16-bit integers + r[0],...,r[2047]. The coefficients of the output polynomial are computed as + HW(r[0])-HW(r[1]), HW(r[2])-HW(r[3]),...,HW(r[2046])-HW(r[2047]), where HW + stands for Hamming weight. + Note that the choice of RNG is a local decision; different implementations are + free to use different RNGs. + Note further that the output of this function is secret; the PRG (and the + computation of HW) need to be protected against timing attacks. + + + poly_ntt(poly f): For a mathematical description of poly_ntt see the [0]; a + pseudocode description of a very naive inplace transformation of an input + polynomial f = f[0] + f[1]*X + f[2]*X^2 + ... + f[1023]*X^1023 is the + following code (all arithmetic on coefficients performed modulo q): + + psi = 7 + omega = 49 + + for i in range(0,n): + t[i] = f[i] * psi^i + + for i in range(0,n): + f[i] = 0 + for j in range(0,n): + f[i] += t[j] * omega^((i*j)%n) + + Note that this is not how poly_ntt should be implemented if performance is + an issue; in particular, efficient algorithms for the number-theoretic + transform take time O(n*log(n)) and not O(n^2) + Note further that all arithmetic in poly_ntt has to be protected against + timing attacks. + + + poly_invntt(poly f): For a mathematical description of poly_invntt see the + [0]; a pseudocode description of a very naive inplace transformation of an + input polynomial f = f[0] + f[1]*X + f[2]*X^2 + ... + f[1023]*X^1023 is the + following code (all arithmetic on coefficients performed modulo q): + + invpsi = 8778; + invomega = 1254; + invn = 12277; + + for i in range(0,n): + t[i] = f[i]; + + for i in range(0,n): + f[i]=0; + for j in range(0,n): + f[i] += t[j] * invomega^((i*j)%n) + f[i] *= invpsi^i + f[i] *= invn + + Note that this is not how poly_invntt should be implemented if performance + is an issue; in particular, efficient algorithms for the inverse + number-theoretic transform take time O(n*log(n)) and not O(n^2) + Note further that all arithmetic in poly_invntt has to be protected against + timing attacks. + + + poly_pointwise(poly f, poly g) performs pointwise multiplication of the two + polynomials. This means that for f = (f0 + f1*X + f2*X^2 + ... + + f1023*X^1023) and g = (g0 + g1*X + g2*X^2 + ... + g1023*X^1023) it computes + and returns h = (h0 + h1*X + h2*X^2 + ... + h1023*X^1023) with h0 = f0*g0, + h1 = f1*g1,..., h1023 = f1023*g1023. + + + poly_tobytes(poly f) first reduces all coefficents of f modulo q, i.e., + brings them to the interval [0,q-1]. Denote these reduced coefficients as + f0,..., f1023; note that they all fit into 14 bits. The function then packs + those coefficients into an array of 1792 bytes r[0],..., r[1792] in "packed + little-endian representation", i.e., + r[0] = f[0] & 0xff; + r[1] = (f[0] >> 8) & ((f[1] & 0x03) << 6) + r[2] = (f[1] >> 2) & 0xff; + r[3] = (f[1] >> 10) & ((f[2] & 0x0f) << 4) + . + . + . + r[1790] = (f[1022]) >> 12) & ((f[1023] & 0x3f) << 2) + r[1791] = f[1023] >> 6 + Note that this function needs to be protected against timing attacks. In + particular, avoid non-constant-time conditional subtractions (or other + non-constant-time expressions) in the reduction modulo q of the coefficients. + + + poly_frombytes(NEWHOPE_POLY b) is the inverse of poly_tobytes; it receives + as input an array of 1792 bytes and coverts it into the internal + representation of a poly. Note that poly_frombytes does not need to check + whether the coefficients are reduced modulo q or reduce coefficients modulo + q. Note further that the function must not leak any information about its + inputs through timing information, as it is also applied to the secret key + of the initiator. + + + helprec(poly f) computes 256 bytes of reconciliation information from the + input poly f. Internally, one byte of reconciliation information is computed + from four coefficients of f by a function helprec4. Let the input polynomial f + = (f0 + f1*X + f2*X^2 + ... + f1023*X^1023); let the output byte array be + r[0],...r[256]. This output byte array is computed as + r[0] = helprec4(f0,f256,f512,f768) + r[1] = helprec4(f1,f257,f513,f769) + r[2] = helprec4(f2,f258,f514,f770) + . + . + . + r[255] = helprec4(f255,f511,f767,f1023), where helprec4 does the following: + + helprec4(x0,x1,x2,x3): + b = randombit() + r0,r1,r2,r3 = CVPD4(8*x0+4*b,8*x1+4*b,8*x2+4*b,8*x3+4*b) + r = (r0 & 0x03) | ((r1 & 0x03) << 2) | ((r2 & 0x03) << 4) | ((r3 & 0x03) << 6) + return r + + The function CVPD4 does the following: + + CVPD4(y0,y1,y2,y3): + v00 = round(y0/2q) + v01 = round(y1/2q) + v02 = round(y2/2q) + v03 = round(y3/2q) + v10 = round((y0-1)/2q) + v11 = round((y1-1)/2q) + v12 = round((y2-1)/2q) + v13 = round((y3-1)/2q) + t = abs(y0 - 2q*v00) + t += abs(y1 - 2q*v01) + t += abs(y2 - 2q*v02) + t += abs(y3 - 2q*v03) + if(t < 2q): + v0 = v00 + v1 = v01 + v2 = v02 + v3 = v03 + k = 0 + else + v0 = v10 + v1 = v11 + v2 = v12 + v3 = v13 + r = 1 + return (v0-v3,v1-v3,v2-v3,k+2*v3) + + In this description, round() returns the closest integer and abs() returns the + absolute value. + Note that all computations involved in helprec operate on secret data and must + be protected against timing attacks. + + + rec(poly f, NEWHOPE_REC r) computes the pre-hash (see paper) Newhope key from + f and r. Specifically, it computes one bit of key from 4 coefficients of f and + one byte of r. Let f = f0 + f1*X + f2*X^2 + ... + f1023*X^1023 and let r = + r[0],r[1],...,r[255]. Let the bytes of the output by k[0],...,k[31] and let + the bits of the output by k0,...,k255, where + k0 = k[0] & 0x01 + k1 = (k[0] >> 1) & 0x01 + k2 = (k[0] >> 2) & 0x01 + . + . + . + k8 = k[1] & 0x01 + k9 = (k[1] >> 1) & 0x01 + . + . + . + k255 = (k[32] >> 7) + The function rec computes k0,...,k255 as + k0 = rec4(f0,f256,f512,f768,r[0]) + k1 = rec4(f1,f257,f513,f769,r[1]) + . + . + . + k255 = rec4(f255,f511,f767,f1023,r[255]) + + The function rec4 does the following: + + rec4(y0,y1,y2,y3,r): + r0 = r & 0x03 + r1 = (r >> 2) & 0x03 + r2 = (r >> 4) & 0x03 + r3 = (r >> 6) & 0x03 + Decode(8*y0-2q*r0, 8*y1-2q*r1, 8*y2-2q*r2, 8*y3-q*r3) + + The function Decode does the following: + + Decode(v0,v1,v2,v3): + t0 = round(v0/8q) + t1 = round(v1/8q) + t2 = round(v2/8q) + t3 = round(v3/8q) + t = abs(v0 - 8q*t0) + t += abs(v0 - 8q*t0) + t += abs(v0 - 8q*t0) + t += abs(v0 - 8q*t0) + if(t > 1) return 1 + else return 0 + + +§C. Test Vectors

1 0

[torspec/master] Fix several typos found in the NewHope proposal.
by isis＠torproject.org 22 Jul '16

22 Jul '16

commit 5c11590522923f8cbca307f6be92357087a5ca2d Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun May 8 15:05:36 2016 +0000 Fix several typos found in the NewHope proposal. * THANKS TO eikovi(a)sigaint.org for pointing them out. --- proposals/XXX-newhope-hybrid-handshake.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/proposals/XXX-newhope-hybrid-handshake.txt b/proposals/XXX-newhope-hybrid-handshake.txt index 2a5e076..d11fbd2 100644 --- a/proposals/XXX-newhope-hybrid-handshake.txt +++ b/proposals/XXX-newhope-hybrid-handshake.txt @@ -46,7 +46,7 @@ Depends: prop#220 prop#249 prop#264 - A NewHope key exchange. The shared keys derived from these two handshakes are then concatenated and - used as input to the SHAKE-256 extendable output function (XOF), as decribed + used as input to the SHAKE-256 extendable output function (XOF), as described in FIPS-PUB-202 [2], in order to produce a shared key of the desired length. The testvectors in §C assume that this key has a length of 32 bytes, but the use of a XOF allows arbitrary lengths to easily support future updates of @@ -84,7 +84,7 @@ Depends: prop#220 prop#249 prop#264 for Curve25519 points. Let `ID` be a router's identity key taken from the router microdescriptor. - In the case for relays possessing Ed25519 identity keys (c.f. Tor proposal + In the case for relays possessing Ed25519 identity keys (cf. Tor proposal #220), this is a 32-byte string representing the public Ed25519 identity key. For backwards and forwards compatibility with routers which do not possess Ed25519 identity keys, this is a 32-byte string created via the output of @@ -180,7 +180,7 @@ Depends: prop#220 prop#249 prop#264 assumed to imply the responder also lacks support for fragmented EXTEND2 cells. Alternatively, for initiators with a sufficiently late consensus method, the initiator MUST check that "proto" line in the responder's - descriptor (c.f. Tor proposal #264) advertises support for the "Relay" + descriptor (cf. Tor proposal #264) advertises support for the "Relay" subprotocol version 3 (see §5). @@ -352,14 +352,14 @@ Depends: prop#220 prop#249 prop#264 Because our proposal requires both the client and server to send more than the 505 bytes possible within a CREATE2 cell's HDATA section, it depends upon the implementation of a mechanism for allowing larger CREATE cells - (c.f. Tor proposal #249). + (cf. Tor proposal #249). We reserve the following handshake type for use in CREATE2V/CREATED2V and EXTEND2V/EXTENDED2V cells: 0x0003 [NEWHOPE + X25519 HYBRID HANDSHAKE] - We introduce a new sub-protocol number, "Relay=3", (c.f. Tor proposal #264 + We introduce a new sub-protocol number, "Relay=3", (cf. Tor proposal #264 §5.3) to signify support this handshake, and hence for the CREATE2V and fragmented EXTEND2 cells which it requires. @@ -564,10 +564,10 @@ Depends: prop#220 prop#249 prop#264 not need any timing-attack protection. - poly_getnoise() first generates 4096 Bytes of uniformly random data. This can + poly_getnoise() first generates 4096 bytes of uniformly random data. This can be done by reading these bytes from the system's RNG; efficient implementations will typically only read a 32-byte seed from the system's RNG - and expand it through some fast PRNG (for example, ChaCha20 or AES-256 in CTR + and expand it through some fast PRG (for example, ChaCha20 or AES-256 in CTR mode). The output of the PRG is considered an array of 2048 16-bit integers r[0],...,r[2047]. The coefficients of the output polynomial are computed as HW(r[0])-HW(r[1]), HW(r[2])-HW(r[3]),...,HW(r[2046])-HW(r[2047]), where HW @@ -579,7 +579,7 @@ Depends: prop#220 prop#249 prop#264 poly_ntt(poly f): For a mathematical description of poly_ntt see the [0]; a - pseudocode description of a very naive inplace transformation of an input + pseudocode description of a very naive in-place transformation of an input polynomial f = f[0] + f[1]*X + f[2]*X^2 + ... + f[1023]*X^1023 is the following code (all arithmetic on coefficients performed modulo q):

1 0

[torspec/master] Updated definition of round(); fixed two typos.
by isis＠torproject.org 22 Jul '16

22 Jul '16

commit 28181cc70fd4c87f156e86bad36dae74b2387117 Author: Peter Schwabe <peter(a)cryptojedi.org> Date: Mon May 9 17:47:03 2016 +0200 Updated definition of round(); fixed two typos. --- proposals/XXX-newhope-hybrid-handshake.txt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/proposals/XXX-newhope-hybrid-handshake.txt b/proposals/XXX-newhope-hybrid-handshake.txt index 6c234cb..86d7b00 100644 --- a/proposals/XXX-newhope-hybrid-handshake.txt +++ b/proposals/XXX-newhope-hybrid-handshake.txt @@ -358,7 +358,7 @@ Depends: prop#220 prop#249 prop#264 There are no additional entries or changes required within either router descriptors or microdescriptors to support this handshake method, due to the NewHope keys being ephemeral and derived on-the-fly, and due to the NTor X25519 - public keys already being in included within the "ntor-onion-key" entry. + public keys already being included within the "ntor-onion-key" entry. Add a "UseNewHopeKEX" configuration option and a corresponding consensus parameter to control whether clients prefer using this NewHope hybrid @@ -594,7 +594,7 @@ Depends: prop#220 prop#249 prop#264 poly_invntt(poly f): For a mathematical description of poly_invntt see the - [0]; a pseudocode description of a very naive inplace transformation of an + [0]; a pseudocode description of a very naive in-place transformation of an input polynomial f = f[0] + f[1]*X + f[2]*X^2 + ... + f[1023]*X^1023 is the following code (all arithmetic on coefficients performed modulo q): @@ -702,8 +702,9 @@ Depends: prop#220 prop#249 prop#264 r = 1 return (v0-v3,v1-v3,v2-v3,k+2*v3) - In this description, round() returns the closest integer and abs() returns the - absolute value. + In this description, round(x) is defined as ⌊x + 0.5⌋, where ⌊x⌋ rounds to + the largest integer that does not exceed x; abs() returns the absolute + value. Note that all computations involved in helprec operate on secret data and must be protected against timing attacks.

1 0