December 2014 - tor-commits - lists.torproject.org

[tor/master] Use END_CIRC_REASON_TORPROTOCOL instead of magic number.
by nickm＠torproject.org 08 Dec '14

08 Dec '14

commit 9c239eccc973042cbaec16fd48a457f44aeafc24 Author: rl1987 <rl1987(a)sdf.lonestar.org> Date: Sun Dec 7 15:47:09 2014 +0200 Use END_CIRC_REASON_TORPROTOCOL instead of magic number. --- changes/bug13840 | 3 +++ src/or/connection_edge.c | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/changes/bug13840 b/changes/bug13840 new file mode 100644 index 0000000..a7204e4 --- /dev/null +++ b/changes/bug13840 @@ -0,0 +1,3 @@ + o Code simplifications and refactoring: + - In connection_exit_begin_conn(), use END_CIRC_REASON_TORPROTOCOL + constant instead of hardcoded value. Fixes issue 13840. diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index 14b3911..9ace375 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -2461,7 +2461,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) relay_header_unpack(&rh, cell->payload); if (rh.length > RELAY_PAYLOAD_SIZE) - return -1; + return -END_CIRC_REASON_TORPROTOCOL; /* Note: we have to use relay_send_command_from_edge here, not * connection_edge_end or connection_edge_send_command, since those require @@ -2479,7 +2479,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) r = begin_cell_parse(cell, &bcell, &end_reason); if (r < -1) { - return -1; + return -END_CIRC_REASON_TORPROTOCOL; } else if (r == -1) { tor_free(bcell.address); relay_send_end_cell_from_edge(rh.stream_id, circ, end_reason, NULL);

1 0

[depictor/master] Note some TODOs
by tom＠torproject.org 08 Dec '14

08 Dec '14

commit 2739306ffd41feb0a273a4fc76d6b5b65a7cb7b6 Author: Tom Ritter <tom(a)ritter.vg> Date: Mon Dec 8 07:42:34 2014 -0600 Note some TODOs --- website.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/website.py b/website.py index 7b88fb8..78fbd2a 100755 --- a/website.py +++ b/website.py @@ -43,6 +43,7 @@ class WebsiteWriter: def set_consensuses(self, c): self.consensuses = c self.consensus = max(c.itervalues(), key=operator.attrgetter('valid_after')) + # XXX - Change this to be a list of known dir auths, don't calculate it off the consensus because sometimes they're missing entirely self.known_authorities = set([r.nickname for r in self.consensus.routers.values() if 'Authority' in r.flags and r.nickname != "Tonga"]) self.known_authorities.update([r.nickname for r in self.consensus.directory_authorities]) def set_votes(self, v): @@ -911,7 +912,7 @@ class WebsiteWriter: Write the footer of the HTML page containing the blurb that is on every page of the metrics website. """ - #XXX Write the git version and steam version the page was generated with + #XXX Write the git version and stem version the page was generated with self.site.write("</div>\n" + "</div>\n" + "<div class=\"bottom\" id=\"bottom\">\n"

1 0

[depictor/master] Move stylesheet to out directory also
by tom＠torproject.org 08 Dec '14

08 Dec '14

commit 9fa4ea27f765a45984a2e114a1b9e51315df5385 Author: Tom Ritter <tom(a)ritter.vg> Date: Mon Dec 8 08:25:16 2014 -0500 Move stylesheet to out directory also --- out/stylesheet-ltr.css | 161 ++++++++++++++++++++++++++++++++++++++++++++++++ stylesheet-ltr.css | 161 ------------------------------------------------ 2 files changed, 161 insertions(+), 161 deletions(-) diff --git a/out/stylesheet-ltr.css b/out/stylesheet-ltr.css new file mode 100644 index 0000000..ce0c54e --- /dev/null +++ b/out/stylesheet-ltr.css @@ -0,0 +1,161 @@ +body { + background-color: white; + margin-top: 0px; + font-family: Arial, Helvetica, sans-serif; + font-size: 1em; + font-style: normal; + color: #000000; + padding-top: 0px; +} + +/* images */ + +img { + border: 0; +} + + +li { + margin: .2em .2em .2em 1em; +} + +/* this centers the page */ + +.center { + text-align: center; + background-color: white; + margin: 0px auto 0 auto; + width: 85%; +} + +.center table { + margin-left: auto; + margin-right: auto; + text-align: left; +} + +div.bottom { + font-size: 0.8em; + margin-top: 0.5cm; + margin-left: 1em; + margin-right: 1em; + text-align: center; +} + +/* The main column (left text) */ + +div.main-column { + padding: 15px 0 10px 10px; + text-indent: 0pt; + font-size: 1em; + direction: ltr; + text-align: left; +} + +/* formatting styles */ + +h2 { + font-size: 1.4em; + margin-bottom: 0em; + font-weight: bold; + margin-top: 0; +} + +h3 { + font-size: 1.2em; + margin-bottom: 0em; + font-weight: bold; + margin-top: 0; +} + +p { + margin-top: 0; + margin-bottom: 1em; +} + +a:link { + color: blue; + font-size: 1em; +} + +a:visited { + color: purple; + font-size: 1em; +} + +a.anchor { + font-size: 1em; + color: black; + font-weight: bold; + text-decoration: none; +} + +td { + vertical-align: top; +} + +/* the banner */ + +table.banner { + width: 100%; + height: 79px; + margin-left: auto; + margin-right: auto; +} + +td.banner-left { + /* This is done with an <img> in the HTML so it can be clickable + background-image: url("images/top-left.png"); + background-repeat: no-repeat; */ + width: 193px; +} + +td.banner-middle { + background-color: #00802B; + background-image: url("/images/top-middle.png"); + background-repeat: repeat-x; + vertical-align: bottom; + padding-bottom: 10px; + color: white; + font-weight: bold; + font-size: 1.2em; +} + +td.banner-middle a, td.banner-middle a:visited { + margin-right: 5px; + color: white; + font-weight: bold; + font-size: 1em; +} + +td.banner-middle a:hover { + color: #FF7F00; + font-weight: bold; + font-size: 1em; +} + +td.banner-right { + background-image: url("/images/top-right.png"); + background-repeat: no-repeat; + width: 15px; + background-position: right; + padding-top: 8px; +} + +.banner-middle a.current { + text-decoration: none; + color: #FF7F00; + font-weight: bold; + font-size: 1em; + width: auto; + left: -50px; +} + +hr { + background-color:#002200; + color:#666666; + font-size:1px; + height:1px; + line-height:0; + margin:15px 0 5px; +} diff --git a/stylesheet-ltr.css b/stylesheet-ltr.css deleted file mode 100644 index ce0c54e..0000000 --- a/stylesheet-ltr.css +++ /dev/null @@ -1,161 +0,0 @@ -body { - background-color: white; - margin-top: 0px; - font-family: Arial, Helvetica, sans-serif; - font-size: 1em; - font-style: normal; - color: #000000; - padding-top: 0px; -} - -/* images */ - -img { - border: 0; -} - - -li { - margin: .2em .2em .2em 1em; -} - -/* this centers the page */ - -.center { - text-align: center; - background-color: white; - margin: 0px auto 0 auto; - width: 85%; -} - -.center table { - margin-left: auto; - margin-right: auto; - text-align: left; -} - -div.bottom { - font-size: 0.8em; - margin-top: 0.5cm; - margin-left: 1em; - margin-right: 1em; - text-align: center; -} - -/* The main column (left text) */ - -div.main-column { - padding: 15px 0 10px 10px; - text-indent: 0pt; - font-size: 1em; - direction: ltr; - text-align: left; -} - -/* formatting styles */ - -h2 { - font-size: 1.4em; - margin-bottom: 0em; - font-weight: bold; - margin-top: 0; -} - -h3 { - font-size: 1.2em; - margin-bottom: 0em; - font-weight: bold; - margin-top: 0; -} - -p { - margin-top: 0; - margin-bottom: 1em; -} - -a:link { - color: blue; - font-size: 1em; -} - -a:visited { - color: purple; - font-size: 1em; -} - -a.anchor { - font-size: 1em; - color: black; - font-weight: bold; - text-decoration: none; -} - -td { - vertical-align: top; -} - -/* the banner */ - -table.banner { - width: 100%; - height: 79px; - margin-left: auto; - margin-right: auto; -} - -td.banner-left { - /* This is done with an <img> in the HTML so it can be clickable - background-image: url("images/top-left.png"); - background-repeat: no-repeat; */ - width: 193px; -} - -td.banner-middle { - background-color: #00802B; - background-image: url("/images/top-middle.png"); - background-repeat: repeat-x; - vertical-align: bottom; - padding-bottom: 10px; - color: white; - font-weight: bold; - font-size: 1.2em; -} - -td.banner-middle a, td.banner-middle a:visited { - margin-right: 5px; - color: white; - font-weight: bold; - font-size: 1em; -} - -td.banner-middle a:hover { - color: #FF7F00; - font-weight: bold; - font-size: 1em; -} - -td.banner-right { - background-image: url("/images/top-right.png"); - background-repeat: no-repeat; - width: 15px; - background-position: right; - padding-top: 8px; -} - -.banner-middle a.current { - text-decoration: none; - color: #FF7F00; - font-weight: bold; - font-size: 1em; - width: auto; - left: -50px; -} - -hr { - background-color:#002200; - color:#666666; - font-size:1px; - height:1px; - line-height:0; - margin:15px 0 5px; -}

1 0

[onionoo/master] Add new method to list recently stored documents.
by karsten＠torproject.org 08 Dec '14

08 Dec '14

commit 99d9f094722b1f0f1b0ba1b51ff53e942a06a263 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Oct 28 14:32:28 2014 +0100 Add new method to list recently stored documents. This is a rather big refactoring of how document writers obtain a list of documents they should write. The old way was to register a "fingerprint listener" at the descriptor source and write documents of all relays and bridges whose fingerprints it they learned about. One drawback (or even bug; cf. #12651) was that document writers didn't learn about relays or bridges that went offline, because there's no descriptor for that. The new way is for document writers to ask for a list of status files that have changed since the last "update status" file was written. This can be implemented for documents cached in memory and for documents stored on disk. Document writers can then use this list to only write the documents that need updating. --- .../org/torproject/onionoo/docs/DocumentStore.java | 85 +++++++++++++------- .../org/torproject/onionoo/docs/UpdateStatus.java | 29 ++++++- .../org/torproject/onionoo/server/NodeIndexer.java | 12 +-- .../onionoo/updater/DescriptorSource.java | 77 +----------------- .../onionoo/updater/FingerprintListener.java | 10 --- .../onionoo/writer/BandwidthDocumentWriter.java | 36 +++------ .../onionoo/writer/ClientsDocumentWriter.java | 35 +++----- .../onionoo/writer/DetailsDocumentWriter.java | 64 +++++++-------- .../onionoo/writer/UptimeDocumentWriter.java | 57 ++++--------- .../onionoo/writer/WeightsDocumentWriter.java | 43 +++------- .../torproject/onionoo/DummyDescriptorSource.java | 47 ----------- .../org/torproject/onionoo/DummyDocumentStore.java | 11 ++- .../torproject/onionoo/ResourceServletTest.java | 3 +- .../onionoo/UptimeDocumentWriterTest.java | 3 - 14 files changed, 174 insertions(+), 338 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java index 23ac03f..5137fb4 100644 --- a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java +++ b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java @@ -66,22 +66,41 @@ public class DocumentStore { private SortedMap<String, NodeStatus> cachedNodeStatuses; private SortedMap<String, SummaryDocument> cachedSummaryDocuments; + /* Last-modified timestamp of cached network statuses and summary + * documents when reading them from disk. */ + private long lastModifiedNodeStatuses = 0L; + private long lastModifiedSummaryDocuments = 0L; + + /* Fingerprints of updated node statuses and summary documents that are + * not yet written to disk. */ + private SortedSet<String> updatedNodeStatuses; + private SortedSet<String> updatedSummaryDocuments; + public <T extends Document> SortedSet<String> list( Class<T> documentType) { + return this.list(documentType, 0L); + } + + public <T extends Document> SortedSet<String> list( + Class<T> documentType, long updatedAfter) { if (documentType.equals(NodeStatus.class)) { - return this.listNodeStatuses(); + return this.listNodeStatuses(updatedAfter); } else if (documentType.equals(SummaryDocument.class)) { - return this.listSummaryDocuments(); + return this.listSummaryDocuments(updatedAfter); } else { - return this.listDocumentFiles(documentType); + return this.listDocumentFiles(documentType, updatedAfter); } } - private SortedSet<String> listNodeStatuses() { + private SortedSet<String> listNodeStatuses(long updatedAfter) { if (this.cachedNodeStatuses == null) { this.cacheNodeStatuses(); } - return new TreeSet<String>(this.cachedNodeStatuses.keySet()); + if (updatedAfter >= this.lastModifiedNodeStatuses) { + return new TreeSet<String>(this.updatedNodeStatuses); + } else { + return new TreeSet<String>(this.cachedNodeStatuses.keySet()); + } } private void cacheNodeStatuses() { @@ -105,6 +124,7 @@ public class DocumentStore { } } br.close(); + this.lastModifiedNodeStatuses = summaryFile.lastModified(); this.listedFiles += parsedNodeStatuses.size(); this.listOperations++; } catch (IOException e) { @@ -114,13 +134,18 @@ public class DocumentStore { } } this.cachedNodeStatuses = parsedNodeStatuses; + this.updatedNodeStatuses = new TreeSet<String>(); } - private SortedSet<String> listSummaryDocuments() { + private SortedSet<String> listSummaryDocuments(long updatedAfter) { if (this.cachedSummaryDocuments == null) { this.cacheSummaryDocuments(); } - return new TreeSet<String>(this.cachedSummaryDocuments.keySet()); + if (updatedAfter >= this.lastModifiedSummaryDocuments) { + return new TreeSet<String>(this.updatedSummaryDocuments); + } else { + return new TreeSet<String>(this.cachedSummaryDocuments.keySet()); + } } private void cacheSummaryDocuments() { @@ -147,6 +172,7 @@ public class DocumentStore { } } br.close(); + this.lastModifiedSummaryDocuments = summaryFile.lastModified(); this.listedFiles += parsedSummaryDocuments.size(); this.listOperations++; } catch (IOException e) { @@ -159,10 +185,11 @@ public class DocumentStore { } } this.cachedSummaryDocuments = parsedSummaryDocuments; + this.updatedSummaryDocuments = new TreeSet<String>(); } private <T extends Document> SortedSet<String> listDocumentFiles( - Class<T> documentType) { + Class<T> documentType, long updatedAfter) { SortedSet<String> fingerprints = new TreeSet<String>(); File directory = null; String subdirectory = null; @@ -204,8 +231,9 @@ public class DocumentStore { File file = files.pop(); if (file.isDirectory()) { files.addAll(Arrays.asList(file.listFiles())); - } else if (file.getName().length() == 40) { - fingerprints.add(file.getName()); + } else if (file.getName().length() == 40 && + (updatedAfter == 0L || file.lastModified() > updatedAfter)) { + fingerprints.add(file.getName()); } } } @@ -235,6 +263,7 @@ public class DocumentStore { if (this.cachedNodeStatuses == null) { this.cacheNodeStatuses(); } + this.updatedNodeStatuses.add(fingerprint); this.cachedNodeStatuses.put(fingerprint, nodeStatus); return true; } @@ -244,6 +273,7 @@ public class DocumentStore { if (this.cachedSummaryDocuments == null) { this.cacheSummaryDocuments(); } + this.updatedSummaryDocuments.add(fingerprint); this.cachedSummaryDocuments.put(fingerprint, summaryDocument); return true; } @@ -289,7 +319,8 @@ public class DocumentStore { } else if (document instanceof BandwidthStatus || document instanceof WeightsStatus || document instanceof ClientsStatus || - document instanceof UptimeStatus) { + document instanceof UptimeStatus || + document instanceof UpdateStatus) { documentString = document.toDocumentString(); } else { log.error("Serializing is not supported for type " @@ -435,7 +466,8 @@ public class DocumentStore { } else if (documentType.equals(BandwidthStatus.class) || documentType.equals(WeightsStatus.class) || documentType.equals(ClientsStatus.class) || - documentType.equals(UptimeStatus.class)) { + documentType.equals(UptimeStatus.class) || + documentType.equals(UpdateStatus.class)) { return this.retrieveParsedStatusFile(documentType, documentString); } else if (documentType.equals(DetailsStatus.class)) { return this.retrieveParsedDocumentFile(documentType, "{" @@ -523,6 +555,7 @@ public class DocumentStore { if (this.cachedNodeStatuses == null) { this.cacheNodeStatuses(); } + this.updatedNodeStatuses.remove(fingerprint); return this.cachedNodeStatuses.remove(fingerprint) != null; } @@ -530,6 +563,7 @@ public class DocumentStore { if (this.cachedSummaryDocuments == null) { this.cacheSummaryDocuments(); } + this.updatedSummaryDocuments.remove(fingerprint); return this.cachedSummaryDocuments.remove(fingerprint) != null; } @@ -550,9 +584,6 @@ public class DocumentStore { File documentFile = null; if (fingerprint == null && !documentType.equals(UpdateStatus.class) && !documentType.equals(UptimeStatus.class)) { - // TODO Instead of using the update file workaround, add new method - // lastModified(Class<T> documentType) that serves a similar - // purpose. return null; } File directory = null; @@ -631,6 +662,10 @@ public class DocumentStore { public void invalidateDocumentCache() { this.cachedNodeStatuses = null; this.cachedSummaryDocuments = null; + this.lastModifiedNodeStatuses = 0L; + this.lastModifiedSummaryDocuments = 0L; + this.updatedNodeStatuses = null; + this.updatedSummaryDocuments = null; } private void writeNodeStatuses() { @@ -675,6 +710,8 @@ public class DocumentStore { BufferedWriter bw = new BufferedWriter(new FileWriter(summaryFile)); bw.write(documentString); bw.close(); + this.lastModifiedNodeStatuses = summaryFile.lastModified(); + this.updatedNodeStatuses.clear(); this.storedFiles++; this.storedBytes += documentString.length(); } catch (IOException e) { @@ -703,6 +740,8 @@ public class DocumentStore { BufferedWriter bw = new BufferedWriter(new FileWriter(summaryFile)); bw.write(documentString); bw.close(); + this.lastModifiedSummaryDocuments = summaryFile.lastModified(); + this.updatedSummaryDocuments.clear(); this.storedFiles++; this.storedBytes += documentString.length(); } catch (IOException e) { @@ -715,19 +754,9 @@ public class DocumentStore { if (this.outDir == null) { return; } - File updateFile = new File(this.outDir, "update"); - String documentString = String.valueOf(this.time.currentTimeMillis()); - try { - updateFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter(updateFile)); - bw.write(documentString); - bw.close(); - this.storedFiles++; - this.storedBytes += documentString.length(); - } catch (IOException e) { - log.error("Could not write file '" - + updateFile.getAbsolutePath() + "'.", e); - } + UpdateStatus updateStatus = new UpdateStatus(); + updateStatus.setUpdatedMillis(this.time.currentTimeMillis()); + this.store(updateStatus); } public String getStatsString() { diff --git a/src/main/java/org/torproject/onionoo/docs/UpdateStatus.java b/src/main/java/org/torproject/onionoo/docs/UpdateStatus.java index 7bd710b..efdcf24 100644 --- a/src/main/java/org/torproject/onionoo/docs/UpdateStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/UpdateStatus.java @@ -1,7 +1,34 @@ -/* Copyright 2013 The Tor Project +/* Copyright 2013, 2014 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class UpdateStatus extends Document { + + private static Logger log = LoggerFactory.getLogger(UpdateStatus.class); + + private long updatedMillis; + public void setUpdatedMillis(long updatedMillis) { + this.updatedMillis = updatedMillis; + } + public long getUpdatedMillis() { + return this.updatedMillis; + } + + public void setFromDocumentString(String documentString) { + try { + this.updatedMillis = Long.parseLong(documentString.trim()); + } catch (NumberFormatException e) { + log.error("Could not parse timestamp '" + documentString + "'. " + + "Setting to 1970-01-01 00:00:00."); + this.updatedMillis = 0L; + } + } + + public String toDocumentString() { + return String.valueOf(this.updatedMillis); + } } diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java index bafd442..f8870db 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java @@ -99,15 +99,9 @@ public class NodeIndexer implements ServletContextListener, Runnable { long updateStatusMillis = -1L; DocumentStore documentStore = DocumentStoreFactory.getDocumentStore(); UpdateStatus updateStatus = documentStore.retrieve(UpdateStatus.class, - false); - if (updateStatus != null && - updateStatus.getDocumentString() != null) { - String updateString = updateStatus.getDocumentString(); - try { - updateStatusMillis = Long.parseLong(updateString.trim()); - } catch (NumberFormatException e) { - /* Handle below. */ - } + true); + if (updateStatus != null) { + updateStatusMillis = updateStatus.getUpdatedMillis(); } synchronized (this) { if (updateStatusMillis <= this.lastIndexed) { diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java index 30001f8..26b7b10 100644 --- a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java +++ b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java @@ -9,19 +9,10 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.torproject.descriptor.BridgeNetworkStatus; -import org.torproject.descriptor.BridgePoolAssignment; import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.ExitList; -import org.torproject.descriptor.ExitListEntry; -import org.torproject.descriptor.ExtraInfoDescriptor; -import org.torproject.descriptor.RelayNetworkStatusConsensus; -import org.torproject.descriptor.ServerDescriptor; import org.torproject.onionoo.util.FormattingUtils; public class DescriptorSource { @@ -39,8 +30,6 @@ public class DescriptorSource { this.descriptorQueues = new ArrayList<DescriptorQueue>(); this.descriptorListeners = new HashMap<DescriptorType, Set<DescriptorListener>>(); - this.fingerprintListeners = - new HashMap<DescriptorType, Set<FingerprintListener>>(); } private DescriptorQueue getDescriptorQueue( @@ -59,9 +48,6 @@ public class DescriptorSource { private Map<DescriptorType, Set<DescriptorListener>> descriptorListeners; - private Map<DescriptorType, Set<FingerprintListener>> - fingerprintListeners; - public void registerDescriptorListener(DescriptorListener listener, DescriptorType descriptorType) { if (!this.descriptorListeners.containsKey(descriptorType)) { @@ -71,15 +57,6 @@ public class DescriptorSource { this.descriptorListeners.get(descriptorType).add(listener); } - public void registerFingerprintListener(FingerprintListener listener, - DescriptorType descriptorType) { - if (!this.fingerprintListeners.containsKey(descriptorType)) { - this.fingerprintListeners.put(descriptorType, - new HashSet<FingerprintListener>()); - } - this.fingerprintListeners.get(descriptorType).add(listener); - } - public void downloadDescriptors() { for (DescriptorType descriptorType : DescriptorType.values()) { log.info("Loading: " + descriptorType); @@ -91,8 +68,7 @@ public class DescriptorSource { downloadedFiles = 0, deletedLocalFiles = 0; private void downloadDescriptors(DescriptorType descriptorType) { - if (!this.descriptorListeners.containsKey(descriptorType) && - !this.fingerprintListeners.containsKey(descriptorType)) { + if (!this.descriptorListeners.containsKey(descriptorType)) { return; } DescriptorDownloader descriptorDownloader = @@ -140,14 +116,11 @@ public class DescriptorSource { private void readDescriptors(DescriptorType descriptorType, DescriptorHistory descriptorHistory, boolean relay) { - if (!this.descriptorListeners.containsKey(descriptorType) && - !this.fingerprintListeners.containsKey(descriptorType)) { + if (!this.descriptorListeners.containsKey(descriptorType)) { return; } Set<DescriptorListener> descriptorListeners = this.descriptorListeners.get(descriptorType); - Set<FingerprintListener> fingerprintListeners = - this.fingerprintListeners.get(descriptorType); DescriptorQueue descriptorQueue = this.getDescriptorQueue( descriptorType, descriptorHistory); Descriptor descriptor; @@ -155,52 +128,6 @@ public class DescriptorSource { for (DescriptorListener descriptorListener : descriptorListeners) { descriptorListener.processDescriptor(descriptor, relay); } - if (fingerprintListeners == null) { - continue; - } - SortedSet<String> fingerprints = new TreeSet<String>(); - if (descriptorType == DescriptorType.RELAY_CONSENSUSES && - descriptor instanceof RelayNetworkStatusConsensus) { - fingerprints.addAll(((RelayNetworkStatusConsensus) descriptor). - getStatusEntries().keySet()); - } else if (descriptorType - == DescriptorType.RELAY_SERVER_DESCRIPTORS && - descriptor instanceof ServerDescriptor) { - fingerprints.add(((ServerDescriptor) descriptor). - getFingerprint()); - } else if (descriptorType == DescriptorType.RELAY_EXTRA_INFOS && - descriptor instanceof ExtraInfoDescriptor) { - fingerprints.add(((ExtraInfoDescriptor) descriptor). - getFingerprint()); - } else if (descriptorType == DescriptorType.EXIT_LISTS && - descriptor instanceof ExitList) { - for (ExitListEntry entry : - ((ExitList) descriptor).getExitListEntries()) { - fingerprints.add(entry.getFingerprint()); - } - } else if (descriptorType == DescriptorType.BRIDGE_STATUSES && - descriptor instanceof BridgeNetworkStatus) { - fingerprints.addAll(((BridgeNetworkStatus) descriptor). - getStatusEntries().keySet()); - } else if (descriptorType == - DescriptorType.BRIDGE_SERVER_DESCRIPTORS && - descriptor instanceof ServerDescriptor) { - fingerprints.add(((ServerDescriptor) descriptor). - getFingerprint()); - } else if (descriptorType == DescriptorType.BRIDGE_EXTRA_INFOS && - descriptor instanceof ExtraInfoDescriptor) { - fingerprints.add(((ExtraInfoDescriptor) descriptor). - getFingerprint()); - } else if (descriptorType == - DescriptorType.BRIDGE_POOL_ASSIGNMENTS && - descriptor instanceof BridgePoolAssignment) { - fingerprints.addAll(((BridgePoolAssignment) descriptor). - getEntries().keySet()); - } - for (FingerprintListener fingerprintListener : - fingerprintListeners) { - fingerprintListener.processFingerprints(fingerprints, relay); - } } switch (descriptorType) { case RELAY_CONSENSUSES: diff --git a/src/main/java/org/torproject/onionoo/updater/FingerprintListener.java b/src/main/java/org/torproject/onionoo/updater/FingerprintListener.java deleted file mode 100644 index 5e16eae..0000000 --- a/src/main/java/org/torproject/onionoo/updater/FingerprintListener.java +++ /dev/null @@ -1,10 +0,0 @@ -/* Copyright 2013, 2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo.updater; - -import java.util.SortedSet; - -public interface FingerprintListener { - abstract void processFingerprints(SortedSet<String> fingerprints, - boolean relay); -} \ No newline at end of file diff --git a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java index c5167d0..9bad965 100644 --- a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java @@ -3,11 +3,9 @@ package org.torproject.onionoo.writer; import java.util.ArrayList; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; @@ -19,47 +17,31 @@ import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; import org.torproject.onionoo.docs.GraphHistory; -import org.torproject.onionoo.updater.DescriptorSource; -import org.torproject.onionoo.updater.DescriptorSourceFactory; -import org.torproject.onionoo.updater.DescriptorType; -import org.torproject.onionoo.updater.FingerprintListener; +import org.torproject.onionoo.docs.UpdateStatus; import org.torproject.onionoo.util.TimeFactory; -public class BandwidthDocumentWriter implements FingerprintListener, - DocumentWriter{ +public class BandwidthDocumentWriter implements DocumentWriter { private static final Logger log = LoggerFactory.getLogger( BandwidthDocumentWriter.class); - private DescriptorSource descriptorSource; - private DocumentStore documentStore; private long now; public BandwidthDocumentWriter() { - this.descriptorSource = DescriptorSourceFactory.getDescriptorSource(); this.documentStore = DocumentStoreFactory.getDocumentStore(); this.now = TimeFactory.getTime().currentTimeMillis(); - this.registerFingerprintListeners(); - } - - private void registerFingerprintListeners() { - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.RELAY_EXTRA_INFOS); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.BRIDGE_EXTRA_INFOS); - } - - private Set<String> updateBandwidthDocuments = new HashSet<String>(); - - public void processFingerprints(SortedSet<String> fingerprints, - boolean relay) { - this.updateBandwidthDocuments.addAll(fingerprints); } public void writeDocuments() { - for (String fingerprint : this.updateBandwidthDocuments) { + UpdateStatus updateStatus = this.documentStore.retrieve( + UpdateStatus.class, true); + long updatedMillis = updateStatus != null ? + updateStatus.getUpdatedMillis() : 0L; + SortedSet<String> updateBandwidthDocuments = this.documentStore.list( + BandwidthStatus.class, updatedMillis); + for (String fingerprint : updateBandwidthDocuments) { BandwidthStatus bandwidthStatus = this.documentStore.retrieve( BandwidthStatus.class, true, fingerprint); if (bandwidthStatus == null) { diff --git a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java index 4086bc7..885494c 100644 --- a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java @@ -9,7 +9,6 @@ import java.util.Map; import java.util.SortedMap; import java.util.SortedSet; import java.util.TreeMap; -import java.util.TreeSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,10 +19,7 @@ import org.torproject.onionoo.docs.ClientsStatus; import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; -import org.torproject.onionoo.updater.DescriptorSource; -import org.torproject.onionoo.updater.DescriptorSourceFactory; -import org.torproject.onionoo.updater.DescriptorType; -import org.torproject.onionoo.updater.FingerprintListener; +import org.torproject.onionoo.docs.UpdateStatus; import org.torproject.onionoo.util.FormattingUtils; import org.torproject.onionoo.util.TimeFactory; @@ -50,43 +46,30 @@ import org.torproject.onionoo.util.TimeFactory; * "transports":{"obfs2":0.4581}, * "versions":{"v4":1.0000}} */ -public class ClientsDocumentWriter implements FingerprintListener, - DocumentWriter { +public class ClientsDocumentWriter implements DocumentWriter { private final static Logger log = LoggerFactory.getLogger( ClientsDocumentWriter.class); - private DescriptorSource descriptorSource; - private DocumentStore documentStore; private long now; public ClientsDocumentWriter() { - this.descriptorSource = DescriptorSourceFactory.getDescriptorSource(); this.documentStore = DocumentStoreFactory.getDocumentStore(); this.now = TimeFactory.getTime().currentTimeMillis(); - this.registerFingerprintListeners(); - } - - private void registerFingerprintListeners() { - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.BRIDGE_EXTRA_INFOS); - } - - private SortedSet<String> updateDocuments = new TreeSet<String>(); - - public void processFingerprints(SortedSet<String> fingerprints, - boolean relay) { - if (!relay) { - this.updateDocuments.addAll(fingerprints); - } } private int writtenDocuments = 0; public void writeDocuments() { - for (String hashedFingerprint : this.updateDocuments) { + UpdateStatus updateStatus = this.documentStore.retrieve( + UpdateStatus.class, true); + long updatedMillis = updateStatus != null ? + updateStatus.getUpdatedMillis() : 0L; + SortedSet<String> updateDocuments = this.documentStore.list( + ClientsStatus.class, updatedMillis); + for (String hashedFingerprint : updateDocuments) { ClientsStatus clientsStatus = this.documentStore.retrieve( ClientsStatus.class, true, hashedFingerprint); if (clientsStatus == null) { diff --git a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java index 47b7c79..0692070 100644 --- a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java @@ -16,59 +16,55 @@ import org.torproject.onionoo.docs.DetailsStatus; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; import org.torproject.onionoo.docs.NodeStatus; -import org.torproject.onionoo.updater.DescriptorSource; -import org.torproject.onionoo.updater.DescriptorSourceFactory; -import org.torproject.onionoo.updater.DescriptorType; -import org.torproject.onionoo.updater.FingerprintListener; +import org.torproject.onionoo.docs.UpdateStatus; import org.torproject.onionoo.util.TimeFactory; -public class DetailsDocumentWriter implements FingerprintListener, - DocumentWriter { +public class DetailsDocumentWriter implements DocumentWriter { private final static Logger log = LoggerFactory.getLogger( DetailsDocumentWriter.class); - private DescriptorSource descriptorSource; - private DocumentStore documentStore; private long now; public DetailsDocumentWriter() { - this.descriptorSource = DescriptorSourceFactory.getDescriptorSource(); this.documentStore = DocumentStoreFactory.getDocumentStore(); this.now = TimeFactory.getTime().currentTimeMillis(); - this.registerFingerprintListeners(); - } - - private void registerFingerprintListeners() { - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.RELAY_CONSENSUSES); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.RELAY_SERVER_DESCRIPTORS); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.BRIDGE_STATUSES); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.BRIDGE_SERVER_DESCRIPTORS); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.BRIDGE_POOL_ASSIGNMENTS); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.EXIT_LISTS); } private SortedSet<String> newRelays = new TreeSet<String>(), newBridges = new TreeSet<String>(); - public void processFingerprints(SortedSet<String> fingerprints, - boolean relay) { - if (relay) { - this.newRelays.addAll(fingerprints); - } else { - this.newBridges.addAll(fingerprints); - } - } - public void writeDocuments() { + UpdateStatus updateStatus = this.documentStore.retrieve( + UpdateStatus.class, true); + long updatedMillis = updateStatus != null ? + updateStatus.getUpdatedMillis() : 0L; + SortedSet<String> updatedNodeStatuses = this.documentStore.list( + NodeStatus.class, updatedMillis); + for (String fingerprint : updatedNodeStatuses) { + NodeStatus nodeStatus = this.documentStore.retrieve( + NodeStatus.class, true, fingerprint); + if (nodeStatus.isRelay()) { + newRelays.add(fingerprint); + } else { + newBridges.add(fingerprint); + } + } + SortedSet<String> updatedDetailsStatuses = this.documentStore.list( + DetailsStatus.class, updatedMillis); + for (String fingerprint : updatedDetailsStatuses) { + NodeStatus nodeStatus = this.documentStore.retrieve( + NodeStatus.class, true, fingerprint); + if (nodeStatus == null) { + continue; + } else if (nodeStatus.isRelay()) { + newRelays.add(fingerprint); + } else { + newBridges.add(fingerprint); + } + } this.updateRelayDetailsFiles(); this.updateBridgeDetailsFiles(); log.info("Wrote details document files"); diff --git a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java index 973fa67..093bc98 100644 --- a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java @@ -7,7 +7,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.SortedSet; -import java.util.TreeSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -15,52 +14,25 @@ import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; import org.torproject.onionoo.docs.GraphHistory; +import org.torproject.onionoo.docs.UpdateStatus; import org.torproject.onionoo.docs.UptimeDocument; import org.torproject.onionoo.docs.UptimeHistory; import org.torproject.onionoo.docs.UptimeStatus; -import org.torproject.onionoo.updater.DescriptorSource; -import org.torproject.onionoo.updater.DescriptorSourceFactory; -import org.torproject.onionoo.updater.DescriptorType; -import org.torproject.onionoo.updater.FingerprintListener; import org.torproject.onionoo.util.FormattingUtils; import org.torproject.onionoo.util.TimeFactory; -public class UptimeDocumentWriter implements FingerprintListener, - DocumentWriter { +public class UptimeDocumentWriter implements DocumentWriter { private final static Logger log = LoggerFactory.getLogger( UptimeDocumentWriter.class); - private DescriptorSource descriptorSource; - private DocumentStore documentStore; private long now; public UptimeDocumentWriter() { - this.descriptorSource = DescriptorSourceFactory.getDescriptorSource(); this.documentStore = DocumentStoreFactory.getDocumentStore(); this.now = TimeFactory.getTime().currentTimeMillis(); - this.registerFingerprintListeners(); - } - - private void registerFingerprintListeners() { - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.RELAY_CONSENSUSES); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.BRIDGE_STATUSES); - } - - private SortedSet<String> newRelayFingerprints = new TreeSet<String>(), - newBridgeFingerprints = new TreeSet<String>(); - - public void processFingerprints(SortedSet<String> fingerprints, - boolean relay) { - if (relay) { - this.newRelayFingerprints.addAll(fingerprints); - } else { - this.newBridgeFingerprints.addAll(fingerprints); - } } public void writeDocuments() { @@ -69,29 +41,34 @@ public class UptimeDocumentWriter implements FingerprintListener, if (uptimeStatus == null) { return; } - for (String fingerprint : this.newRelayFingerprints) { - this.updateDocument(true, fingerprint, - uptimeStatus.getRelayHistory()); - } - for (String fingerprint : this.newBridgeFingerprints) { - this.updateDocument(false, fingerprint, - uptimeStatus.getBridgeHistory()); + UpdateStatus updateStatus = this.documentStore.retrieve( + UpdateStatus.class, true); + long updatedMillis = updateStatus != null ? + updateStatus.getUpdatedMillis() : 0L; + SortedSet<String> updatedUptimeStatuses = this.documentStore.list( + UptimeStatus.class, updatedMillis); + for (String fingerprint : updatedUptimeStatuses) { + this.updateDocument(fingerprint, uptimeStatus); } log.info("Wrote uptime document files"); } private int writtenDocuments = 0; - private void updateDocument(boolean relay, String fingerprint, - SortedSet<UptimeHistory> knownStatuses) { + private void updateDocument(String fingerprint, + UptimeStatus knownStatuses) { UptimeStatus uptimeStatus = this.documentStore.retrieve( UptimeStatus.class, true, fingerprint); if (uptimeStatus != null) { + boolean relay = uptimeStatus.getBridgeHistory().isEmpty(); SortedSet<UptimeHistory> history = relay ? uptimeStatus.getRelayHistory() : uptimeStatus.getBridgeHistory(); + SortedSet<UptimeHistory> knownStatusesHistory = relay + ? knownStatuses.getRelayHistory() + : knownStatuses.getBridgeHistory(); UptimeDocument uptimeDocument = this.compileUptimeDocument(relay, - fingerprint, history, knownStatuses); + fingerprint, history, knownStatusesHistory); this.documentStore.store(uptimeDocument, fingerprint); this.writtenDocuments++; } diff --git a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java index 00c529e..7d52a47 100644 --- a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java @@ -3,11 +3,9 @@ package org.torproject.onionoo.writer; import java.util.ArrayList; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; @@ -17,56 +15,33 @@ import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; import org.torproject.onionoo.docs.GraphHistory; +import org.torproject.onionoo.docs.UpdateStatus; import org.torproject.onionoo.docs.WeightsDocument; import org.torproject.onionoo.docs.WeightsStatus; -import org.torproject.onionoo.updater.DescriptorSource; -import org.torproject.onionoo.updater.DescriptorSourceFactory; -import org.torproject.onionoo.updater.DescriptorType; -import org.torproject.onionoo.updater.FingerprintListener; import org.torproject.onionoo.util.TimeFactory; -public class WeightsDocumentWriter implements FingerprintListener, - DocumentWriter { +public class WeightsDocumentWriter implements DocumentWriter { private final static Logger log = LoggerFactory.getLogger( WeightsDocumentWriter.class); - private DescriptorSource descriptorSource; - private DocumentStore documentStore; private long now; public WeightsDocumentWriter() { - this.descriptorSource = DescriptorSourceFactory.getDescriptorSource(); this.documentStore = DocumentStoreFactory.getDocumentStore(); this.now = TimeFactory.getTime().currentTimeMillis(); - this.registerFingerprintListeners(); - } - - private void registerFingerprintListeners() { - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.RELAY_CONSENSUSES); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.RELAY_SERVER_DESCRIPTORS); - } - - private Set<String> updateWeightsDocuments = new HashSet<String>(); - - public void processFingerprints(SortedSet<String> fingerprints, - boolean relay) { - if (relay) { - this.updateWeightsDocuments.addAll(fingerprints); - } } public void writeDocuments() { - this.writeWeightsDataFiles(); - log.info("Wrote weights document files"); - } - - private void writeWeightsDataFiles() { - for (String fingerprint : this.updateWeightsDocuments) { + UpdateStatus updateStatus = this.documentStore.retrieve( + UpdateStatus.class, true); + long updatedMillis = updateStatus != null ? + updateStatus.getUpdatedMillis() : 0L; + SortedSet<String> updateWeightsDocuments = this.documentStore.list( + WeightsStatus.class, updatedMillis); + for (String fingerprint : updateWeightsDocuments) { WeightsStatus weightsStatus = this.documentStore.retrieve( WeightsStatus.class, true, fingerprint); if (weightsStatus == null) { diff --git a/src/test/java/org/torproject/onionoo/DummyDescriptorSource.java b/src/test/java/org/torproject/onionoo/DummyDescriptorSource.java index e93b063..4dbf066 100644 --- a/src/test/java/org/torproject/onionoo/DummyDescriptorSource.java +++ b/src/test/java/org/torproject/onionoo/DummyDescriptorSource.java @@ -5,14 +5,11 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; import org.torproject.descriptor.Descriptor; import org.torproject.onionoo.updater.DescriptorListener; import org.torproject.onionoo.updater.DescriptorSource; import org.torproject.onionoo.updater.DescriptorType; -import org.torproject.onionoo.updater.FingerprintListener; public class DummyDescriptorSource extends DescriptorSource { @@ -39,27 +36,6 @@ public class DummyDescriptorSource extends DescriptorSource { return this.descriptors.get(descriptorType); } - private Map<DescriptorType, SortedSet<String>> fingerprints = - new HashMap<DescriptorType, SortedSet<String>>(); - - public void addFingerprints(DescriptorType descriptorType, - Collection<String> fingerprints) { - this.getFingerprintsByType(descriptorType).addAll(fingerprints); - } - - public void addFingerprint(DescriptorType descriptorType, - String fingerprint) { - this.getFingerprintsByType(descriptorType).add(fingerprint); - } - - private SortedSet<String> getFingerprintsByType( - DescriptorType descriptorType) { - if (!this.fingerprints.containsKey(descriptorType)) { - this.fingerprints.put(descriptorType, new TreeSet<String>()); - } - return this.fingerprints.get(descriptorType); - } - private Map<DescriptorType, Set<DescriptorListener>> descriptorListeners = new HashMap<DescriptorType, Set<DescriptorListener>>(); @@ -73,23 +49,9 @@ public class DummyDescriptorSource extends DescriptorSource { this.descriptorListeners.get(descriptorType).add(listener); } - private Map<DescriptorType, Set<FingerprintListener>> - fingerprintListeners = new HashMap<DescriptorType, - Set<FingerprintListener>>(); - - public void registerFingerprintListener(FingerprintListener listener, - DescriptorType descriptorType) { - if (!this.fingerprintListeners.containsKey(descriptorType)) { - this.fingerprintListeners.put(descriptorType, - new HashSet<FingerprintListener>()); - } - this.fingerprintListeners.get(descriptorType).add(listener); - } - public void readDescriptors() { Set<DescriptorType> descriptorTypes = new HashSet<DescriptorType>(); descriptorTypes.addAll(this.descriptorListeners.keySet()); - descriptorTypes.addAll(this.fingerprintListeners.keySet()); for (DescriptorType descriptorType : descriptorTypes) { boolean relay; switch (descriptorType) { @@ -118,15 +80,6 @@ public class DummyDescriptorSource extends DescriptorSource { } } } - if (this.fingerprints.containsKey(descriptorType) && - this.fingerprintListeners.containsKey(descriptorType)) { - Set<FingerprintListener> listeners = - this.fingerprintListeners.get(descriptorType); - for (FingerprintListener listener : listeners) { - listener.processFingerprints(this.getFingerprintsByType( - descriptorType), relay); - } - } } } diff --git a/src/test/java/org/torproject/onionoo/DummyDocumentStore.java b/src/test/java/org/torproject/onionoo/DummyDocumentStore.java index 54311aa..f5601b0 100644 --- a/src/test/java/org/torproject/onionoo/DummyDocumentStore.java +++ b/src/test/java/org/torproject/onionoo/DummyDocumentStore.java @@ -54,10 +54,17 @@ public class DummyDocumentStore extends DocumentStore { } public <T extends Document> SortedSet<String> list( + Class<T> documentType, long modifiedAfter) { + return this.list(documentType); + } + + public <T extends Document> SortedSet<String> list( Class<T> documentType) { this.performedListOperations++; - return new TreeSet<String>(this.getStoredDocumentsByClass( - documentType).keySet()); + SortedSet<String> fingerprints = new TreeSet<String>( + this.getStoredDocumentsByClass(documentType).keySet()); + fingerprints.remove(FINGERPRINT_NULL); + return fingerprints; } private int performedRemoveOperations = 0; diff --git a/src/test/java/org/torproject/onionoo/ResourceServletTest.java b/src/test/java/org/torproject/onionoo/ResourceServletTest.java index 1631661..cc48295 100644 --- a/src/test/java/org/torproject/onionoo/ResourceServletTest.java +++ b/src/test/java/org/torproject/onionoo/ResourceServletTest.java @@ -210,8 +210,7 @@ public class ResourceServletTest { private void createDummyDocumentStore() { DummyDocumentStore documentStore = new DummyDocumentStore(); UpdateStatus updateStatus = new UpdateStatus(); - updateStatus.setDocumentString(String.valueOf( - this.currentTimeMillis)); + updateStatus.setUpdatedMillis(this.currentTimeMillis); documentStore.addDocument(updateStatus, null); for (Map.Entry<String, org.torproject.onionoo.docs.SummaryDocument> e : this.relays.entrySet()) { diff --git a/src/test/java/org/torproject/onionoo/UptimeDocumentWriterTest.java b/src/test/java/org/torproject/onionoo/UptimeDocumentWriterTest.java index 9b19e79..8f7fff0 100644 --- a/src/test/java/org/torproject/onionoo/UptimeDocumentWriterTest.java +++ b/src/test/java/org/torproject/onionoo/UptimeDocumentWriterTest.java @@ -16,7 +16,6 @@ import org.torproject.onionoo.docs.GraphHistory; import org.torproject.onionoo.docs.UptimeDocument; import org.torproject.onionoo.docs.UptimeStatus; import org.torproject.onionoo.updater.DescriptorSourceFactory; -import org.torproject.onionoo.updater.DescriptorType; import org.torproject.onionoo.util.TimeFactory; import org.torproject.onionoo.writer.UptimeDocumentWriter; @@ -71,8 +70,6 @@ public class UptimeDocumentWriterTest { status = new UptimeStatus(); status.setFromDocumentString(gabelmooUptime); this.documentStore.addDocument(status, GABELMOO_FINGERPRINT); - this.descriptorSource.addFingerprint(DescriptorType.RELAY_CONSENSUSES, - GABELMOO_FINGERPRINT); } private static final long ONE_SECOND = 1000L,

1 0

[onionoo/master] Clarify responsibilities of node and details statuses.
by karsten＠torproject.org 08 Dec '14

08 Dec '14

commit a96e55ce18e3e4b89eae5bc6e318db6217460192 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Oct 29 22:39:06 2014 +0100 Clarify responsibilities of node and details statuses. This big refactoring patch clarifies responsibilities of node and details statuses. These responsibilities have been watered down by past feature implementations which made it hard to add future enhancements. This patch builds upon the earlier separation of node status parts into summary documents. It does not change the role of details documents. This patch is an important requirement for separating the update and the write step of the hourly updater. New responsibilities of node and details statuses are: - Node statuses hold selected information about relays and bridges that must be retrieved efficiently in the update process. They are kept in memory and not written to disk until the update process has completed. They are also the sole basis for writing summary documents that are used in the server package to build the node index. - Details statuses hold detailed information about relays and bridges and are written only once at the end of the update process. They are not kept in memory. They are the sole basis for writing details documents. Changes in more detail: - NodeStatus now only contains attributes that are made persistent. Previously, some attributes, like lookup results, were set by NodeDetailsStatusUpdater and later read by DetailsDocumentWriter, but that approach will break as soon as the two classes may be executed in distinct runs. - NodeStatus now has a constructor with fewer arguments, and attributes now have setters. NodeStatus also doesn't have the confusing update() method anymore. - DetailsStatus contains the removed attributes from NodeStatus which are made persistent and used by DetailsDocumentWriter to create and write DetailsDocument instances. - NodeDetailsStatusUpdater tries harder not to retrieve and update DetailsStatus documents in order to reduce disk I/O operations. DetailsStatus documents are only retrieved and possibly written when parsing server descriptors and at the end of the update process. - NodeDetailsStatusUpdater handles bridge pool assignments in the same way as exit lists: by storing updates in memory during the parse step and updating DetailsStatus instances as part of the update step. - DetailsDocumentWriter only uses DetailsStatus documents and no NodeStatus documents to write new DetailsDocument files. --- .../torproject/onionoo/docs/DetailsDocument.java | 23 +- .../org/torproject/onionoo/docs/DetailsStatus.java | 273 +++++++++- .../org/torproject/onionoo/docs/NodeStatus.java | 565 ++++++++----------- .../onionoo/updater/NodeDetailsStatusUpdater.java | 571 ++++++++++++-------- .../onionoo/writer/DetailsDocumentWriter.java | 270 ++++----- .../onionoo/writer/SummaryDocumentWriter.java | 21 +- 6 files changed, 978 insertions(+), 745 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java index 455097a..86abf9f 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java @@ -2,8 +2,10 @@ * See LICENSE for licensing information */ package org.torproject.onionoo.docs; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.SortedSet; import org.apache.commons.lang3.StringEscapeUtils; import org.apache.commons.lang3.StringUtils; @@ -61,10 +63,11 @@ public class DetailsDocument extends Document { private List<String> exit_addresses; public void setExitAddresses(List<String> exitAddresses) { - this.exit_addresses = exitAddresses; + this.exit_addresses = !exitAddresses.isEmpty() ? exitAddresses : null; } public List<String> getExitAddresses() { - return this.exit_addresses; + return this.exit_addresses == null ? new ArrayList<String>() + : this.exit_addresses; } private String dir_address; @@ -109,11 +112,11 @@ public class DetailsDocument extends Document { return this.running; } - private List<String> flags; - public void setFlags(List<String> flags) { + private SortedSet<String> flags; + public void setFlags(SortedSet<String> flags) { this.flags = flags; } - public List<String> getFlags() { + public SortedSet<String> getFlags() { return this.flags; } @@ -198,11 +201,13 @@ public class DetailsDocument extends Document { } private String last_restarted; - public void setLastRestarted(long lastRestarted) { - this.last_restarted = DateTimeHelper.format(lastRestarted); + public void setLastRestarted(Long lastRestarted) { + this.last_restarted = (lastRestarted == null ? null : + DateTimeHelper.format(lastRestarted)); } - public long getLastRestarted() { - return DateTimeHelper.parse(this.last_restarted); + public Long getLastRestarted() { + return this.last_restarted == null ? null : + DateTimeHelper.parse(this.last_restarted); } private Integer bandwidth_rate; diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java index 42c835f..967e493 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java @@ -4,6 +4,8 @@ package org.torproject.onionoo.docs; import java.util.List; import java.util.Map; +import java.util.SortedSet; +import java.util.TreeSet; import org.apache.commons.lang3.StringEscapeUtils; import org.apache.commons.lang3.StringUtils; @@ -27,20 +29,24 @@ public class DetailsStatus extends Document { new String[] { "'" }, new String[] { "\\'" })); } + /* From most recently published server descriptor: */ + private String desc_published; - public void setDescPublished(long descPublished) { + public void setDescPublished(Long descPublished) { this.desc_published = DateTimeHelper.format(descPublished); } - public long getDescPublished() { - return DateTimeHelper.parse(this.desc_published); + public Long getDescPublished() { + return this.desc_published == null ? null : + DateTimeHelper.parse(this.desc_published); } private String last_restarted; - public void setLastRestarted(long lastRestarted) { + public void setLastRestarted(Long lastRestarted) { this.last_restarted = DateTimeHelper.format(lastRestarted); } - public long getLastRestarted() { - return DateTimeHelper.parse(this.last_restarted); + public Long getLastRestarted() { + return this.last_restarted == null ? null : + DateTimeHelper.parse(this.last_restarted); } private Integer bandwidth_rate; @@ -124,6 +130,149 @@ public class DetailsStatus extends Document { return this.hibernating; } + /* From network status entries: */ + + private boolean is_relay; + public void setRelay(boolean isRelay) { + this.is_relay = isRelay; + } + public boolean isRelay() { + return this.is_relay; + } + + private boolean running; + public void setRunning(boolean isRunning) { + this.running = isRunning; + } + public boolean isRunning() { + return this.running; + } + + private String nickname; + public void setNickname(String nickname) { + this.nickname = nickname; + } + public String getNickname() { + return this.nickname; + } + + private String address; + public void setAddress(String address) { + this.address = address; + } + public String getAddress() { + return this.address; + } + + private SortedSet<String> or_addresses_and_ports; + public void setOrAddressesAndPorts( + SortedSet<String> orAddressesAndPorts) { + this.or_addresses_and_ports = orAddressesAndPorts; + } + public SortedSet<String> getOrAddressesAndPorts() { + return this.or_addresses_and_ports == null ? new TreeSet<String>() : + this.or_addresses_and_ports; + } + public SortedSet<String> getOrAddresses() { + SortedSet<String> orAddresses = new TreeSet<String>(); + if (this.address != null) { + orAddresses.add(this.address); + } + if (this.or_addresses_and_ports != null) { + for (String orAddressAndPort : this.or_addresses_and_ports) { + if (orAddressAndPort.contains(":")) { + String orAddress = orAddressAndPort.substring(0, + orAddressAndPort.lastIndexOf(':')); + orAddresses.add(orAddress); + } + } + } + return orAddresses; + } + + private long first_seen_millis; + public void setFirstSeenMillis(long firstSeenMillis) { + this.first_seen_millis = firstSeenMillis; + } + public long getFirstSeenMillis() { + return this.first_seen_millis; + } + + private long last_seen_millis; + public void setLastSeenMillis(long lastSeenMillis) { + this.last_seen_millis = lastSeenMillis; + } + public long getLastSeenMillis() { + return this.last_seen_millis; + } + + private int or_port; + public void setOrPort(int orPort) { + this.or_port = orPort; + } + public int getOrPort() { + return this.or_port; + } + + private int dir_port; + public void setDirPort(int dirPort) { + this.dir_port = dirPort; + } + public int getDirPort() { + return this.dir_port; + } + + private SortedSet<String> relay_flags; + public void setRelayFlags(SortedSet<String> relayFlags) { + this.relay_flags = relayFlags; + } + public SortedSet<String> getRelayFlags() { + return this.relay_flags; + } + + private long consensus_weight; + public void setConsensusWeight(long consensusWeight) { + this.consensus_weight = consensusWeight; + } + public long getConsensusWeight() { + return this.consensus_weight; + } + + private String default_policy; + public void setDefaultPolicy(String defaultPolicy) { + this.default_policy = defaultPolicy; + } + public String getDefaultPolicy() { + return this.default_policy; + } + + private String port_list; + public void setPortList(String portList) { + this.port_list = portList; + } + public String getPortList() { + return this.port_list; + } + + private long last_changed_or_address_or_port; + public void setLastChangedOrAddressOrPort( + long lastChangedOrAddressOrPort) { + this.last_changed_or_address_or_port = lastChangedOrAddressOrPort; + } + public long getLastChangedOrAddressOrPort() { + return this.last_changed_or_address_or_port; + } + + private Boolean recommended_version; + public void setRecommendedVersion(Boolean recommendedVersion) { + this.recommended_version = recommendedVersion; + } + public Boolean getRecommendedVersion() { + return this.recommended_version; + } + + /* From bridge pool assignments: */ + private String pool_assignment; public void setPoolAssignment(String poolAssignment) { this.pool_assignment = poolAssignment; @@ -132,6 +281,8 @@ public class DetailsStatus extends Document { return this.pool_assignment; } + /* From exit lists: */ + private Map<String, Long> exit_addresses; public void setExitAddresses(Map<String, Long> exitAddresses) { this.exit_addresses = exitAddresses; @@ -139,4 +290,114 @@ public class DetailsStatus extends Document { public Map<String, Long> getExitAddresses() { return this.exit_addresses; } + + /* Calculated path-selection probabilities: */ + + private Float consensus_weight_fraction; + public void setConsensusWeightFraction(Float consensusWeightFraction) { + this.consensus_weight_fraction = consensusWeightFraction; + } + public Float getConsensusWeightFraction() { + return this.consensus_weight_fraction; + } + + private Float guard_probability; + public void setGuardProbability(Float guardProbability) { + this.guard_probability = guardProbability; + } + public Float getGuardProbability() { + return this.guard_probability; + } + + private Float middle_probability; + public void setMiddleProbability(Float middleProbability) { + this.middle_probability = middleProbability; + } + public Float getMiddleProbability() { + return this.middle_probability; + } + + private Float exit_probability; + public void setExitProbability(Float exitProbability) { + this.exit_probability = exitProbability; + } + public Float getExitProbability() { + return this.exit_probability; + } + + /* GeoIP lookup results: */ + + private Float latitude; + public void setLatitude(Float latitude) { + this.latitude = latitude; + } + public Float getLatitude() { + return this.latitude; + } + + private Float longitude; + public void setLongitude(Float longitude) { + this.longitude = longitude; + } + public Float getLongitude() { + return this.longitude; + } + + private String country_code; + public void setCountryCode(String countryCode) { + this.country_code = countryCode; + } + public String getCountryCode() { + return this.country_code; + } + + private String country_name; + public void setCountryName(String countryName) { + this.country_name = countryName; + } + public String getCountryName() { + return this.country_name; + } + + private String region_name; + public void setRegionName(String regionName) { + this.region_name = regionName; + } + public String getRegionName() { + return this.region_name; + } + + private String city_name; + public void setCityName(String cityName) { + this.city_name = cityName; + } + public String getCityName() { + return this.city_name; + } + + private String as_name; + public void setASName(String aSName) { + this.as_name = aSName; + } + public String getASName() { + return this.as_name; + } + + private String as_number; + public void setASNumber(String aSNumber) { + this.as_number = aSNumber; + } + public String getASNumber() { + return this.as_number; + } + + /* Reverse DNS lookup result: */ + + private String host_name; + public void setHostName(String hostName) { + this.host_name = hostName; + } + public String getHostName() { + return this.host_name; + } } diff --git a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java index ced6cbe..e3dcdaf 100644 --- a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java @@ -18,144 +18,148 @@ import java.util.TreeSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/* Store search data of a single relay that was running in the past seven - * days. */ public class NodeStatus extends Document { private final static Logger log = LoggerFactory.getLogger( NodeStatus.class); - private boolean isRelay; - public boolean isRelay() { - return this.isRelay; - } - - private String fingerprint; - public String getFingerprint() { - return this.fingerprint; - } - - private String nickname; - public String getNickname() { - return this.nickname; - } - - private String address; - public String getAddress() { - return this.address; - } + /* From most recently published server descriptor: */ - private SortedSet<String> orAddresses; - private SortedSet<String> orAddressesAndPorts; - public SortedSet<String> getOrAddresses() { - return new TreeSet<String>(this.orAddresses); - } - public void addOrAddressAndPort(String orAddressAndPort) { - if (orAddressAndPort.contains(":") && orAddressAndPort.length() > 0) { - String orAddress = orAddressAndPort.substring(0, - orAddressAndPort.lastIndexOf(':')); - if (this.exitAddresses.contains(orAddress)) { - this.exitAddresses.remove(orAddress); + private String contact; + public void setContact(String contact) { + if (contact == null) { + this.contact = null; + } else { + StringBuilder sb = new StringBuilder(); + for (char c : contact.toLowerCase().toCharArray()) { + if (c >= 32 && c < 127) { + sb.append(c); + } else { + sb.append(" "); + } } - this.orAddresses.add(orAddress); - this.orAddressesAndPorts.add(orAddressAndPort); + this.contact = sb.toString(); } } - public SortedSet<String> getOrAddressesAndPorts() { - return new TreeSet<String>(this.orAddressesAndPorts); + public String getContact() { + return this.contact; } - private SortedSet<String> exitAddresses; - public void addExitAddress(String exitAddress) { - if (exitAddress.length() > 0 && !this.address.equals(exitAddress) && - !this.orAddresses.contains(exitAddress)) { - this.exitAddresses.add(exitAddress); - } + private String[] familyFingerprints; + public void setFamilyFingerprints( + SortedSet<String> familyFingerprints) { + this.familyFingerprints = collectionToStringArray(familyFingerprints); } - public SortedSet<String> getExitAddresses() { - return new TreeSet<String>(this.exitAddresses); + public SortedSet<String> getFamilyFingerprints() { + return stringArrayToSortedSet(this.familyFingerprints); } - private Float latitude; - public void setLatitude(Float latitude) { - this.latitude = latitude; + private static String[] collectionToStringArray( + Collection<String> collection) { + String[] stringArray = null; + if (collection != null && !collection.isEmpty()) { + stringArray = new String[collection.size()]; + int i = 0; + for (String string : collection) { + stringArray[i++] = string; + } + } + return stringArray; } - public Float getLatitude() { - return this.latitude; + private SortedSet<String> stringArrayToSortedSet(String[] stringArray) { + SortedSet<String> sortedSet = new TreeSet<String>(); + if (stringArray != null) { + sortedSet.addAll(Arrays.asList(stringArray)); + } + return sortedSet; } - private Float longitude; - public void setLongitude(Float longitude) { - this.longitude = longitude; - } - public Float getLongitude() { - return this.longitude; - } + /* From network status entries: */ - private String countryCode; - public void setCountryCode(String countryCode) { - this.countryCode = countryCode; + private boolean isRelay; + public void setRelay(boolean isRelay) { + this.isRelay = isRelay; } - public String getCountryCode() { - return this.countryCode; + public boolean isRelay() { + return this.isRelay; } - private String countryName; - public void setCountryName(String countryName) { - this.countryName = countryName; - } - public String getCountryName() { - return this.countryName; + private final String fingerprint; + public String getFingerprint() { + return this.fingerprint; } - private String regionName; - public void setRegionName(String regionName) { - this.regionName = regionName; + private String nickname; + public void setNickname(String nickname) { + this.nickname = nickname; } - public String getRegionName() { - return this.regionName; + public String getNickname() { + return this.nickname; } - private String cityName; - public void setCityName(String cityName) { - this.cityName = cityName; + private String address; + public void setAddress(String address) { + this.address = address; } - public String getCityName() { - return this.cityName; + public String getAddress() { + return this.address; } - private String aSName; - public void setASName(String aSName) { - this.aSName = aSName; - } - public String getASName() { - return this.aSName; + private SortedSet<String> orAddressesAndPorts; + public void setOrAddressesAndPorts( + SortedSet<String> orAddressesAndPorts) { + this.orAddressesAndPorts = orAddressesAndPorts; } - - private String aSNumber; - public void setASNumber(String aSNumber) { - this.aSNumber = aSNumber; + public SortedSet<String> getOrAddressesAndPorts() { + return this.orAddressesAndPorts == null ? new TreeSet<String>() : + this.orAddressesAndPorts; } - public String getASNumber() { - return this.aSNumber; + public SortedSet<String> getOrAddresses() { + SortedSet<String> orAddresses = new TreeSet<String>(); + if (this.address != null) { + orAddresses.add(this.address); + } + if (this.orAddressesAndPorts != null) { + for (String orAddressAndPort : this.orAddressesAndPorts) { + if (orAddressAndPort.contains(":") && + orAddressAndPort.length() > 0) { + String orAddress = orAddressAndPort.substring(0, + orAddressAndPort.lastIndexOf(':')); + orAddresses.add(orAddress); + } + } + } + return orAddresses; } private long firstSeenMillis; + public void setFirstSeenMillis(long firstSeenMillis) { + this.firstSeenMillis = firstSeenMillis; + } public long getFirstSeenMillis() { return this.firstSeenMillis; } private long lastSeenMillis; + public void setLastSeenMillis(long lastSeenMillis) { + this.lastSeenMillis = lastSeenMillis; + } public long getLastSeenMillis() { return this.lastSeenMillis; } private int orPort; + public void setOrPort(int orPort) { + this.orPort = orPort; + } public int getOrPort() { return this.orPort; } private int dirPort; + public void setDirPort(int dirPort) { + this.dirPort = dirPort; + } public int getDirPort() { return this.dirPort; } @@ -189,86 +193,54 @@ public class NodeStatus extends Document { } private long consensusWeight; + public void setConsensusWeight(long consensusWeight) { + this.consensusWeight = consensusWeight; + } public long getConsensusWeight() { return this.consensusWeight; } - private boolean running; - public void setRunning(boolean running) { - this.running = running; - } - public boolean getRunning() { - return this.running; - } - - private String hostName; - public void setHostName(String hostName) { - this.hostName = hostName; - } - public String getHostName() { - return this.hostName; - } - - private long lastRdnsLookup = -1L; - public void setLastRdnsLookup(long lastRdnsLookup) { - this.lastRdnsLookup = lastRdnsLookup; - } - public long getLastRdnsLookup() { - return this.lastRdnsLookup; - } - - private double consensusWeightFraction = -1.0; - public void setConsensusWeightFraction(double consensusWeightFraction) { - this.consensusWeightFraction = consensusWeightFraction; - } - public double getConsensusWeightFraction() { - return this.consensusWeightFraction; - } - - private double guardProbability = -1.0; - public void setGuardProbability(double guardProbability) { - this.guardProbability = guardProbability; - } - public double getGuardProbability() { - return this.guardProbability; - } - - private double middleProbability = -1.0; - public void setMiddleProbability(double middleProbability) { - this.middleProbability = middleProbability; - } - public double getMiddleProbability() { - return this.middleProbability; - } - - private double exitProbability = -1.0; - public void setExitProbability(double exitProbability) { - this.exitProbability = exitProbability; - } - public double getExitProbability() { - return this.exitProbability; - } - private String defaultPolicy; + public void setDefaultPolicy(String defaultPolicy) { + this.defaultPolicy = defaultPolicy; + } public String getDefaultPolicy() { return this.defaultPolicy; } private String portList; + public void setPortList(String portList) { + this.portList = portList; + } public String getPortList() { return this.portList; } - private SortedMap<Long, Set<String>> lastAddresses; + private SortedMap<Long, Set<String>> lastAddresses = + new TreeMap<Long, Set<String>>(Collections.reverseOrder()); public SortedMap<Long, Set<String>> getLastAddresses() { - return this.lastAddresses == null ? null : - new TreeMap<Long, Set<String>>(this.lastAddresses); + return new TreeMap<Long, Set<String>>(this.lastAddresses); + } + public void addLastAddresses(long lastSeenMillis, String address, + int orPort, int dirPort, SortedSet<String> orAddressesAndPorts) { + Set<String> addressesAndPorts = new HashSet<String>(); + addressesAndPorts.add(address + ":" + orPort); + if (dirPort > 0) { + addressesAndPorts.add(address + ":" + dirPort); + } + addressesAndPorts.addAll(orAddressesAndPorts); + if (this.lastAddresses.containsKey(lastSeenMillis)) { + this.lastAddresses.get(lastSeenMillis).addAll(addressesAndPorts); + } else { + this.lastAddresses.put(lastSeenMillis, addressesAndPorts); + } } - public long getLastChangedOrAddress() { + public long getLastChangedOrAddressOrPort() { long lastChangedAddressesMillis = -1L; if (this.lastAddresses != null) { Set<String> lastAddresses = null; - for (Map.Entry<Long, Set<String>> e : this.lastAddresses.entrySet()) { + for (Map.Entry<Long, Set<String>> e : + this.lastAddresses.entrySet()) { if (lastAddresses != null) { for (String address : e.getValue()) { if (!lastAddresses.contains(address)) { @@ -283,136 +255,73 @@ public class NodeStatus extends Document { return lastChangedAddressesMillis; } - private String contact; - public void setContact(String contact) { - if (contact == null) { - this.contact = null; - } else { - StringBuilder sb = new StringBuilder(); - for (char c : contact.toLowerCase().toCharArray()) { - if (c >= 32 && c < 127) { - sb.append(c); - } else { - sb.append(" "); - } - } - this.contact = sb.toString(); - } - } - public String getContact() { - return this.contact; - } - private Boolean recommendedVersion; + public void setRecommendedVersion(Boolean recommendedVersion) { + this.recommendedVersion = recommendedVersion; + } public Boolean getRecommendedVersion() { return this.recommendedVersion; } - private String[] familyFingerprints; - public void setFamilyFingerprints( - SortedSet<String> familyFingerprints) { - this.familyFingerprints = collectionToStringArray(familyFingerprints); + /* From exit lists: */ + + private SortedSet<String> exitAddresses; + public void setExitAddresses(SortedSet<String> exitAddresses) { + this.exitAddresses = exitAddresses; } - public SortedSet<String> getFamilyFingerprints() { - return stringArrayToSortedSet(this.familyFingerprints); + public SortedSet<String> getExitAddresses() { + return new TreeSet<String>(this.exitAddresses); } - private static String[] collectionToStringArray( - Collection<String> collection) { - String[] stringArray = null; - if (collection != null && !collection.isEmpty()) { - stringArray = new String[collection.size()]; - int i = 0; - for (String string : collection) { - stringArray[i++] = string; - } - } - return stringArray; + /* GeoIP lookup results: */ + + private String countryCode; + public void setCountryCode(String countryCode) { + this.countryCode = countryCode; } - private SortedSet<String> stringArrayToSortedSet(String[] stringArray) { - SortedSet<String> sortedSet = new TreeSet<String>(); - if (stringArray != null) { - sortedSet.addAll(Arrays.asList(stringArray)); - } - return sortedSet; + public String getCountryCode() { + return this.countryCode; } - public NodeStatus(boolean isRelay, String nickname, String fingerprint, - String address, SortedSet<String> orAddressesAndPorts, - SortedSet<String> exitAddresses, long lastSeenMillis, int orPort, - int dirPort, SortedSet<String> relayFlags, long consensusWeight, - String countryCode, String hostName, long lastRdnsLookup, - String defaultPolicy, String portList, long firstSeenMillis, - long lastChangedAddresses, String aSNumber, String contact, - Boolean recommendedVersion, SortedSet<String> familyFingerprints) { - this.isRelay = isRelay; - this.nickname = nickname; - this.fingerprint = fingerprint; - this.address = address; - this.exitAddresses = new TreeSet<String>(); - if (exitAddresses != null) { - this.exitAddresses.addAll(exitAddresses); - } - this.exitAddresses.remove(this.address); - this.orAddresses = new TreeSet<String>(); - this.orAddressesAndPorts = new TreeSet<String>(); - if (orAddressesAndPorts != null) { - for (String orAddressAndPort : orAddressesAndPorts) { - this.addOrAddressAndPort(orAddressAndPort); - } - } - this.lastSeenMillis = lastSeenMillis; - this.orPort = orPort; - this.dirPort = dirPort; - this.setRelayFlags(relayFlags); - this.consensusWeight = consensusWeight; - this.countryCode = countryCode; - this.hostName = hostName; - this.lastRdnsLookup = lastRdnsLookup; - this.defaultPolicy = defaultPolicy; - this.portList = portList; - this.firstSeenMillis = firstSeenMillis; - this.lastAddresses = - new TreeMap<Long, Set<String>>(Collections.reverseOrder()); - Set<String> addresses = new HashSet<String>(); - addresses.add(address + ":" + orPort); - if (dirPort > 0) { - addresses.add(address + ":" + dirPort); - } - addresses.addAll(orAddressesAndPorts); - this.lastAddresses.put(lastChangedAddresses, addresses); + private String aSNumber; + public void setASNumber(String aSNumber) { this.aSNumber = aSNumber; - this.contact = contact; - this.recommendedVersion = recommendedVersion; - this.setFamilyFingerprints(familyFingerprints); + } + public String getASNumber() { + return this.aSNumber; + } + + /* Reverse DNS lookup result */ + + private long lastRdnsLookup = -1L; + public void setLastRdnsLookup(long lastRdnsLookup) { + this.lastRdnsLookup = lastRdnsLookup; + } + public long getLastRdnsLookup() { + return this.lastRdnsLookup; + } + + /* Constructor and (de-)serialization methods: */ + + public NodeStatus(String fingerprint) { + this.fingerprint = fingerprint; } public static NodeStatus fromString(String documentString) { - boolean isRelay = false; - String nickname = null, fingerprint = null, address = null, - countryCode = null, hostName = null, defaultPolicy = null, - portList = null, aSNumber = null, contact = null; - SortedSet<String> orAddressesAndPorts = null, exitAddresses = null, - relayFlags = null, familyFingerprints = null; - long lastSeenMillis = -1L, consensusWeight = -1L, - lastRdnsLookup = -1L, firstSeenMillis = -1L, - lastChangedAddresses = -1L; - int orPort = -1, dirPort = -1; - Boolean recommendedVersion = null; try { - String separator = documentString.contains("\t") ? "\t" : " "; - String[] parts = documentString.trim().split(separator); - isRelay = parts[0].equals("r"); - if (parts.length < 9) { + String[] parts = documentString.trim().split("\t"); + if (parts.length < 23) { log.error("Too few space-separated values in line '" + documentString.trim() + "'. Skipping."); return null; } - nickname = parts[1]; - fingerprint = parts[2]; - orAddressesAndPorts = new TreeSet<String>(); - exitAddresses = new TreeSet<String>(); - String addresses = parts[3]; + String fingerprint = parts[2]; + NodeStatus nodeStatus = new NodeStatus(fingerprint); + nodeStatus.setRelay(parts[0].equals("r")); + nodeStatus.setNickname(parts[1]); + SortedSet<String> orAddressesAndPorts = new TreeSet<String>(); + SortedSet<String> exitAddresses = new TreeSet<String>(); + String addresses = parts[3], address = null; if (addresses.contains(";")) { String[] addressParts = addresses.split(";", -1); if (addressParts.length != 3) { @@ -432,48 +341,51 @@ public class NodeStatus extends Document { } else { address = addresses; } - lastSeenMillis = DateTimeHelper.parse(parts[4] + " " + parts[5]); + nodeStatus.setAddress(address); + nodeStatus.setOrAddressesAndPorts(orAddressesAndPorts); + nodeStatus.setExitAddresses(exitAddresses); + long lastSeenMillis = DateTimeHelper.parse(parts[4] + " " + + parts[5]); if (lastSeenMillis < 0L) { log.error("Parse exception while parsing node status " + "line '" + documentString + "'. Skipping."); return null; } - orPort = Integer.parseInt(parts[6]); - dirPort = Integer.parseInt(parts[7]); - relayFlags = new TreeSet<String>(); - if (parts[8].length() > 0) { - relayFlags.addAll(Arrays.asList(parts[8].split(","))); - } - if (parts.length > 9) { - consensusWeight = Long.parseLong(parts[9]); + nodeStatus.setLastSeenMillis(lastSeenMillis); + int orPort = Integer.parseInt(parts[6]), + dirPort = Integer.parseInt(parts[7]); + nodeStatus.setOrPort(orPort); + nodeStatus.setDirPort(dirPort); + nodeStatus.setRelayFlags(new TreeSet<String>( + Arrays.asList(parts[8].split(",")))); + nodeStatus.setConsensusWeight(Long.parseLong(parts[9])); + nodeStatus.setCountryCode(parts[10]); + if (!parts[11].equals("")) { + /* This is a (possibly surprising) hack that is part of moving the + * host name field from node status to details status. As part of + * that move we ignore all previously looked up host names trigger + * a new lookup by setting the last lookup time to 1969-12-31 + * 23:59:59.999. This hack may be removed after it has been run + * at least once. */ + parts[12] = "-1"; } - if (parts.length > 10) { - countryCode = parts[10]; + nodeStatus.setLastRdnsLookup(Long.parseLong(parts[12])); + if (!parts[13].equals("null")) { + nodeStatus.setDefaultPolicy(parts[13]); } - if (parts.length > 12) { - hostName = parts[11].equals("null") ? null : parts[11]; - lastRdnsLookup = Long.parseLong(parts[12]); + if (!parts[14].equals("null")) { + nodeStatus.setPortList(parts[14]); } - if (parts.length > 14) { - if (!parts[13].equals("null")) { - defaultPolicy = parts[13]; - } - if (!parts[14].equals("null")) { - portList = parts[14]; - } - } - firstSeenMillis = lastSeenMillis; - if (parts.length > 16) { - firstSeenMillis = DateTimeHelper.parse(parts[15] + " " - + parts[16]); - if (firstSeenMillis < 0L) { - log.error("Parse exception while parsing node status " - + "line '" + documentString + "'. Skipping."); - return null; - } + long firstSeenMillis = lastSeenMillis; + firstSeenMillis = DateTimeHelper.parse(parts[15] + " " + parts[16]); + if (firstSeenMillis < 0L) { + log.error("Parse exception while parsing node status " + + "line '" + documentString + "'. Skipping."); + return null; } - lastChangedAddresses = lastSeenMillis; - if (parts.length > 18 && !parts[17].equals("null")) { + nodeStatus.setFirstSeenMillis(firstSeenMillis); + long lastChangedAddresses = lastSeenMillis; + if (!parts[17].equals("null")) { lastChangedAddresses = DateTimeHelper.parse(parts[17] + " " + parts[18]); if (lastChangedAddresses < 0L) { @@ -482,20 +394,18 @@ public class NodeStatus extends Document { return null; } } - if (parts.length > 19) { - aSNumber = parts[19]; - } - if (parts.length > 20) { - contact = parts[20]; - } - if (parts.length > 21) { - recommendedVersion = parts[21].equals("null") ? null : - parts[21].equals("true"); + nodeStatus.addLastAddresses(lastChangedAddresses, address, orPort, + dirPort, orAddressesAndPorts); + nodeStatus.setASNumber(parts[19]); + nodeStatus.setContact(parts[20]); + if (!parts[21].equals("null")) { + nodeStatus.setRecommendedVersion(parts[21].equals("true")); } - if (parts.length > 22 && !parts[22].equals("null")) { - familyFingerprints = new TreeSet<String>(Arrays.asList( - parts[22].split(";"))); + if (!parts[22].equals("null")) { + nodeStatus.setFamilyFingerprints(new TreeSet<String>( + Arrays.asList(parts[22].split(";")))); } + return nodeStatus; } catch (NumberFormatException e) { log.error("Number format exception while parsing node " + "status line '" + documentString + "': " + e.getMessage() @@ -509,36 +419,6 @@ public class NodeStatus extends Document { + "Skipping."); return null; } - NodeStatus newNodeStatus = new NodeStatus(isRelay, nickname, - fingerprint, address, orAddressesAndPorts, exitAddresses, - lastSeenMillis, orPort, dirPort, relayFlags, consensusWeight, - countryCode, hostName, lastRdnsLookup, defaultPolicy, portList, - firstSeenMillis, lastChangedAddresses, aSNumber, contact, - recommendedVersion, familyFingerprints); - return newNodeStatus; - } - - public void update(NodeStatus newNodeStatus) { - if (newNodeStatus.lastSeenMillis > this.lastSeenMillis) { - this.nickname = newNodeStatus.nickname; - this.address = newNodeStatus.address; - this.orAddressesAndPorts = newNodeStatus.orAddressesAndPorts; - this.lastSeenMillis = newNodeStatus.lastSeenMillis; - this.orPort = newNodeStatus.orPort; - this.dirPort = newNodeStatus.dirPort; - this.relayFlags = newNodeStatus.relayFlags; - this.consensusWeight = newNodeStatus.consensusWeight; - this.countryCode = newNodeStatus.countryCode; - this.defaultPolicy = newNodeStatus.defaultPolicy; - this.portList = newNodeStatus.portList; - this.aSNumber = newNodeStatus.aSNumber; - this.recommendedVersion = newNodeStatus.recommendedVersion; - } - if (this.isRelay && newNodeStatus.isRelay) { - this.lastAddresses.putAll(newNodeStatus.lastAddresses); - } - this.firstSeenMillis = Math.min(newNodeStatus.firstSeenMillis, - this.firstSeenMillis); } public String toString() { @@ -548,15 +428,16 @@ public class NodeStatus extends Document { sb.append("\t" + this.fingerprint); sb.append("\t" + this.address + ";"); int written = 0; - for (String orAddressAndPort : this.orAddressesAndPorts) { - sb.append((written++ > 0 ? "+" : "") + orAddressAndPort); + if (this.orAddressesAndPorts != null) { + for (String orAddressAndPort : this.orAddressesAndPorts) { + sb.append((written++ > 0 ? "+" : "") + orAddressAndPort); + } } sb.append(";"); if (this.isRelay) { written = 0; for (String exitAddress : this.exitAddresses) { - sb.append((written++ > 0 ? "+" : "") - + exitAddress); + sb.append((written++ > 0 ? "+" : "") + exitAddress); } } sb.append("\t" + DateTimeHelper.format(this.lastSeenMillis, @@ -571,19 +452,19 @@ public class NodeStatus extends Document { sb.append("\t" + String.valueOf(this.consensusWeight)); sb.append("\t" + (this.countryCode != null ? this.countryCode : "??")); - sb.append("\t" + (this.hostName != null ? this.hostName : "null")); + sb.append("\t"); /* formerly used for storing host names */ sb.append("\t" + String.valueOf(this.lastRdnsLookup)); sb.append("\t" + (this.defaultPolicy != null ? this.defaultPolicy : "null")); sb.append("\t" + (this.portList != null ? this.portList : "null")); } else { - sb.append("\t-1\t??\tnull\t-1\tnull\tnull"); + sb.append("\t-1\t??\t\t-1\tnull\tnull"); } sb.append("\t" + DateTimeHelper.format(this.firstSeenMillis, DateTimeHelper.ISO_DATETIME_TAB_FORMAT)); if (this.isRelay) { sb.append("\t" + DateTimeHelper.format( - this.getLastChangedOrAddress(), + this.getLastChangedOrAddressOrPort(), DateTimeHelper.ISO_DATETIME_TAB_FORMAT)); sb.append("\t" + (this.aSNumber != null ? this.aSNumber : "null")); } else { diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java index 9963a74..0ce17e4 100644 --- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java @@ -31,6 +31,40 @@ import org.torproject.onionoo.docs.NodeStatus; import org.torproject.onionoo.util.FormattingUtils; import org.torproject.onionoo.util.TimeFactory; +/* + * Status updater for both node and details statuses. + * + * Node statuses hold selected information about relays and bridges that + * must be retrieved efficiently in the update process. They are kept in + * memory and not written to disk until the update process has completed. + * They are also the sole basis for writing summary documents that are + * used in the server package to build the node index. + * + * Details statuses hold detailed information about relays and bridges and + * are written at the end of the update process. They are not kept in + * memory. They are the sole basis for writing details documents. + * + * The complete update process as implemented in this class is as follows: + * + * 1. Parse descriptors and either write their contents to details + * statuses (which requires retrieving them and storing them back to + * disk, which is why this is only done for detailed information that + * don't fit into memory) and/or local data structures in memory + * (which may include node statuses). + * 2. Read all known node statuses from disk and merge their contents + * with the node statuses from parsing descriptors. Node statuses + * are not loaded from disk before the parse step in order to save + * memory for parsed descriptors. + * 3. Perform reverse DNS lookups, Look up relay IP addresses in a + * GeoIP database, and calculate path selection probabilities. + * Update node statuses accordingly. + * 4. Retrieve details statuses corresponding to nodes that have been + * changed since the start of the update process, possibly update the + * node statuses with contents from newly parsed descriptors, update + * details statuses with results from lookup operations and new path + * selection probabilities, and store details statuses and node + * statuses back to disk. + */ public class NodeDetailsStatusUpdater implements DescriptorListener, StatusUpdater { @@ -50,10 +84,6 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, private SortedMap<String, NodeStatus> knownNodes = new TreeMap<String, NodeStatus>(); - private SortedMap<String, NodeStatus> relays; - - private SortedMap<String, NodeStatus> bridges; - private long relaysLastValidAfterMillis = -1L; private long bridgesLastPublishedMillis = -1L; @@ -88,6 +118,10 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, DescriptorType.EXIT_LISTS); } + /* Step 1: parse descriptors. */ + + private SortedSet<String> updatedNodes = new TreeSet<String>(); + public void processDescriptor(Descriptor descriptor, boolean relay) { if (descriptor instanceof ServerDescriptor && relay) { this.processRelayServerDescriptor((ServerDescriptor) descriptor); @@ -112,8 +146,9 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, DetailsStatus.class, true, fingerprint); if (detailsStatus == null) { detailsStatus = new DetailsStatus(); - } else if (descriptor.getPublishedMillis() <= - detailsStatus.getDescPublished()) { + } else if (detailsStatus.getDescPublished() != null && + detailsStatus.getDescPublished() >= + descriptor.getPublishedMillis()) { return; } long lastRestartedMillis = descriptor.getPublishedMillis() @@ -145,9 +180,8 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, portsOrPortRanges); detailsStatus.setExitPolicyV6Summary(exitPolicyV6Summary); } - if (descriptor.isHibernating()) { - detailsStatus.setHibernating(true); - } + detailsStatus.setHibernating(descriptor.isHibernating() ? true : + null); this.documentStore.store(detailsStatus, fingerprint); } @@ -157,8 +191,8 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, private void processExitList(ExitList exitList) { for (ExitListEntry exitListEntry : exitList.getExitListEntries()) { String fingerprint = exitListEntry.getFingerprint(); - if (exitListEntry.getScanMillis() < - this.now - DateTimeHelper.ONE_DAY) { + long scanMillis = exitListEntry.getScanMillis(); + if (scanMillis < this.now - DateTimeHelper.ONE_DAY) { continue; } if (!this.exitListEntries.containsKey(fingerprint)) { @@ -166,7 +200,6 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, new HashMap<String, Long>()); } String exitAddress = exitListEntry.getExitAddress(); - long scanMillis = exitListEntry.getScanMillis(); if (!this.exitListEntries.get(fingerprint).containsKey(exitAddress) || this.exitListEntries.get(fingerprint).get(exitAddress) < scanMillis) { @@ -190,31 +223,40 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, recommendedVersions.add("Tor " + recommendedVersion); } } - for (NetworkStatusEntry entry : - consensus.getStatusEntries().values()) { - String nickname = entry.getNickname(); - String fingerprint = entry.getFingerprint(); + for (Map.Entry<String, NetworkStatusEntry> e : + consensus.getStatusEntries().entrySet()) { + String fingerprint = e.getKey(); + NetworkStatusEntry entry = e.getValue(); + NodeStatus nodeStatus = this.knownNodes.get(fingerprint); + if (nodeStatus == null) { + nodeStatus = new NodeStatus(fingerprint); + this.knownNodes.put(fingerprint, nodeStatus); + } String address = entry.getAddress(); + int orPort = entry.getOrPort(), dirPort = entry.getDirPort(); SortedSet<String> orAddressesAndPorts = new TreeSet<String>( entry.getOrAddresses()); - int orPort = entry.getOrPort(); - int dirPort = entry.getDirPort(); - SortedSet<String> relayFlags = entry.getFlags(); - long consensusWeight = entry.getBandwidth(); - String defaultPolicy = entry.getDefaultPolicy(); - String portList = entry.getPortList(); - Boolean recommendedVersion = (recommendedVersions == null || - entry.getVersion() == null) ? null : - recommendedVersions.contains(entry.getVersion()); - NodeStatus newNodeStatus = new NodeStatus(true, nickname, - fingerprint, address, orAddressesAndPorts, null, - validAfterMillis, orPort, dirPort, relayFlags, consensusWeight, - null, null, -1L, defaultPolicy, portList, validAfterMillis, - validAfterMillis, null, null, recommendedVersion, null); - if (this.knownNodes.containsKey(fingerprint)) { - this.knownNodes.get(fingerprint).update(newNodeStatus); - } else { - this.knownNodes.put(fingerprint, newNodeStatus); + nodeStatus.addLastAddresses(validAfterMillis, address, orPort, + dirPort, orAddressesAndPorts); + if (nodeStatus.getFirstSeenMillis() == 0L || + validAfterMillis < nodeStatus.getFirstSeenMillis()) { + nodeStatus.setFirstSeenMillis(validAfterMillis); + } + if (validAfterMillis > nodeStatus.getLastSeenMillis()) { + nodeStatus.setLastSeenMillis(validAfterMillis); + nodeStatus.setRelay(true); + nodeStatus.setNickname(entry.getNickname()); + nodeStatus.setAddress(address); + nodeStatus.setOrAddressesAndPorts(orAddressesAndPorts); + nodeStatus.setOrPort(orPort); + nodeStatus.setDirPort(dirPort); + nodeStatus.setRelayFlags(entry.getFlags()); + nodeStatus.setConsensusWeight(entry.getBandwidth()); + nodeStatus.setDefaultPolicy(entry.getDefaultPolicy()); + nodeStatus.setPortList(entry.getPortList()); + nodeStatus.setRecommendedVersion((recommendedVersions == null || + entry.getVersion() == null) ? null : + recommendedVersions.contains(entry.getVersion())); } } this.relayConsensusesProcessed++; @@ -244,24 +286,15 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, detailsStatus.setAdvertisedBandwidth(advertisedBandwidth); detailsStatus.setPlatform(descriptor.getPlatform()); this.documentStore.store(detailsStatus, fingerprint); + this.updatedNodes.add(fingerprint); } + private Map<String, String> bridgePoolAssignments = + new HashMap<String, String>(); + private void processBridgePoolAssignment( BridgePoolAssignment bridgePoolAssignment) { - for (Map.Entry<String, String> e : - bridgePoolAssignment.getEntries().entrySet()) { - String fingerprint = e.getKey(); - String details = e.getValue(); - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, true, fingerprint); - if (detailsStatus == null) { - detailsStatus = new DetailsStatus(); - } else if (details.equals(detailsStatus.getPoolAssignment())) { - continue; - } - detailsStatus.setPoolAssignment(details); - this.documentStore.store(detailsStatus, fingerprint); - } + this.bridgePoolAssignments.putAll(bridgePoolAssignment.getEntries()); } private void processBridgeNetworkStatus(BridgeNetworkStatus status) { @@ -269,108 +302,185 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, if (publishedMillis > this.bridgesLastPublishedMillis) { this.bridgesLastPublishedMillis = publishedMillis; } - for (NetworkStatusEntry entry : status.getStatusEntries().values()) { - String nickname = entry.getNickname(); - String fingerprint = entry.getFingerprint(); - String address = entry.getAddress(); - SortedSet<String> orAddressesAndPorts = new TreeSet<String>( - entry.getOrAddresses()); - int orPort = entry.getOrPort(); - int dirPort = entry.getDirPort(); - SortedSet<String> relayFlags = entry.getFlags(); - NodeStatus newNodeStatus = new NodeStatus(false, nickname, - fingerprint, address, orAddressesAndPorts, null, - publishedMillis, orPort, dirPort, relayFlags, -1L, "??", null, - -1L, null, null, publishedMillis, -1L, null, null, null, null); - if (this.knownNodes.containsKey(fingerprint)) { - this.knownNodes.get(fingerprint).update(newNodeStatus); - } else { - this.knownNodes.put(fingerprint, newNodeStatus); + for (Map.Entry<String, NetworkStatusEntry> e : + status.getStatusEntries().entrySet()) { + String fingerprint = e.getKey(); + NetworkStatusEntry entry = e.getValue(); + NodeStatus nodeStatus = this.knownNodes.get(fingerprint); + if (nodeStatus == null) { + nodeStatus = new NodeStatus(fingerprint); + this.knownNodes.put(fingerprint, nodeStatus); + } + if (nodeStatus.getFirstSeenMillis() == 0L || + publishedMillis < nodeStatus.getFirstSeenMillis()) { + nodeStatus.setFirstSeenMillis(publishedMillis); + } + if (publishedMillis > nodeStatus.getLastSeenMillis()) { + nodeStatus.setRelay(false); + nodeStatus.setNickname(entry.getNickname()); + nodeStatus.setAddress(entry.getAddress()); + nodeStatus.setOrAddressesAndPorts(new TreeSet<String>( + entry.getOrAddresses())); + nodeStatus.setOrPort(entry.getOrPort()); + nodeStatus.setDirPort(entry.getDirPort()); + nodeStatus.setRelayFlags(entry.getFlags()); + nodeStatus.setLastSeenMillis(publishedMillis); } } this.bridgeStatusesProcessed++; } public void updateStatuses() { - this.readStatusSummary(); - log.info("Read status summary"); - this.setCurrentNodes(); - log.info("Set current node fingerprints"); + this.readNodeStatuses(); + log.info("Read node statuses"); this.startReverseDomainNameLookups(); log.info("Started reverse domain name lookups"); this.lookUpCitiesAndASes(); log.info("Looked up cities and ASes"); - this.setDescriptorPartsOfNodeStatus(); - log.info("Set descriptor parts of node statuses."); this.calculatePathSelectionProbabilities(); log.info("Calculated path selection probabilities"); this.finishReverseDomainNameLookups(); log.info("Finished reverse domain name lookups"); - this.writeStatusSummary(); - log.info("Wrote status summary"); - this.updateDetailsStatuses(); - log.info("Updated exit addresses in details statuses"); + this.updateNodeDetailsStatuses(); + log.info("Updated node and details statuses"); } - private void readStatusSummary() { - SortedSet<String> fingerprints = this.documentStore.list( + /* Step 2: read node statuses from disk. */ + + private SortedSet<String> currentRelays = new TreeSet<String>(), + runningRelays = new TreeSet<String>(); + + private void readNodeStatuses() { + SortedSet<String> previouslyKnownNodes = this.documentStore.list( NodeStatus.class); - for (String fingerprint : fingerprints) { - NodeStatus node = this.documentStore.retrieve(NodeStatus.class, - true, fingerprint); - if (node.isRelay()) { - this.relaysLastValidAfterMillis = Math.max( - this.relaysLastValidAfterMillis, node.getLastSeenMillis()); - } else { - this.bridgesLastPublishedMillis = Math.max( - this.bridgesLastPublishedMillis, node.getLastSeenMillis()); - } - if (this.knownNodes.containsKey(fingerprint)) { - this.knownNodes.get(fingerprint).update(node); - } else { - this.knownNodes.put(fingerprint, node); + long previousRelaysLastValidAfterMillis = -1L, + previousBridgesLastValidAfterMillis = -1L; + for (String fingerprint : previouslyKnownNodes) { + NodeStatus nodeStatus = this.documentStore.retrieve( + NodeStatus.class, true, fingerprint); + if (nodeStatus.isRelay() && nodeStatus.getLastSeenMillis() > + previousRelaysLastValidAfterMillis) { + previousRelaysLastValidAfterMillis = + nodeStatus.getLastSeenMillis(); + } else if (!nodeStatus.isRelay() && nodeStatus.getLastSeenMillis() > + previousBridgesLastValidAfterMillis) { + previousBridgesLastValidAfterMillis = + nodeStatus.getLastSeenMillis(); } } - } - - private void setCurrentNodes() { + if (previousRelaysLastValidAfterMillis > + this.relaysLastValidAfterMillis) { + this.relaysLastValidAfterMillis = + previousRelaysLastValidAfterMillis; + } + if (previousBridgesLastValidAfterMillis > + this.bridgesLastPublishedMillis) { + this.bridgesLastPublishedMillis = + previousBridgesLastValidAfterMillis; + } long cutoff = Math.max(this.relaysLastValidAfterMillis, - this.bridgesLastPublishedMillis) - 7L * 24L * 60L * 60L * 1000L; - SortedMap<String, NodeStatus> currentNodes = - new TreeMap<String, NodeStatus>(); + this.bridgesLastPublishedMillis) - DateTimeHelper.ONE_WEEK; for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) { - if (e.getValue().getLastSeenMillis() >= cutoff) { - currentNodes.put(e.getKey(), e.getValue()); + String fingerprint = e.getKey(); + NodeStatus nodeStatus = e.getValue(); + this.updatedNodes.add(fingerprint); + if (nodeStatus.isRelay() && + nodeStatus.getLastSeenMillis() >= cutoff) { + this.currentRelays.add(fingerprint); + if (nodeStatus.getLastSeenMillis() == + this.relaysLastValidAfterMillis) { + this.runningRelays.add(fingerprint); + } } } - this.relays = new TreeMap<String, NodeStatus>(); - this.bridges = new TreeMap<String, NodeStatus>(); - for (Map.Entry<String, NodeStatus> e : currentNodes.entrySet()) { - if (e.getValue().isRelay()) { - this.relays.put(e.getKey(), e.getValue()); + for (String fingerprint : previouslyKnownNodes) { + NodeStatus nodeStatus = this.documentStore.retrieve( + NodeStatus.class, true, fingerprint); + NodeStatus updatedNodeStatus = null; + if (this.knownNodes.containsKey(fingerprint)) { + updatedNodeStatus = this.knownNodes.get(fingerprint); + String address = nodeStatus.getAddress(); + int orPort = nodeStatus.getOrPort(), + dirPort = nodeStatus.getDirPort(); + SortedSet<String> orAddressesAndPorts = + nodeStatus.getOrAddressesAndPorts(); + updatedNodeStatus.addLastAddresses( + nodeStatus.getLastChangedOrAddressOrPort(), address, orPort, + dirPort, orAddressesAndPorts); + if (nodeStatus.getLastSeenMillis() > + updatedNodeStatus.getLastSeenMillis()) { + updatedNodeStatus.setNickname(nodeStatus.getNickname()); + updatedNodeStatus.setAddress(address); + updatedNodeStatus.setOrAddressesAndPorts(orAddressesAndPorts); + updatedNodeStatus.setLastSeenMillis( + nodeStatus.getLastSeenMillis()); + updatedNodeStatus.setOrPort(orPort); + updatedNodeStatus.setDirPort(dirPort); + updatedNodeStatus.setRelayFlags(nodeStatus.getRelayFlags()); + updatedNodeStatus.setConsensusWeight( + nodeStatus.getConsensusWeight()); + updatedNodeStatus.setCountryCode(nodeStatus.getCountryCode()); + updatedNodeStatus.setDefaultPolicy( + nodeStatus.getDefaultPolicy()); + updatedNodeStatus.setPortList(nodeStatus.getPortList()); + updatedNodeStatus.setASNumber(nodeStatus.getASNumber()); + updatedNodeStatus.setRecommendedVersion( + nodeStatus.getRecommendedVersion()); + } + if (nodeStatus.getFirstSeenMillis() < + updatedNodeStatus.getFirstSeenMillis()) { + updatedNodeStatus.setFirstSeenMillis( + nodeStatus.getFirstSeenMillis()); + } + updatedNodeStatus.setLastRdnsLookup( + nodeStatus.getLastRdnsLookup()); } else { - this.bridges.put(e.getKey(), e.getValue()); + updatedNodeStatus = nodeStatus; + this.knownNodes.put(fingerprint, nodeStatus); + if (nodeStatus.getLastSeenMillis() == (nodeStatus.isRelay() ? + previousRelaysLastValidAfterMillis : + previousBridgesLastValidAfterMillis)) { + /* This relay or bridge was previously running, but we didn't + * parse any descriptors with its fingerprint. Make sure to + * update its details status file later on, so it has the + * correct running bit. */ + this.updatedNodes.add(fingerprint); + } + } + if (updatedNodeStatus.isRelay() && + updatedNodeStatus.getLastSeenMillis() >= cutoff) { + this.currentRelays.add(fingerprint); + if (updatedNodeStatus.getLastSeenMillis() == + this.relaysLastValidAfterMillis) { + this.runningRelays.add(fingerprint); + } } } } + /* Step 3: perform lookups and calculate path selection + * probabilities. */ + private void startReverseDomainNameLookups() { Map<String, Long> addressLastLookupTimes = new HashMap<String, Long>(); - for (NodeStatus relay : relays.values()) { - addressLastLookupTimes.put(relay.getAddress(), - relay.getLastRdnsLookup()); + for (String fingerprint : this.currentRelays) { + NodeStatus nodeStatus = this.knownNodes.get(fingerprint); + addressLastLookupTimes.put(nodeStatus.getAddress(), + nodeStatus.getLastRdnsLookup()); } this.reverseDomainNameResolver.setAddresses(addressLastLookupTimes); this.reverseDomainNameResolver.startReverseDomainNameLookups(); } + private SortedMap<String, LookupResult> geoIpLookupResults = + new TreeMap<String, LookupResult>(); + private void lookUpCitiesAndASes() { SortedSet<String> addressStrings = new TreeSet<String>(); - for (NodeStatus node : this.knownNodes.values()) { - if (node.isRelay()) { - addressStrings.add(node.getAddress()); - } + for (String fingerprint : this.currentRelays) { + NodeStatus nodeStatus = this.knownNodes.get(fingerprint); + addressStrings.add(nodeStatus.getAddress()); } if (addressStrings.isEmpty()) { log.error("No relay IP addresses to resolve to cities or " @@ -379,67 +489,22 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, } SortedMap<String, LookupResult> lookupResults = this.lookupService.lookup(addressStrings); - for (NodeStatus node : knownNodes.values()) { - if (!node.isRelay()) { - continue; - } - String addressString = node.getAddress(); - if (lookupResults.containsKey(addressString)) { - LookupResult lookupResult = lookupResults.get(addressString); - node.setCountryCode(lookupResult.getCountryCode()); - node.setCountryName(lookupResult.getCountryName()); - node.setRegionName(lookupResult.getRegionName()); - node.setCityName(lookupResult.getCityName()); - node.setLatitude(lookupResult.getLatitude()); - node.setLongitude(lookupResult.getLongitude()); - node.setASNumber(lookupResult.getAsNumber()); - node.setASName(lookupResult.getAsName()); + for (String fingerprint : this.currentRelays) { + NodeStatus nodeStatus = this.knownNodes.get(fingerprint); + LookupResult lookupResult = lookupResults.get( + nodeStatus.getAddress()); + if (lookupResult != null) { + this.geoIpLookupResults.put(fingerprint, lookupResult); + this.updatedNodes.add(fingerprint); } } } - private void setDescriptorPartsOfNodeStatus() { - for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) { - String fingerprint = e.getKey(); - NodeStatus node = e.getValue(); - if (node.isRelay()) { - if (node.getRelayFlags().contains("Running") && - node.getLastSeenMillis() == this.relaysLastValidAfterMillis) { - node.setRunning(true); - } - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, true, fingerprint); - if (detailsStatus != null) { - node.setContact(detailsStatus.getContact()); - if (detailsStatus.getExitAddresses() != null) { - for (Map.Entry<String, Long> ea : - detailsStatus.getExitAddresses().entrySet()) { - if (ea.getValue() >= this.now - DateTimeHelper.ONE_DAY) { - node.addExitAddress(ea.getKey()); - } - } - } - if (detailsStatus.getFamily() != null && - !detailsStatus.getFamily().isEmpty()) { - SortedSet<String> familyFingerprints = new TreeSet<String>(); - for (String familyMember : detailsStatus.getFamily()) { - if (familyMember.startsWith("$") && - familyMember.length() == 41) { - familyFingerprints.add(familyMember.substring(1)); - } - } - if (!familyFingerprints.isEmpty()) { - node.setFamilyFingerprints(familyFingerprints); - } - } - } - } - if (!node.isRelay() && node.getRelayFlags().contains("Running") && - node.getLastSeenMillis() == this.bridgesLastPublishedMillis) { - node.setRunning(true); - } - } - } + private SortedMap<String, Float> + consensusWeightFractions = new TreeMap<String, Float>(), + guardProbabilities = new TreeMap<String, Float>(), + middleProbabilities = new TreeMap<String, Float>(), + exitProbabilities = new TreeMap<String, Float>(); private void calculatePathSelectionProbabilities() { boolean consensusContainsBandwidthWeights = false; @@ -475,16 +540,12 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, double totalGuardWeight = 0.0; double totalMiddleWeight = 0.0; double totalExitWeight = 0.0; - for (Map.Entry<String, NodeStatus> e : this.relays.entrySet()) { - String fingerprint = e.getKey(); - NodeStatus relay = e.getValue(); - if (!relay.getRunning()) { - continue; - } - boolean isExit = relay.getRelayFlags().contains("Exit") && - !relay.getRelayFlags().contains("BadExit"); - boolean isGuard = relay.getRelayFlags().contains("Guard"); - double consensusWeight = (double) relay.getConsensusWeight(); + for (String fingerprint : this.runningRelays) { + NodeStatus nodeStatus = this.knownNodes.get(fingerprint); + boolean isExit = nodeStatus.getRelayFlags().contains("Exit") && + !nodeStatus.getRelayFlags().contains("BadExit"); + boolean isGuard = nodeStatus.getRelayFlags().contains("Guard"); + double consensusWeight = (double) nodeStatus.getConsensusWeight(); consensusWeights.put(fingerprint, consensusWeight); totalConsensusWeight += consensusWeight; if (consensusContainsBandwidthWeights) { @@ -516,57 +577,67 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, totalExitWeight += exitWeight; } } - for (Map.Entry<String, NodeStatus> e : this.relays.entrySet()) { - String fingerprint = e.getKey(); - NodeStatus relay = e.getValue(); + for (String fingerprint : this.runningRelays) { if (consensusWeights.containsKey(fingerprint)) { - relay.setConsensusWeightFraction(consensusWeights.get(fingerprint) - / totalConsensusWeight); + this.consensusWeightFractions.put(fingerprint, (float) + (consensusWeights.get(fingerprint) / totalConsensusWeight)); + this.updatedNodes.add(fingerprint); } if (guardWeights.containsKey(fingerprint)) { - relay.setGuardProbability(guardWeights.get(fingerprint) - / totalGuardWeight); + this.guardProbabilities.put(fingerprint, (float) + (guardWeights.get(fingerprint) / totalGuardWeight)); + this.updatedNodes.add(fingerprint); } if (middleWeights.containsKey(fingerprint)) { - relay.setMiddleProbability(middleWeights.get(fingerprint) - / totalMiddleWeight); + this.middleProbabilities.put(fingerprint, (float) + (middleWeights.get(fingerprint) / totalMiddleWeight)); + this.updatedNodes.add(fingerprint); } if (exitWeights.containsKey(fingerprint)) { - relay.setExitProbability(exitWeights.get(fingerprint) - / totalExitWeight); + this.exitProbabilities.put(fingerprint, (float) + (exitWeights.get(fingerprint) / totalExitWeight)); + this.updatedNodes.add(fingerprint); } } } + private long startedRdnsLookups = -1L; + + private SortedMap<String, String> rdnsLookupResults = + new TreeMap<String, String>(); + private void finishReverseDomainNameLookups() { this.reverseDomainNameResolver.finishReverseDomainNameLookups(); + this.startedRdnsLookups = + this.reverseDomainNameResolver.getLookupStartMillis(); Map<String, String> lookupResults = this.reverseDomainNameResolver.getLookupResults(); - long startedRdnsLookups = - this.reverseDomainNameResolver.getLookupStartMillis(); - for (NodeStatus relay : relays.values()) { - if (lookupResults.containsKey(relay.getAddress())) { - relay.setHostName(lookupResults.get(relay.getAddress())); - relay.setLastRdnsLookup(startedRdnsLookups); + for (String fingerprint : this.currentRelays) { + NodeStatus nodeStatus = this.knownNodes.get(fingerprint); + String hostName = lookupResults.get(nodeStatus.getAddress()); + if (hostName != null) { + this.rdnsLookupResults.put(fingerprint, hostName); + this.updatedNodes.add(fingerprint); } } } - private void writeStatusSummary() { - for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) { - this.documentStore.store(e.getValue(), e.getKey()); - } - } + /* Step 4: update details statuses and then node statuses. */ - private void updateDetailsStatuses() { - SortedSet<String> fingerprints = new TreeSet<String>(); - fingerprints.addAll(this.exitListEntries.keySet()); - for (String fingerprint : fingerprints) { + private void updateNodeDetailsStatuses() { + for (String fingerprint : this.updatedNodes) { + NodeStatus nodeStatus = this.knownNodes.get(fingerprint); + if (nodeStatus == null) { + nodeStatus = new NodeStatus(fingerprint); + } DetailsStatus detailsStatus = this.documentStore.retrieve( DetailsStatus.class, true, fingerprint); if (detailsStatus == null) { detailsStatus = new DetailsStatus(); } + + nodeStatus.setContact(detailsStatus.getContact()); + Map<String, Long> exitAddresses = new HashMap<String, Long>(); if (detailsStatus.getExitAddresses() != null) { for (Map.Entry<String, Long> e : @@ -579,20 +650,94 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, if (this.exitListEntries.containsKey(fingerprint)) { for (Map.Entry<String, Long> e : this.exitListEntries.get(fingerprint).entrySet()) { - if (!exitAddresses.containsKey(e.getKey()) || - exitAddresses.get(e.getKey()) < e.getValue()) { - exitAddresses.put(e.getKey(), e.getValue()); + String exitAddress = e.getKey(); + long scanMillis = e.getValue(); + if (!exitAddresses.containsKey(exitAddress) || + exitAddresses.get(exitAddress) < scanMillis) { + exitAddresses.put(exitAddress, scanMillis); } } } - if (this.knownNodes.containsKey(fingerprint)) { - for (String orAddress : - this.knownNodes.get(fingerprint).getOrAddresses()) { - this.exitListEntries.remove(orAddress); + detailsStatus.setExitAddresses(exitAddresses); + SortedSet<String> exitAddressesWithoutOrAddresses = + new TreeSet<String>(exitAddresses.keySet()); + exitAddressesWithoutOrAddresses.removeAll( + nodeStatus.getOrAddresses()); + nodeStatus.setExitAddresses(exitAddressesWithoutOrAddresses); + + if (detailsStatus.getFamily() != null && + !detailsStatus.getFamily().isEmpty()) { + SortedSet<String> familyFingerprints = new TreeSet<String>(); + for (String familyMember : detailsStatus.getFamily()) { + if (familyMember.startsWith("$") && + familyMember.length() == 41) { + familyFingerprints.add(familyMember.substring(1)); + } + } + if (!familyFingerprints.isEmpty()) { + nodeStatus.setFamilyFingerprints(familyFingerprints); } } - detailsStatus.setExitAddresses(exitAddresses); + + if (this.geoIpLookupResults.containsKey(fingerprint)) { + LookupResult lookupResult = this.geoIpLookupResults.get( + fingerprint); + detailsStatus.setCountryCode(lookupResult.getCountryCode()); + detailsStatus.setCountryName(lookupResult.getCountryName()); + detailsStatus.setRegionName(lookupResult.getRegionName()); + detailsStatus.setCityName(lookupResult.getCityName()); + detailsStatus.setLatitude(lookupResult.getLatitude()); + detailsStatus.setLongitude(lookupResult.getLongitude()); + detailsStatus.setASNumber(lookupResult.getAsNumber()); + detailsStatus.setASName(lookupResult.getAsName()); + nodeStatus.setCountryCode(lookupResult.getCountryCode()); + nodeStatus.setASNumber(lookupResult.getAsNumber()); + } + + detailsStatus.setConsensusWeightFraction( + this.consensusWeightFractions.get(fingerprint)); + detailsStatus.setGuardProbability( + this.guardProbabilities.get(fingerprint)); + detailsStatus.setMiddleProbability( + this.middleProbabilities.get(fingerprint)); + detailsStatus.setExitProbability( + this.exitProbabilities.get(fingerprint)); + + if (this.rdnsLookupResults.containsKey(fingerprint)) { + String hostName = this.rdnsLookupResults.get(fingerprint); + detailsStatus.setHostName(hostName); + nodeStatus.setLastRdnsLookup(this.startedRdnsLookups); + } + + if (this.bridgePoolAssignments.containsKey(fingerprint)) { + detailsStatus.setPoolAssignment( + this.bridgePoolAssignments.get(fingerprint)); + } + + detailsStatus.setRelay(nodeStatus.isRelay()); + detailsStatus.setRunning(nodeStatus.getLastSeenMillis() == + (nodeStatus.isRelay() + ? this.relaysLastValidAfterMillis + : this.bridgesLastPublishedMillis)); + detailsStatus.setNickname(nodeStatus.getNickname()); + detailsStatus.setAddress(nodeStatus.getAddress()); + detailsStatus.setOrAddressesAndPorts( + nodeStatus.getOrAddressesAndPorts()); + detailsStatus.setFirstSeenMillis(nodeStatus.getFirstSeenMillis()); + detailsStatus.setLastSeenMillis(nodeStatus.getLastSeenMillis()); + detailsStatus.setOrPort(nodeStatus.getOrPort()); + detailsStatus.setDirPort(nodeStatus.getDirPort()); + detailsStatus.setRelayFlags(nodeStatus.getRelayFlags()); + detailsStatus.setConsensusWeight(nodeStatus.getConsensusWeight()); + detailsStatus.setDefaultPolicy(nodeStatus.getDefaultPolicy()); + detailsStatus.setPortList(nodeStatus.getPortList()); + detailsStatus.setRecommendedVersion( + nodeStatus.getRecommendedVersion()); + detailsStatus.setLastChangedOrAddressOrPort( + nodeStatus.getLastChangedOrAddressOrPort()); + this.documentStore.store(detailsStatus, fingerprint); + this.documentStore.store(nodeStatus, fingerprint); } } diff --git a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java index 0692070..7f464b6 100644 --- a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java @@ -10,14 +10,11 @@ import java.util.TreeSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DetailsDocument; import org.torproject.onionoo.docs.DetailsStatus; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; -import org.torproject.onionoo.docs.NodeStatus; import org.torproject.onionoo.docs.UpdateStatus; -import org.torproject.onionoo.util.TimeFactory; public class DetailsDocumentWriter implements DocumentWriter { @@ -26,200 +23,135 @@ public class DetailsDocumentWriter implements DocumentWriter { private DocumentStore documentStore; - private long now; - public DetailsDocumentWriter() { this.documentStore = DocumentStoreFactory.getDocumentStore(); - this.now = TimeFactory.getTime().currentTimeMillis(); } - private SortedSet<String> newRelays = new TreeSet<String>(), - newBridges = new TreeSet<String>(); - public void writeDocuments() { UpdateStatus updateStatus = this.documentStore.retrieve( UpdateStatus.class, true); long updatedMillis = updateStatus != null ? updateStatus.getUpdatedMillis() : 0L; - SortedSet<String> updatedNodeStatuses = this.documentStore.list( - NodeStatus.class, updatedMillis); - for (String fingerprint : updatedNodeStatuses) { - NodeStatus nodeStatus = this.documentStore.retrieve( - NodeStatus.class, true, fingerprint); - if (nodeStatus.isRelay()) { - newRelays.add(fingerprint); - } else { - newBridges.add(fingerprint); - } - } SortedSet<String> updatedDetailsStatuses = this.documentStore.list( DetailsStatus.class, updatedMillis); for (String fingerprint : updatedDetailsStatuses) { - NodeStatus nodeStatus = this.documentStore.retrieve( - NodeStatus.class, true, fingerprint); - if (nodeStatus == null) { - continue; - } else if (nodeStatus.isRelay()) { - newRelays.add(fingerprint); + DetailsStatus detailsStatus = this.documentStore.retrieve( + DetailsStatus.class, true, fingerprint); + if (detailsStatus.isRelay()) { + this.updateRelayDetailsFile(fingerprint, detailsStatus); } else { - newBridges.add(fingerprint); + this.updateBridgeDetailsFile(fingerprint, detailsStatus); } } - this.updateRelayDetailsFiles(); - this.updateBridgeDetailsFiles(); log.info("Wrote details document files"); } - private void updateRelayDetailsFiles() { - for (String fingerprint : this.newRelays) { - - /* Generate network-status-specific part. */ - NodeStatus entry = this.documentStore.retrieve(NodeStatus.class, - true, fingerprint); - if (entry == null) { - continue; - } - DetailsDocument detailsDocument = new DetailsDocument(); - detailsDocument.setNickname(entry.getNickname()); - detailsDocument.setFingerprint(fingerprint); - List<String> orAddresses = new ArrayList<String>(); - orAddresses.add(entry.getAddress() + ":" + entry.getOrPort()); - for (String orAddress : entry.getOrAddressesAndPorts()) { - orAddresses.add(orAddress.toLowerCase()); - } - detailsDocument.setOrAddresses(orAddresses); - if (entry.getDirPort() != 0) { - detailsDocument.setDirAddress(entry.getAddress() + ":" - + entry.getDirPort()); - } - detailsDocument.setLastSeen(entry.getLastSeenMillis()); - detailsDocument.setFirstSeen(entry.getFirstSeenMillis()); - detailsDocument.setLastChangedAddressOrPort( - entry.getLastChangedOrAddress()); - detailsDocument.setRunning(entry.getRunning()); - if (!entry.getRelayFlags().isEmpty()) { - detailsDocument.setFlags(new ArrayList<String>( - entry.getRelayFlags())); - } - detailsDocument.setCountry(entry.getCountryCode()); - detailsDocument.setLatitude(entry.getLatitude()); - detailsDocument.setLongitude(entry.getLongitude()); - detailsDocument.setCountryName(entry.getCountryName()); - detailsDocument.setRegionName(entry.getRegionName()); - detailsDocument.setCityName(entry.getCityName()); - detailsDocument.setAsNumber(entry.getASNumber()); - detailsDocument.setAsName(entry.getASName()); - detailsDocument.setConsensusWeight(entry.getConsensusWeight()); - detailsDocument.setHostName(entry.getHostName()); + private void updateRelayDetailsFile(String fingerprint, + DetailsStatus detailsStatus) { + DetailsDocument detailsDocument = new DetailsDocument(); + detailsDocument.setNickname(detailsStatus.getNickname()); + detailsDocument.setFingerprint(fingerprint); + List<String> orAddresses = new ArrayList<String>(); + orAddresses.add(detailsStatus.getAddress() + ":" + + detailsStatus.getOrPort()); + for (String orAddress : detailsStatus.getOrAddressesAndPorts()) { + orAddresses.add(orAddress.toLowerCase()); + } + detailsDocument.setOrAddresses(orAddresses); + if (detailsStatus.getDirPort() != 0) { + detailsDocument.setDirAddress(detailsStatus.getAddress() + ":" + + detailsStatus.getDirPort()); + } + detailsDocument.setLastSeen(detailsStatus.getLastSeenMillis()); + detailsDocument.setFirstSeen(detailsStatus.getFirstSeenMillis()); + detailsDocument.setLastChangedAddressOrPort( + detailsStatus.getLastChangedOrAddressOrPort()); + detailsDocument.setRunning(detailsStatus.isRunning()); + detailsDocument.setFlags(detailsStatus.getRelayFlags()); + detailsDocument.setConsensusWeight( + detailsStatus.getConsensusWeight()); + detailsDocument.setHostName(detailsStatus.getHostName()); + String defaultPolicy = detailsStatus.getDefaultPolicy(); + String portList = detailsStatus.getPortList(); + if (defaultPolicy != null && (defaultPolicy.equals("accept") || + defaultPolicy.equals("reject")) && portList != null) { + Map<String, List<String>> exitPolicySummary = + new HashMap<String, List<String>>(); + List<String> portsOrPortRanges = Arrays.asList(portList.split(",")); + exitPolicySummary.put(defaultPolicy, portsOrPortRanges); + detailsDocument.setExitPolicySummary(exitPolicySummary); + } + detailsDocument.setRecommendedVersion( + detailsStatus.getRecommendedVersion()); + detailsDocument.setCountry(detailsStatus.getCountryCode()); + detailsDocument.setLatitude(detailsStatus.getLatitude()); + detailsDocument.setLongitude(detailsStatus.getLongitude()); + detailsDocument.setCountryName(detailsStatus.getCountryName()); + detailsDocument.setRegionName(detailsStatus.getRegionName()); + detailsDocument.setCityName(detailsStatus.getCityName()); + detailsDocument.setAsNumber(detailsStatus.getASNumber()); + detailsDocument.setAsName(detailsStatus.getASName()); + if (detailsStatus.isRunning()) { detailsDocument.setConsensusWeightFraction( - (float) entry.getConsensusWeightFraction()); + detailsStatus.getConsensusWeightFraction()); detailsDocument.setGuardProbability( - (float) entry.getGuardProbability()); + detailsStatus.getGuardProbability()); detailsDocument.setMiddleProbability( - (float) entry.getMiddleProbability()); + detailsStatus.getMiddleProbability()); detailsDocument.setExitProbability( - (float) entry.getExitProbability()); - String defaultPolicy = entry.getDefaultPolicy(); - String portList = entry.getPortList(); - if (defaultPolicy != null && (defaultPolicy.equals("accept") || - defaultPolicy.equals("reject")) && portList != null) { - Map<String, List<String>> exitPolicySummary = - new HashMap<String, List<String>>(); - List<String> portsOrPortRanges = Arrays.asList( - portList.split(",")); - exitPolicySummary.put(defaultPolicy, portsOrPortRanges); - detailsDocument.setExitPolicySummary(exitPolicySummary); - } - detailsDocument.setRecommendedVersion( - entry.getRecommendedVersion()); - - /* Append descriptor-specific part and exit addresses from details - * status file. */ - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, true, fingerprint); - if (detailsStatus != null) { - detailsDocument.setLastRestarted( - detailsStatus.getLastRestarted()); - detailsDocument.setBandwidthRate( - detailsStatus.getBandwidthRate()); - detailsDocument.setBandwidthBurst( - detailsStatus.getBandwidthBurst()); - detailsDocument.setObservedBandwidth( - detailsStatus.getObservedBandwidth()); - detailsDocument.setAdvertisedBandwidth( - detailsStatus.getAdvertisedBandwidth()); - detailsDocument.setExitPolicy(detailsStatus.getExitPolicy()); - detailsDocument.setContact(detailsStatus.getContact()); - detailsDocument.setPlatform(detailsStatus.getPlatform()); - detailsDocument.setFamily(detailsStatus.getFamily()); - detailsDocument.setExitPolicyV6Summary( - detailsStatus.getExitPolicyV6Summary()); - detailsDocument.setHibernating(detailsStatus.getHibernating()); - if (detailsStatus.getExitAddresses() != null) { - SortedSet<String> exitAddresses = new TreeSet<String>(); - for (Map.Entry<String, Long> e : - detailsStatus.getExitAddresses().entrySet()) { - String exitAddress = e.getKey().toLowerCase(); - long scanMillis = e.getValue(); - if (!entry.getAddress().equals(exitAddress) && - !entry.getOrAddresses().contains(exitAddress) && - scanMillis >= this.now - DateTimeHelper.ONE_DAY) { - exitAddresses.add(exitAddress); - } - } - if (!exitAddresses.isEmpty()) { - detailsDocument.setExitAddresses(new ArrayList<String>( - exitAddresses)); - } - } - } - - /* Write details file to disk. */ - this.documentStore.store(detailsDocument, fingerprint); + detailsStatus.getExitProbability()); + } + detailsDocument.setLastRestarted(detailsStatus.getLastRestarted()); + detailsDocument.setBandwidthRate(detailsStatus.getBandwidthRate()); + detailsDocument.setBandwidthBurst(detailsStatus.getBandwidthBurst()); + detailsDocument.setObservedBandwidth( + detailsStatus.getObservedBandwidth()); + detailsDocument.setAdvertisedBandwidth( + detailsStatus.getAdvertisedBandwidth()); + detailsDocument.setExitPolicy(detailsStatus.getExitPolicy()); + detailsDocument.setContact(detailsStatus.getContact()); + detailsDocument.setPlatform(detailsStatus.getPlatform()); + detailsDocument.setFamily(detailsStatus.getFamily()); + detailsDocument.setExitPolicyV6Summary( + detailsStatus.getExitPolicyV6Summary()); + detailsDocument.setHibernating(detailsStatus.getHibernating()); + if (detailsStatus.getExitAddresses() != null) { + SortedSet<String> exitAddressesWithoutOrAddresses = + new TreeSet<String>(detailsStatus.getExitAddresses().keySet()); + exitAddressesWithoutOrAddresses.removeAll( + detailsStatus.getOrAddresses()); + detailsDocument.setExitAddresses(new ArrayList<String>( + exitAddressesWithoutOrAddresses)); } + this.documentStore.store(detailsDocument, fingerprint); } - private void updateBridgeDetailsFiles() { - for (String fingerprint : this.newBridges) { - - /* Generate network-status-specific part. */ - NodeStatus entry = this.documentStore.retrieve(NodeStatus.class, - true, fingerprint); - if (entry == null) { - continue; - } - DetailsDocument detailsDocument = new DetailsDocument(); - detailsDocument.setNickname(entry.getNickname()); - detailsDocument.setHashedFingerprint(fingerprint); - String address = entry.getAddress(); - List<String> orAddresses = new ArrayList<String>(); - orAddresses.add(address + ":" + entry.getOrPort()); - for (String orAddress : entry.getOrAddressesAndPorts()) { + private void updateBridgeDetailsFile(String fingerprint, + DetailsStatus detailsStatus) { + DetailsDocument detailsDocument = new DetailsDocument(); + detailsDocument.setNickname(detailsStatus.getNickname()); + detailsDocument.setHashedFingerprint(fingerprint); + String address = detailsStatus.getAddress(); + List<String> orAddresses = new ArrayList<String>(); + orAddresses.add(address + ":" + detailsStatus.getOrPort()); + SortedSet<String> orAddressesAndPorts = + detailsStatus.getOrAddressesAndPorts(); + if (orAddressesAndPorts != null) { + for (String orAddress : orAddressesAndPorts) { orAddresses.add(orAddress.toLowerCase()); } - detailsDocument.setOrAddresses(orAddresses); - detailsDocument.setLastSeen(entry.getLastSeenMillis()); - detailsDocument.setFirstSeen(entry.getFirstSeenMillis()); - detailsDocument.setRunning(entry.getRunning()); - detailsDocument.setFlags(new ArrayList<String>( - entry.getRelayFlags())); - - /* Append descriptor-specific part from details status file. */ - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, true, fingerprint); - if (detailsStatus != null) { - detailsDocument.setLastRestarted( - detailsStatus.getLastRestarted()); - detailsDocument.setAdvertisedBandwidth( - detailsStatus.getAdvertisedBandwidth()); - detailsDocument.setPlatform(detailsStatus.getPlatform()); - detailsDocument.setPoolAssignment( - detailsStatus.getPoolAssignment()); - } - - /* Write details file to disk. */ - this.documentStore.store(detailsDocument, fingerprint); } + detailsDocument.setOrAddresses(orAddresses); + detailsDocument.setLastSeen(detailsStatus.getLastSeenMillis()); + detailsDocument.setFirstSeen(detailsStatus.getFirstSeenMillis()); + detailsDocument.setRunning(detailsStatus.isRunning()); + detailsDocument.setFlags(detailsStatus.getRelayFlags()); + detailsDocument.setLastRestarted(detailsStatus.getLastRestarted()); + detailsDocument.setAdvertisedBandwidth( + detailsStatus.getAdvertisedBandwidth()); + detailsDocument.setPlatform(detailsStatus.getPlatform()); + detailsDocument.setPoolAssignment(detailsStatus.getPoolAssignment()); + this.documentStore.store(detailsDocument, fingerprint); } public String getStatsString() { diff --git a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java index d7686cb..6406a28 100644 --- a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java @@ -29,16 +29,23 @@ public class SummaryDocumentWriter implements DocumentWriter { private int writtenDocuments = 0, deletedDocuments = 0; public void writeDocuments() { - long maxLastSeenMillis = 0L; + long relaysLastValidAfterMillis = -1L, + bridgesLastPublishedMillis = -1L; for (String fingerprint : this.documentStore.list(NodeStatus.class)) { NodeStatus nodeStatus = this.documentStore.retrieve( NodeStatus.class, true, fingerprint); - if (nodeStatus != null && - nodeStatus.getLastSeenMillis() > maxLastSeenMillis) { - maxLastSeenMillis = nodeStatus.getLastSeenMillis(); + if (nodeStatus != null) { + if (nodeStatus.isRelay()) { + relaysLastValidAfterMillis = Math.max( + relaysLastValidAfterMillis, nodeStatus.getLastSeenMillis()); + } else { + bridgesLastPublishedMillis = Math.max( + bridgesLastPublishedMillis, nodeStatus.getLastSeenMillis()); + } } } - long cutoff = maxLastSeenMillis - DateTimeHelper.ONE_WEEK; + long cutoff = Math.max(relaysLastValidAfterMillis, + bridgesLastPublishedMillis) - DateTimeHelper.ONE_WEEK; for (String fingerprint : this.documentStore.list(NodeStatus.class)) { NodeStatus nodeStatus = this.documentStore.retrieve( NodeStatus.class, @@ -68,8 +75,10 @@ public class SummaryDocumentWriter implements DocumentWriter { } } long lastSeenMillis = nodeStatus.getLastSeenMillis(); - boolean running = nodeStatus.getRunning(); SortedSet<String> relayFlags = nodeStatus.getRelayFlags(); + boolean running = relayFlags.contains("Running") && (isRelay ? + lastSeenMillis == relaysLastValidAfterMillis : + lastSeenMillis == bridgesLastPublishedMillis); long consensusWeight = nodeStatus.getConsensusWeight(); String countryCode = nodeStatus.getCountryCode(); long firstSeenMillis = nodeStatus.getFirstSeenMillis();

1 0

[onionoo/master] Add transport names to bridge details documents.
by karsten＠torproject.org 08 Dec '14

08 Dec '14

commit 655a63dc1da746b4ebf98f8315c12b3d9f1c8873 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Dec 2 11:31:53 2014 +0100 Add transport names to bridge details documents. Implements #11052. --- .../torproject/onionoo/docs/DetailsDocument.java | 9 ++++++++ .../org/torproject/onionoo/docs/DetailsStatus.java | 19 ++++++++++++++++ .../torproject/onionoo/server/ResponseBuilder.java | 2 ++ .../onionoo/updater/NodeDetailsStatusUpdater.java | 24 ++++++++++++++++++++ .../onionoo/writer/DetailsDocumentWriter.java | 1 + web/protocol.html | 10 ++++++++ 6 files changed, 65 insertions(+) diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java index 86abf9f..9e0a067 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java @@ -356,5 +356,14 @@ public class DetailsDocument extends Document { public String getPoolAssignment() { return this.pool_assignment; } + + private List<String> transports; + public void setTransports(List<String> transports) { + this.transports = (transports != null && !transports.isEmpty()) ? + transports : null; + } + public List<String> getTransports() { + return this.transports; + } } diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java index 967e493..085ce8d 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java @@ -130,6 +130,25 @@ public class DetailsStatus extends Document { return this.hibernating; } + /* From most recently published extra-info descriptor: */ + + private Long extra_info_desc_published; + public void setExtraInfoDescPublished(Long extraInfoDescPublished) { + this.extra_info_desc_published = extraInfoDescPublished; + } + public Long getExtraInfoDescPublished() { + return this.extra_info_desc_published; + } + + private List<String> transports; + public void setTransports(List<String> transports) { + this.transports = (transports != null && !transports.isEmpty()) ? + transports : null; + } + public List<String> getTransports() { + return this.transports; + } + /* From network status entries: */ private boolean is_relay; diff --git a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java index d2239aa..ee92e95 100644 --- a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java +++ b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java @@ -266,6 +266,8 @@ public class ResponseBuilder { dd.setHibernating(detailsDocument.getHibernating()); } else if (field.equals("pool_assignment")) { dd.setPoolAssignment(detailsDocument.getPoolAssignment()); + } else if (field.equals("transports")) { + dd.setTransports(detailsDocument.getTransports()); } } /* Don't escape HTML characters, like < and >, contained in diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java index 0ce17e4..102c6df 100644 --- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java @@ -20,6 +20,7 @@ import org.torproject.descriptor.BridgePoolAssignment; import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.ExitList; import org.torproject.descriptor.ExitListEntry; +import org.torproject.descriptor.ExtraInfoDescriptor; import org.torproject.descriptor.NetworkStatusEntry; import org.torproject.descriptor.RelayNetworkStatusConsensus; import org.torproject.descriptor.ServerDescriptor; @@ -113,6 +114,8 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, this.descriptorSource.registerDescriptorListener(this, DescriptorType.BRIDGE_SERVER_DESCRIPTORS); this.descriptorSource.registerDescriptorListener(this, + DescriptorType.BRIDGE_EXTRA_INFOS); + this.descriptorSource.registerDescriptorListener(this, DescriptorType.BRIDGE_POOL_ASSIGNMENTS); this.descriptorSource.registerDescriptorListener(this, DescriptorType.EXIT_LISTS); @@ -132,6 +135,9 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, (RelayNetworkStatusConsensus) descriptor); } else if (descriptor instanceof ServerDescriptor && !relay) { this.processBridgeServerDescriptor((ServerDescriptor) descriptor); + } else if (descriptor instanceof ExtraInfoDescriptor && !relay) { + this.processBridgeExtraInfoDescriptor( + (ExtraInfoDescriptor) descriptor); } else if (descriptor instanceof BridgePoolAssignment) { this.processBridgePoolAssignment((BridgePoolAssignment) descriptor); } else if (descriptor instanceof BridgeNetworkStatus) { @@ -289,6 +295,24 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, this.updatedNodes.add(fingerprint); } + private void processBridgeExtraInfoDescriptor( + ExtraInfoDescriptor descriptor) { + String fingerprint = descriptor.getFingerprint(); + DetailsStatus detailsStatus = this.documentStore.retrieve( + DetailsStatus.class, true, fingerprint); + if (detailsStatus == null) { + detailsStatus = new DetailsStatus(); + } else if (null == detailsStatus.getExtraInfoDescPublished() || + descriptor.getPublishedMillis() > + detailsStatus.getExtraInfoDescPublished()) { + detailsStatus.setExtraInfoDescPublished( + descriptor.getPublishedMillis()); + detailsStatus.setTransports(descriptor.getTransports()); + this.documentStore.store(detailsStatus, fingerprint); + this.updatedNodes.add(fingerprint); + } + } + private Map<String, String> bridgePoolAssignments = new HashMap<String, String>(); diff --git a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java index 7f464b6..59bb269 100644 --- a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java @@ -151,6 +151,7 @@ public class DetailsDocumentWriter implements DocumentWriter { detailsStatus.getAdvertisedBandwidth()); detailsDocument.setPlatform(detailsStatus.getPlatform()); detailsDocument.setPoolAssignment(detailsStatus.getPoolAssignment()); + detailsDocument.setTransports(detailsStatus.getTransports()); this.documentStore.store(detailsDocument, fingerprint); } diff --git a/web/protocol.html b/web/protocol.html index fed1c36..3766d90 100644 --- a/web/protocol.html +++ b/web/protocol.html @@ -1394,6 +1394,16 @@ available. </li> +<li> +transports +<code class="typeof">array of strings</code> +optional + +Array of (pluggable) transport names supported by this bridge. +Added on December 2, 2014. + +</li> + </ul> </div>

1 0

[onionoo/master] Remove pool assignments from bridge details documents.
by karsten＠torproject.org 08 Dec '14

08 Dec '14

commit 30809d9e600019c0e3a14967f963e9f13527c8d1 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Dec 2 11:36:59 2014 +0100 Remove pool assignments from bridge details documents. Related to #11052. --- .../org/torproject/onionoo/docs/DetailsDocument.java | 8 -------- .../org/torproject/onionoo/docs/DetailsStatus.java | 10 ---------- .../torproject/onionoo/server/ResponseBuilder.java | 2 -- .../onionoo/updater/NodeDetailsStatusUpdater.java | 18 ------------------ .../onionoo/writer/DetailsDocumentWriter.java | 1 - web/protocol.html | 3 ++- 6 files changed, 2 insertions(+), 40 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java index 9e0a067..06b74cf 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java @@ -349,14 +349,6 @@ public class DetailsDocument extends Document { return this.hibernating; } - private String pool_assignment; - public void setPoolAssignment(String poolAssignment) { - this.pool_assignment = poolAssignment; - } - public String getPoolAssignment() { - return this.pool_assignment; - } - private List<String> transports; public void setTransports(List<String> transports) { this.transports = (transports != null && !transports.isEmpty()) ? diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java index 085ce8d..96b01c2 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java @@ -290,16 +290,6 @@ public class DetailsStatus extends Document { return this.recommended_version; } - /* From bridge pool assignments: */ - - private String pool_assignment; - public void setPoolAssignment(String poolAssignment) { - this.pool_assignment = poolAssignment; - } - public String getPoolAssignment() { - return this.pool_assignment; - } - /* From exit lists: */ private Map<String, Long> exit_addresses; diff --git a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java index ee92e95..fc22a7c 100644 --- a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java +++ b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java @@ -264,8 +264,6 @@ public class ResponseBuilder { detailsDocument.getRecommendedVersion()); } else if (field.equals("hibernating")) { dd.setHibernating(detailsDocument.getHibernating()); - } else if (field.equals("pool_assignment")) { - dd.setPoolAssignment(detailsDocument.getPoolAssignment()); } else if (field.equals("transports")) { dd.setTransports(detailsDocument.getTransports()); } diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java index 102c6df..5b879a6 100644 --- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java @@ -16,7 +16,6 @@ import java.util.TreeSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.torproject.descriptor.BridgeNetworkStatus; -import org.torproject.descriptor.BridgePoolAssignment; import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.ExitList; import org.torproject.descriptor.ExitListEntry; @@ -116,8 +115,6 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, this.descriptorSource.registerDescriptorListener(this, DescriptorType.BRIDGE_EXTRA_INFOS); this.descriptorSource.registerDescriptorListener(this, - DescriptorType.BRIDGE_POOL_ASSIGNMENTS); - this.descriptorSource.registerDescriptorListener(this, DescriptorType.EXIT_LISTS); } @@ -138,8 +135,6 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, } else if (descriptor instanceof ExtraInfoDescriptor && !relay) { this.processBridgeExtraInfoDescriptor( (ExtraInfoDescriptor) descriptor); - } else if (descriptor instanceof BridgePoolAssignment) { - this.processBridgePoolAssignment((BridgePoolAssignment) descriptor); } else if (descriptor instanceof BridgeNetworkStatus) { this.processBridgeNetworkStatus((BridgeNetworkStatus) descriptor); } @@ -313,14 +308,6 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, } } - private Map<String, String> bridgePoolAssignments = - new HashMap<String, String>(); - - private void processBridgePoolAssignment( - BridgePoolAssignment bridgePoolAssignment) { - this.bridgePoolAssignments.putAll(bridgePoolAssignment.getEntries()); - } - private void processBridgeNetworkStatus(BridgeNetworkStatus status) { long publishedMillis = status.getPublishedMillis(); if (publishedMillis > this.bridgesLastPublishedMillis) { @@ -733,11 +720,6 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, nodeStatus.setLastRdnsLookup(this.startedRdnsLookups); } - if (this.bridgePoolAssignments.containsKey(fingerprint)) { - detailsStatus.setPoolAssignment( - this.bridgePoolAssignments.get(fingerprint)); - } - detailsStatus.setRelay(nodeStatus.isRelay()); detailsStatus.setRunning(nodeStatus.getLastSeenMillis() == (nodeStatus.isRelay() diff --git a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java index 59bb269..1a1ddc3 100644 --- a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java @@ -150,7 +150,6 @@ public class DetailsDocumentWriter implements DocumentWriter { detailsDocument.setAdvertisedBandwidth( detailsStatus.getAdvertisedBandwidth()); detailsDocument.setPlatform(detailsStatus.getPlatform()); - detailsDocument.setPoolAssignment(detailsStatus.getPoolAssignment()); detailsDocument.setTransports(detailsStatus.getTransports()); this.documentStore.store(detailsDocument, fingerprint); } diff --git a/web/protocol.html b/web/protocol.html index 3766d90..5f3b85f 100644 --- a/web/protocol.html +++ b/web/protocol.html @@ -1384,13 +1384,14 @@ information cannot be found. </li> <li> -pool_assignment +pool_assignment <code class="typeof">string</code> optional Information of the pool that BridgeDB assigned this bridge to, including further assignment information if available. +Removed on December 2, 2014. </li>

1 0

[onionoo/master] Update to protocol version 2.2.
by karsten＠torproject.org 08 Dec '14

08 Dec '14

commit f44f08918649b5e0bbc6f56ba1369ff594489b9a Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Dec 2 11:40:00 2014 +0100 Update to protocol version 2.2. Related to #11052. --- build.xml | 2 +- src/main/java/org/torproject/onionoo/server/ResponseBuilder.java | 2 +- web/protocol.html | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/build.xml b/build.xml index 9aa5fd5..fbb3837 100644 --- a/build.xml +++ b/build.xml @@ -1,6 +1,6 @@ <project default="dist" name="onionoo" basedir="."> - <property name="onionoo.protocol.version" value="2.1"/> + <property name="onionoo.protocol.version" value="2.2"/> <property name="release.version" value="${onionoo.protocol.version}.0"/> <property name="javasources" value="src/main/java"/> diff --git a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java index fc22a7c..371b79f 100644 --- a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java +++ b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java @@ -69,7 +69,7 @@ public class ResponseBuilder { return this.charsWritten; } - private static final String PROTOCOL_VERSION = "2.1"; + private static final String PROTOCOL_VERSION = "2.2"; private static final String NEXT_MAJOR_VERSION_SCHEDULED = null; diff --git a/web/protocol.html b/web/protocol.html index 5f3b85f..f19d0d8 100644 --- a/web/protocol.html +++ b/web/protocol.html @@ -171,6 +171,9 @@ fingerprints on November 15, 2014.</li> field from details documents and optional "advertised_bandwidth" and "advertised_bandwidth_fraction" fields from weights documents on November 16, 2014.</li> +<li>2.2: Removed optional "pool_assignment" field and +added "transports" field to bridge details documents on December 8, +2014.</li> </ul> </div>

1 0

[onionoo/master] Use StringUtils.join() in a few places.
by karsten＠torproject.org 08 Dec '14

08 Dec '14

commit 3c8330946171946f3764aac258bd23517397bc34 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Dec 8 09:57:04 2014 +0100 Use StringUtils.join() in a few places. Suggested by iwakeh in #12651. --- .../org/torproject/onionoo/docs/NodeStatus.java | 26 ++++---------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java index 99436bf..0985491 100644 --- a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java @@ -15,6 +15,7 @@ import java.util.SortedSet; import java.util.TreeMap; import java.util.TreeSet; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -426,27 +427,18 @@ public class NodeStatus extends Document { sb.append("\t" + this.nickname); sb.append("\t" + this.fingerprint); sb.append("\t" + this.address + ";"); - int written = 0; if (this.orAddressesAndPorts != null) { - for (String orAddressAndPort : this.orAddressesAndPorts) { - sb.append((written++ > 0 ? "+" : "") + orAddressAndPort); - } + sb.append(StringUtils.join(this.orAddressesAndPorts, "+")); } sb.append(";"); if (this.isRelay) { - written = 0; - for (String exitAddress : this.exitAddresses) { - sb.append((written++ > 0 ? "+" : "") + exitAddress); - } + sb.append(StringUtils.join(this.exitAddresses, "+")); } sb.append("\t" + DateTimeHelper.format(this.lastSeenMillis, DateTimeHelper.ISO_DATETIME_TAB_FORMAT)); sb.append("\t" + this.orPort); sb.append("\t" + this.dirPort + "\t"); - written = 0; - for (String relayFlag : this.getRelayFlags()) { - sb.append((written++ > 0 ? "," : "") + relayFlag); - } + sb.append(StringUtils.join(this.getRelayFlags(), ",")); if (this.isRelay) { sb.append("\t" + String.valueOf(this.consensusWeight)); sb.append("\t" @@ -472,15 +464,7 @@ public class NodeStatus extends Document { sb.append("\t" + (this.contact != null ? this.contact : "")); sb.append("\t" + (this.recommendedVersion == null ? "null" : this.recommendedVersion ? "true" : "false")); - if (this.familyFingerprints == null) { - sb.append("\tnull"); - } else { - sb.append("\t"); - written = 0; - for (String familyFingerprint : this.familyFingerprints) { - sb.append((written++ > 0 ? ";" : "") + familyFingerprint); - } - } + sb.append("\t" + StringUtils.join(this.familyFingerprints, ";")); return sb.toString(); } }

1 0

[onionoo/master] Add warnings for oversize documents.
by karsten＠torproject.org 08 Dec '14

08 Dec '14

commit 5abb70205061049f7a7c681f618f55d0f421f7bf Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Dec 8 08:58:41 2014 +0100 Add warnings for oversize documents. --- .../java/org/torproject/onionoo/docs/DocumentStore.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java index 5137fb4..4b7bca5 100644 --- a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java +++ b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java @@ -278,6 +278,10 @@ public class DocumentStore { return true; } + private static final long ONE_BYTE = 1L, + ONE_KIBIBYTE = 1024L * ONE_BYTE, + ONE_MIBIBYTE = 1024L * ONE_KIBIBYTE; + private <T extends Document> boolean storeDocumentFile(T document, String fingerprint) { File documentFile = this.getDocumentFile(document.getClass(), @@ -328,6 +332,11 @@ public class DocumentStore { return false; } try { + if (documentString.length() > ONE_MIBIBYTE) { + log.warn("Attempting to store very large document file: path='" + + documentFile.getAbsolutePath() + "', bytes=" + + documentString.length()); + } documentFile.getParentFile().mkdirs(); File documentTempFile = new File( documentFile.getAbsolutePath() + ".tmp"); @@ -452,6 +461,11 @@ public class DocumentStore { + documentFile.getAbsolutePath() + "'.", e); return null; } + if (documentString.length() > ONE_MIBIBYTE) { + log.warn("Retrieved very large document file: path='" + + documentFile.getAbsolutePath() + "', bytes=" + + documentString.length()); + } T result = null; if (!parse) { return this.retrieveUnparsedDocumentFile(documentType,

1 0