commit 5463eb8d9c3de6ddecb011da992b14d6524c3693 Author: Iain R. Learmonth <irl@fsfe.org> Date: Sun Nov 26 00:52:21 2017 +0000 Adds aggregated results table for relays grouped by country and/or AS (Fixes: #23517) --- css/atlas.css | 2 +- img/cc/README.flags | 2 +- img/cc/xz.png | Bin 0 -> 393 bytes index.html | 7 +-- js/collections/aggregates.js | 111 ++++++++++++++++++++++++++++++++++++++++ js/helpers.js | 1 + js/models/aggregate.js | 23 +++++++++ js/router.js | 73 +++++++++++++++++++++++--- js/views/aggregate/search.js | 47 +++++++++++++++++ js/views/search/main.js | 15 ++++++ templates/aggregate/search.html | 100 ++++++++++++++++++++++++++++++++++++ templates/search/main.html | 27 ++++++++++ 12 files changed, 397 insertions(+), 11 deletions(-) diff --git a/css/atlas.css b/css/atlas.css index 9069a55..cdbec5e 100644 --- a/css/atlas.css +++ b/css/atlas.css @@ -48,7 +48,7 @@ span.flags { background: #ff1515; } -#home-search { +#home-search, #home-aggregate-search { padding: 0; margin: 0 0 10px 0; width: 100%; diff --git a/img/cc/README.flags b/img/cc/README.flags index 7bf7691..95f81c4 100644 --- a/img/cc/README.flags +++ b/img/cc/README.flags @@ -8,7 +8,7 @@ To update these flags: * git clone https://github.com/gosquared/flags * cd flags; make * cp flags/flags-iso/shiny/16/* $RELAYSEARCH/img/cc/ - * cd $RELAYSEARCH/img/cc/; rename 'y/A-Z/a-z/' *.png + * cd $RELAYSEARCH/img/cc/; rename 'y/A-Z/a-z/' *.png; cp _unknown.png xz.png The flags are made available under the MIT license: diff --git a/img/cc/xz.png b/img/cc/xz.png new file mode 100644 index 0000000..44a6fc9 Binary files /dev/null and b/img/cc/xz.png differ diff --git a/index.html b/index.html index 8648eb5..408a94a 100644 --- a/index.html +++ b/index.html @@ -143,14 +143,15 @@ <div class="input-group add-on"> <input class="form-control" placeholder="Search" name="secondary-search-query" id="secondary-search-query" type="text" autocorrect="off" autocapitalize="none"> <div class="input-group-btn"> - <button class="btn btn-secondary" id="secondary-search-clear" type="button"><i class="glyphicon glyphicon-remove-circle"></i></span> - <button class="btn btn-primary" id="secondary-search-submit" type="submit"><i class="glyphicon glyphicon-search"></i></button> + <button class="btn btn-danger" id="secondary-search-clear" type="button" title="Clear Search Query"><i class="glyphicon glyphicon-remove-circle"></i></span> + <button class="btn btn-primary" id="secondary-search-submit" type="submit" title="Perform Search"><i class="glyphicon glyphicon-search"></i></button> + <button class="btn btn-secondary" id="secondary-search-aggregate" type="button" title="Perform Aggregated Search"><i class="fa fa-compress"></i></button> </div> </div> </form> <h1>Relay Search</h1> <div class="progress progress-info progress-striped active"> - <div class="bar"></div> + <div class="progress-bar"></div> </div> <div id="content"> <noscript> diff --git a/js/collections/aggregates.js b/js/collections/aggregates.js new file mode 100644 index 0000000..1161ff4 --- /dev/null +++ b/js/collections/aggregates.js @@ -0,0 +1,111 @@ +// ~ collections/aggregates ~ +define([ + 'jquery', + 'underscore', + 'backbone', + 'models/aggregate' +], function($, _, Backbone, aggregateModel){ + var aggregatesCollection = Backbone.Collection.extend({ + model: aggregateModel, + baseurl: 'https://onionoo.torproject.org/details?running=true&type=relay&fields=countr...', + url: '', + aType: 'cc', + lookup: function(options) { + var success = options.success; + var error = options.error; + var err = -1; + var collection = this; + options.success = $.getJSON(this.url, function(response) { + checkIfDataIsUpToDate(options.success.getResponseHeader("Last-Modified")); + this.fresh_until = response.fresh_until; + this.valid_after = response.valid_after; + var aggregates = {}; + var relaysPublished = response.relays_published; + var bridgesPublished = response.bridges_published; + options.error = function(options) { + error(options.error, collection, options); + } + _.each(response.relays, function(relay) { + /* If a relay country is unknown, use XZ as the country code. + This code will never be assigned for use with ISO 3166-1 and is "user-assigned". + Fun fact: UN/LOCODE assigns XZ to represent installations in international waters. */ + relay.country = ((typeof relay.country) == "undefined") ? "xz" : relay.country; + relay.as_number = ((typeof relay.as_number) == "undefined") ? 0 : relay.as_number; + if (relay.as_number == 0) relay.as_name = "Unknown"; + + var ccAggregate = false; + var asAggregate = false; + + if (collection.aType == "all") { + aggregateKey = "zz"; // A user-assigned ISO 3166-1 code, but really just a static key + } else if (collection.aType == "cc") { + aggregateKey = relay.country; + ccAggregate = true; + } else if (collection.aType == "as") { + aggregateKey = relay.as_number; + asAggregate = true; + } else { + aggregateKey = relay.country + "/" + relay.as_number; + ccAggregate = asAggregate = true; + } + + if (!(aggregateKey in aggregates)) { + aggregates[aggregateKey] = new aggregateModel; + if (ccAggregate) { + aggregates[aggregateKey].country = relay.country; + } else { + aggregates[aggregateKey].country = new Set(); + } + if (asAggregate) { + aggregates[aggregateKey].as = relay.as_number; + } else { + aggregates[aggregateKey].as = new Set(); + } + aggregates[aggregateKey].as_name = relay.as_name; + } + + if (!ccAggregate) { + if (relay.country !== "xz") aggregates[aggregateKey].country.add(relay.country); + } + if (!asAggregate) { + if (relay.as_number !== 0) aggregates[aggregateKey].as.add(relay.as_number); + } + + aggregates[aggregateKey].relays++; + if ((typeof relay.guard_probability) !== "undefined") aggregates[aggregateKey].guard_probability += relay.guard_probability; + if ((typeof relay.middle_probability) !== "undefined") aggregates[aggregateKey].middle_probability += relay.middle_probability; + if ((typeof relay.exit_probability) !== "undefined") aggregates[aggregateKey].exit_probability += relay.exit_probability; + if ((typeof relay.consensus_weight) !== "undefined") aggregates[aggregateKey].consensus_weight += relay.consensus_weight; + if ((typeof relay.consensus_weight_fraction) !== "undefined") aggregates[aggregateKey].consensus_weight_fraction += relay.consensus_weight_fraction; + if ((typeof relay.advertised_bandwidth) !== "undefined") aggregates[aggregateKey].advertised_bandwidth += relay.advertised_bandwidth; + _.each(relay.flags, function(flag) { + if (flag == "Guard") aggregates[aggregateKey].guards++; + if (flag == "Exit") aggregates[aggregateKey].exits++; + }); + }); + if (Object.keys(aggregates).length == 0) { + error(0); + return false; + } + _.each(Object.values(aggregates), function(aggregate) { + if ((typeof aggregate.as) !== "string") { + if (aggregate.as.size == 1) aggregate.as = Array.from(aggregate.as.values())[0]; + } + if ((typeof aggregate.country) !== "string") { + if (aggregate.country.size == 1) aggregate.country = Array.from(aggregate.country.values())[0]; + } + }); + collection[options.add ? 'add' : 'reset'](Object.values(aggregates), options); + success(err, relaysPublished, bridgesPublished); + }).fail(function(jqXHR, textStatus, errorThrown) { + if(jqXHR.statusText == "error") { + error(2); + } else { + error(3); + } + }); + } + }); + return aggregatesCollection; +}); + diff --git a/js/helpers.js b/js/helpers.js index 1037bb0..214db67 100644 --- a/js/helpers.js +++ b/js/helpers.js @@ -241,6 +241,7 @@ var CountryCodes = { "vu" : "Vanuatu", "wf" : "Wallis and Futuna", "ws" : "Samoa", + "xz" : "Unknown", "ye" : "Yemen", "yt" : "Mayotte", "za" : "South Africa", diff --git a/js/models/aggregate.js b/js/models/aggregate.js new file mode 100644 index 0000000..0b85050 --- /dev/null +++ b/js/models/aggregate.js @@ -0,0 +1,23 @@ +// ~ models/aggregateModel ~ +define([ + 'jquery', + 'underscore', + 'backbone', + 'helpers' +], function($, _, Backbone){ + var aggregateModel = Backbone.Model.extend({ + country: null, + as: null, + as_name: null, + guard_probability: 0, + middle_probability: 0, + exit_probability: 0, + advertised_bandwidth: 0, + consensus_weight: 0, + consensus_weight_fraction: 0, + relays: 0, + guards: 0, + exits: 0 + }); + return aggregateModel; +}); diff --git a/js/router.js b/js/router.js index 8b6034b..118c094 100644 --- a/js/router.js +++ b/js/router.js @@ -6,8 +6,9 @@ define([ 'views/details/main', 'views/search/main', 'views/search/do', + 'views/aggregate/search', 'jssha' -], function($, _, Backbone, mainDetailsView, mainSearchView, doSearchView, jsSHA){ +], function($, _, Backbone, mainDetailsView, mainSearchView, doSearchView, aggregateSearchView, jsSHA){ var AppRouter = Backbone.Router.extend({ routes: { // Define the routes for the actions in Atlas @@ -17,6 +18,8 @@ define([ 'search/': 'doSearch', 'top10': 'showTopRelays', 'toprelays': 'showTopRelays', + 'aggregate(/:aType)(/:query)': 'aggregateSearch', + 'aggregate(/:aType)/': 'emptyAggregateSearch', // Default '*actions': 'defaultAction' }, @@ -39,7 +42,7 @@ define([ mainDetailsView.model.fingerprint = this.hashFingerprint(fingerprint); mainDetailsView.model.lookup({ success: function(relay) { - mainDetailsView.render(); + mainDetailsView.render(); $(".progress").hide(); $("#content").show(); $(".breadcrumb").html("<li><a href=\"https://metrics.torproject.org/\">Home</a></li><li><a href=\"https://metrics.torproject.org/services.html\">Services</a></li><li><a href=\"#\">Relay Search</a></li><li class=\"active\">Details for " + relay.get('nickname') + "</li>"); @@ -55,12 +58,63 @@ define([ } }); }, + // Empty aggregation query + emptyAggregateSearch: function() { + $(".breadcrumb").html("<li><a href=\"https://metrics.torproject.org/\">Home</a></li><li><a href=\"https://metrics.torproject.org/services.html\">Services</a></li><li><a href=\"#\">Relay Search</a></li><li class=\"active\">Error</li>"); + $("#secondary-search").show(); + $("#secondary-search-query").val(""); + + $("#content").hide(); + $(".progress").show(); + doSearchView.error = 5; + doSearchView.renderError(); + $(".progress").hide(); + $("#content").show(); + + }, + // Perform a countries aggregation + aggregateSearch: function(aType, query){ + $(".breadcrumb").html("<li><a href=\"https://metrics.torproject.org/\">Home</a></li><li><a href=\"https://metrics.torproject.org/services.html\">Services</a></li><li><a href=\"#\">Relay Search</a></li><li class=\"active\">Aggregated search" + ((query) ? " for " + query : "") + "</li>"); + $("#secondary-search").show(); + + $("#content").hide(); + $(".progress").show(); + + aggregateSearchView.collection.aType = (aType) ? aType : "all"; + + if (query) { + query = query.trim(); + $("#secondary-search-query").val(query); + aggregateSearchView.collection.url = + aggregateSearchView.collection.baseurl + "&search=" + this.hashFingerprint(query); + } else { + aggregateSearchView.collection.url = + aggregateSearchView.collection.baseurl; + query = ""; + } + aggregateSearchView.collection.lookup({ + success: function(err, relaysPublished, bridgesPublished){ + aggregateSearchView.error = err; + aggregateSearchView.relaysPublished = relaysPublished; + aggregateSearchView.bridgesPublished = bridgesPublished; + aggregateSearchView.render(query); + $("#search-title").text("Aggregated results" + ((query) ? " for " + query : "")); + $(".progress").hide(); + $("#content").show(); + }, + error: function(err){ + aggregateSearchView.error = err; + aggregateSearchView.renderError(); + $(".progress").hide(); + $("#content").show(); + } + }); + }, // Perform a search on Atlas doSearch: function(query){ $(".breadcrumb").html("<li><a href=\"https://metrics.torproject.org/\">Home</a></li><li><a href=\"https://metrics.torproject.org/services.html\">Services</a></li><li><a href=\"#\">Relay Search</a></li><li class=\"active\">Search for " + query + "</li>"); $("#secondary-search").show(); - $("#secondary-search-query").val(query); $("#content").hide(); $(".progress").show(); @@ -71,9 +125,11 @@ define([ $(".progress").hide(); $("#content").show(); } else { - doSearchView.collection.url = - doSearchView.collection.baseurl + this.hashFingerprint(query); - doSearchView.collection.lookup({ + query = query.trim(); + $("#secondary-search-query").val(query); + doSearchView.collection.url = + doSearchView.collection.baseurl + this.hashFingerprint(query); + doSearchView.collection.lookup({ success: function(err, relaysPublished, bridgesPublished){ doSearchView.relays = doSearchView.collection.models; // Redirect to the details page when there is exactly one @@ -153,6 +209,11 @@ define([ return false; }); + $("#secondary-search-aggregate").bind('click', function(){ + document.location = "#aggregate/all/"+encodeURI($('#secondary-search-query').val()); + return false; + }); + $("#secondary-search-clear").bind('click', function(){ $("#secondary-search-query").val(""); return false; diff --git a/js/views/aggregate/search.js b/js/views/aggregate/search.js new file mode 100644 index 0000000..d189908 --- /dev/null +++ b/js/views/aggregate/search.js @@ -0,0 +1,47 @@ +// ~ views/search/do ~ +define([ + 'jquery', + 'underscore', + 'backbone', + 'collections/aggregates', + 'text!templates/aggregate/search.html', + 'datatables', + 'datatablessort', + 'helpers', + 'bootstrap', + 'datatablesbs' +], function($, _, Backbone, aggregatesCollection, aggregateSearchTemplate){ + var doCountriesView = Backbone.View.extend({ + el: "#content", + initialize: function() { + this.collection = new aggregatesCollection; + }, + render: function(query){ + document.title = "Relay Search"; + var compiledTemplate = _.template(aggregateSearchTemplate) + this.$el.html(compiledTemplate({query: query, + aggregates: this.collection.models, + countries: CountryCodes, + error: this.error, + relaysPublished: this.relaysPublished, + bridgesPublished: this.bridgesPublished})); + + // This creates the table using DataTables + //loadSortingExtensions(); + var oTable = $('#torstatus_results').dataTable({ + "sDom": "<\"top\"l>rt<\"bottom\"ip><\"clear\">", + "bStateSave": false, + "aaSorting": [[2, "desc"]], + "fnDrawCallback": function( oSettings ) { + $(".tip").tooltip({'html': true}); + } + }); + }, + renderError: function(){ + var compiledTemplate = _.template(aggregateSearchTemplate); + this.$el.html(compiledTemplate({aggregates: null, error: this.error, countries: null})); + } + }); + return new doCountriesView; +}); + diff --git a/js/views/search/main.js b/js/views/search/main.js index 7bf97b1..6eedbe2 100644 --- a/js/views/search/main.js +++ b/js/views/search/main.js @@ -28,6 +28,21 @@ define([ document.location = "#search/"+encodeURI($('#query').val()); return false; }); + + $("#do-aggregate").bind('click', function(){ + document.location = "#aggregate/all/"+encodeURI($('#aggregated-query').val()); + return false; + }); + + $("#do-full-aggregation").bind('click', function(){ + document.location = "#aggregate/all"; + return false; + }); + + $("#home-aggregate-search").bind('submit', function(){ + document.location = "#aggregate/all/"+encodeURI($('#aggregated-query').val()); + return false; + }); } }); return new mainSearchView; diff --git a/templates/aggregate/search.html b/templates/aggregate/search.html new file mode 100644 index 0000000..34db02c --- /dev/null +++ b/templates/aggregate/search.html @@ -0,0 +1,100 @@ + +<h2 id="search-title"></h2> + +<div class="results_box"> +<% if(!aggregates) { %> + <% if(error == 0) { %> + <div class="alert alert-info"> + <strong>No Results found!</strong><p> + No Tor relays matched your query :(</p> + <p><a href="#">Return to home page</a></p> + </div> + <% } else if (error == 2) { %> + <div class="alert alert-error"> + <strong>Backend error!</strong> + <p>Relay Search is unable to get a response from its backend server. This + probably means that the backend server is unavailable right now. This + can also happen, however, if you did not format your query correctly. + Please have a look at <a href="#about">the About page</a> that explains + what type of search queries are supported by Relay Search.</p> + </div> + <% } else if (error == 3) { %> + <div class="alert alert-error"> + <strong>JavaScript Error!</strong><p>There is a problem with your + javascript environment, you may have noscript enabled on the remote + onionoo backend. Try temporarily allowing noscript to connect to the + backend IP address. If the problem persits consult <a + href="https://trac.torproject.org/">the bugtracker.</a></p> + </div> + <% } else if (error == 5) { %> + <div class="alert alert-error"> + <strong>No query submitted!</strong> + <p>The search query was found to be empty, which is not supported. You + must enter a search query in order to generate results. Please have a + look at <a href="#about">the About page</a> that explains what type of + search queries are supported by Relay Search.</p> + </div> + <% } %> +<% } else { %> + +<table class="table table-hover table-striped" id="torstatus_results"> + <thead> + <tr> + <th>Country</sup></th> + <th>Autonomous System</th> + <th>Consensus Weight</th> + <th>Advertised Bandwidth</th> + <th>Guard Probability</th> + <th>Middle Probability</th> + <th>Exit Probability</th> + <th>Relays</th> + <th>Guard</th> + <th>Exit</th> + </tr> + </thead> + <tbody> + +<% _.each(aggregates, function(aggregate) { %> + <tr> + <td> + <% if ((typeof aggregate.country) == "string") { %> + <a href="#search/<%= (query) ? query + " " : "" %><% if (query.indexOf("country:") == -1) { %>country:<%= aggregate.country %><% } %>"><img class="inline country" src="img/cc/<%= aggregate.country %>.png"> <%= countries[aggregate.country] %></a> + <% } else { %> + <% if ((typeof aggregate.as) == "string") { %> + (<a href="#aggregate/ascc/<%= (query) ? query + " " : "" %><% if (query.indexOf("as:") == -1) { %>as:<%= aggregate.as %><% } %>"><%= aggregate.country.size %> distinct</a>) + <% } else { %> + (<a href="#aggregate/cc<%= (query) ? "/" + query : "" %>"><%= aggregate.country.size %> distinct</a>) + <% } %> + <% } %> + </td> + <td> + <% if ((typeof aggregate.as) == "string") { %> + <a href="#search/<%= (query) ? query + " " : "" %><% if (query.indexOf("as:") == -1) { %>as:<%= aggregate.as %><% } %>"><%= aggregate.as_name %> (<%= aggregate.as %>)</a> + <% } else { %> + <% if ((typeof aggregate.country) == "string") { %> + (<a href="#aggregate/ascc/<%= (query) ? query + " " : "" %><% if (query.indexOf("country:") == -1) { %>country:<%= aggregate.country %><% } %>"><%= aggregate.as.size %> distinct</a>) + <% } else { %> + (<a href="#aggregate/as<%= (query) ? "/" + query : "" %>"><%= aggregate.as.size %> distinct</a>) + <% } %> + <% } %> + </td> + <td data-order="<%= aggregate.consensus_weight_fraction %>"><span class="tip" title="<%= aggregate.consensus_weight %>"><%= (aggregate.consensus_weight_fraction * 100).toFixed(4) %>%</span></td> + <td data-order="<%= aggregate.advertised_bandwidth %>"><%= hrBandwidth(aggregate.advertised_bandwidth) %></span></td> + <td data-order="<%= aggregate.guard_probability %>"><%= (aggregate.guard_probability * 100).toFixed(4) %>%</td> + <td data-order="<%= aggregate.middle_probability %>"><%= (aggregate.middle_probability * 100).toFixed(4) %>%</td> + <td data-order="<%= aggregate.exit_probability %>"><%= (aggregate.exit_probability * 100).toFixed(4) %>%</td> + <td><%= aggregate.relays %></td> + <td><%= aggregate.guards %></td> + <td><%= aggregate.exits %></td> + </tr> +<% }); %> +</tbody> +</table> + <p>The aggregated search tool displays aggregated data about relays in the +Tor network. It provides insight into diversity in the network and the +probabilities of using relays in a particular country or AS as a guard, middle +or exit relay. The results are restricted to only relays that were running at +the last time the relays data was updated and do not include bridge data.</p> +<p>Information for relays was published: <%= relaysPublished %>.<p> +<% } %> +</div> diff --git a/templates/search/main.html b/templates/search/main.html index d5f6d6d..14a4916 100644 --- a/templates/search/main.html +++ b/templates/search/main.html @@ -1,3 +1,11 @@ +<ul class="nav nav-tabs"> + <li id="main-search-tab" class="search-tabs active"><a onclick="$('.search').hide();$('#main-search-tab-content').fadeIn();$('.search-tabs').removeClass('active');$('#main-search-tab').addClass('active');">Simple Search</a></li> + <li id="aggregated-search-tab" class="search-tabs"><a onclick="$('.search').hide();$('#aggregated-search-tab-content').fadeIn();$('.search-tabs').removeClass('active');$('#aggregated-search-tab').addClass('active');">Aggregated Search</a></li> +</ul> + +<div class="tab-content" id="search-tab-content"> + <div id="main-search-tab-content" class="search tab-pane active"> + <p>The relay search tool displays data about single relays and bridges in the Tor network. It provides useful information on how relays are configured along with graphs about their past.</p> @@ -7,6 +15,24 @@ with graphs about their past.</p> <span class="input-group-btn"><button id="do-search" class="btn btn-primary" type="button">Search</button><button class="btn btn-secondary" type="button" id="do-top-relays">Top Relays</button></span> </div> </form> +</div> + + <div id="aggregated-search-tab-content" class="search tab-pane"> + + <p>The aggregated search tool displays aggregated data about relays in the +Tor network. It provides insight into diversity in the network and the +probabilities of using relays in a particular country or AS as a guard, middle +or exit relay. The results are restricted to only currently running relays and +do not include bridge data.</p> + <form id="home-aggregate-search"> + <div class="input-group"> + <input class="search-query form-control" id="aggregated-query" placeholder="Query" type="text" autocorrect="off" autocapitalize="none"> + <span class="input-group-btn"><button id="do-aggregate" class="btn btn-primary" type="button">Aggregated Search</button><button class="btn btn-secondary" type="button" id="do-full-aggregation">Entire Network</button></span> + </div> + </form> + </div> +</div> + <div class="well"> <p>You can search for Tor relays and bridges by using keywords. In particular, this tool enables you to search for (partial) nicknames (e.g., @@ -25,3 +51,4 @@ Tor data directory. On Debian systems, this is in <code>/var/lib/tor</code> but may be in another location on your system. The location is specified as <code>DataDirectory</code> in your <code>torrc</code>.</p> </div> +