November 2019 - tor-commits - lists.torproject.org

[metrics-web/master] Update to metrics-lib 2.9.1 and ExoneraTor 4.2.0.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 9d34311e53389493e68d5b885da9928454807044 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sat Nov 9 20:38:51 2019 +0100 Update to metrics-lib 2.9.1 and ExoneraTor 4.2.0. --- CHANGELOG.md | 1 + build.xml | 4 ++-- src/submods/metrics-lib | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2147ed7..b8a4ed0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ few exceptions. - Remove Cobertura from the build process. - Update PostgreSQL JDBC driver version to 42.2.5. + - Update to metrics-lib 2.9.1 and ExoneraTor 4.2.0. # Changes in version 1.2.0 - 2018-08-25 diff --git a/build.xml b/build.xml index 959d069..3efe4bc 100644 --- a/build.xml +++ b/build.xml @@ -10,8 +10,8 @@ <property name="javadoc-title" value="MetricsWeb API Documentation"/> <property name="implementation-title" value="metrics-web" /> <property name="release.version" value="1.2.0-dev" /> - <property name="metricslibversion" value="2.6.2" /> - <property name="exoneratorversion" value="4.1.0" /> + <property name="metricslibversion" value="2.9.1" /> + <property name="exoneratorversion" value="4.2.0" /> <property name="jetty.version" value="-9.2.21.v20170120" /> <property name="warfile" value="metrics-web-${release.version}.war"/> diff --git a/src/submods/metrics-lib b/src/submods/metrics-lib index 8763af4..87d998c 160000 --- a/src/submods/metrics-lib +++ b/src/submods/metrics-lib @@ -1 +1 @@ -Subproject commit 8763af44465445b2257f9740d4ce341f8334eead +Subproject commit 87d998c542688c7e465486de24ef07c1cbece0b0

1 0

[metrics-web/master] Prepare for 1.3.0 release.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit d4581513c4ca31a007df243c10e697600d2b99e6 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sat Nov 9 21:43:49 2019 +0100 Prepare for 1.3.0 release. --- CHANGELOG.md | 2 +- build.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8a4ed0..5cccaab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 1.3.0 - 2018-??-?? +# Changes in version 1.3.0 - 2019-11-09 * Medium changes - Start downloading and processing votes. diff --git a/build.xml b/build.xml index 3efe4bc..e08a44a 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ <property name="javadoc-title" value="MetricsWeb API Documentation"/> <property name="implementation-title" value="metrics-web" /> - <property name="release.version" value="1.2.0-dev" /> + <property name="release.version" value="1.3.0" /> <property name="metricslibversion" value="2.9.1" /> <property name="exoneratorversion" value="4.2.0" /> <property name="jetty.version" value="-9.2.21.v20170120" />

1 0

[exonerator/release] Adds GitLab CI for tests and checks
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 64782695e66679f485eeaa6030a80ec01598c25c Author: Iain R. Learmonth <irl(a)fsfe.org> Date: Thu Sep 5 13:59:05 2019 +0100 Adds GitLab CI for tests and checks Configuration is held in the .gitlab-ci.yml file. Fixes: #31401 --- .gitlab-ci.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..b67b5a4 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,25 @@ +variables: + GIT_STRATEGY: clone + METRICS_LIB_VERSION: "2.6.2" + JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8" + +stages: + - test + +test: + stage: test + image: debian:buster + script: + - apt update + - apt -y install default-jdk ant ivy git curl + - git submodule init + - git submodule update + - mkdir lib + - mkdir tmp + - pushd tmp + - curl https://dist.torproject.org/metrics-lib/$METRICS_LIB_VERSION/metrics-lib-$M… | tar xzf - + - popd + - mv tmp/metrics-lib-$METRICS_LIB_VERSION/generated/dist/metrics-lib-$METRICS_LIB_VERSION-thin.jar lib + - ant -lib /usr/share/java resolve + - ant test + - ant checks

1 0

[exonerator/release] Update PostgreSQL JDBC driver version to 42.2.5.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 6babf3f62e7b23beed348d27b25e897936f96b72 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Sep 17 10:27:28 2019 +0200 Update PostgreSQL JDBC driver version to 42.2.5. --- CHANGELOG.md | 1 + build.xml | 2 +- src/build | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f0db8f..ddbae79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ versions resolved by Ivy are the same as in Debian stretch with few exceptions. - Remove Cobertura from the build process. + - Update PostgreSQL JDBC driver version to 42.2.5. # Changes in version 4.1.0 - 2019-05-13 diff --git a/build.xml b/build.xml index 9aab930..9d522fe 100644 --- a/build.xml +++ b/build.xml @@ -40,7 +40,7 @@ <include name="jackson-databind-2.8.6.jar"/> <include name="logback-core-1.1.9.jar" /> <include name="logback-classic-1.1.9.jar" /> - <include name="postgresql-9.4.1212.jar"/> + <include name="postgresql-42.2.5.jar"/> <include name="slf4j-api-1.7.22.jar"/> </patternset> diff --git a/src/build b/src/build index 75cda35..0674958 160000 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 75cda35ba7375ab0f5b12fa214620c3a29917097 +Subproject commit 0674958512c705255d4ec4631d3280068ce0a7af

1 0

[exonerator/release] Use metrics-base ant task to fetch metrics-lib.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit bc0c4db8ad9048d8f826b781274055ab9d475e9a Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Sep 27 10:06:08 2019 +0200 Use metrics-base ant task to fetch metrics-lib. Implements part of #31649. --- .gitlab-ci.yml | 7 +------ src/build | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b67b5a4..897c9da 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,5 @@ variables: GIT_STRATEGY: clone - METRICS_LIB_VERSION: "2.6.2" JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8" stages: @@ -15,11 +14,7 @@ test: - git submodule init - git submodule update - mkdir lib - - mkdir tmp - - pushd tmp - - curl https://dist.torproject.org/metrics-lib/$METRICS_LIB_VERSION/metrics-lib-$M… | tar xzf - - - popd - - mv tmp/metrics-lib-$METRICS_LIB_VERSION/generated/dist/metrics-lib-$METRICS_LIB_VERSION-thin.jar lib + - ant fetch-metrics-lib - ant -lib /usr/share/java resolve - ant test - ant checks diff --git a/src/build b/src/build index 0674958..07c2a00 160000 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 0674958512c705255d4ec4631d3280068ce0a7af +Subproject commit 07c2a00c27f0d536223f8b5a61fc91e60eb524d4

1 0

[exonerator/release] Update to latest metrics-base.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 45c5285a3514d7c5acc3fcf040eb18dcfd8b7e16 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Sep 30 17:42:33 2019 +0200 Update to latest metrics-base. --- src/build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build b/src/build index 07c2a00..62a964a 160000 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 07c2a00c27f0d536223f8b5a61fc91e60eb524d4 +Subproject commit 62a964a7702086ca6ba0e2307809d4f983163b47

1 0

[exonerator/release] Make Jetty listen host configurable.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit c6f7b83915888c9b4b61a8aa6ae39ec1b2248b71 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Sep 30 17:43:22 2019 +0200 Make Jetty listen host configurable. The default listen host is still 127.0.0.1, but now it's possible to configure a different listen host, like 0.0.0.0, which can facilitate local testing. --- src/main/resources/jetty.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/resources/jetty.xml b/src/main/resources/jetty.xml index e1e4d4d..56887f7 100644 --- a/src/main/resources/jetty.xml +++ b/src/main/resources/jetty.xml @@ -75,7 +75,9 @@ <Set name="port"> <SystemProperty name="exonerator.jetty.port" default="8080" /> </Set> - <Set name="host">127.0.0.1</Set> + <Set name="host"> + <SystemProperty name="exonerator.jetty.host" default="127.0.0.1" /> + </Set> </New> </Arg> </Call>

1 0

[exonerator/release] Print out a warning if we're missing data.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit b0165cb03138e933a337ddc8400c3a8bbe889b3d Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Sep 30 17:44:38 2019 +0200 Print out a warning if we're missing data. We're now counting the hours for which we have statuses and exit lists. More precisely, we're truncating consensus valid-after times and exit scan times to the hour and counting how many distinct hours we saw during the requested time. If we're missing 18 or more hours of statuses or exit lists we're printing out a note in the summary. Fixes #31071. --- .../metrics/exonerator/ExoneraTorServlet.java | 43 ++++++++++++++++------ .../metrics/exonerator/QueryResponse.java | 21 ++++++++++- .../metrics/exonerator/QueryServlet.java | 24 +++++++++++- src/main/resources/ExoneraTor.properties | 1 + src/main/sql/exonerator2.sql | 42 +++++++++++++++------ .../metrics/exonerator/ExoneraTorServletTest.java | 2 +- .../metrics/exonerator/QueryResponseTest.java | 12 ++++-- 7 files changed, 112 insertions(+), 33 deletions(-) diff --git a/src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java b/src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java index a74aa60..4b0cf16 100644 --- a/src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java +++ b/src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java @@ -89,6 +89,7 @@ public class ExoneraTorServlet extends HttpServlet { ExoneraTorDate firstDate = ExoneraTorDate.INVALID; ExoneraTorDate lastDate = ExoneraTorDate.INVALID; boolean noRelevantConsensuses = true; + boolean missingData = false; List<String[]> statusEntries = new ArrayList<>(); List<String> addressesInSameNetwork = null; @@ -105,6 +106,14 @@ public class ExoneraTorServlet extends HttpServlet { && queryResponse.relevantStatuses) { noRelevantConsensuses = false; } + if (null != queryResponse.missingStatuses + && queryResponse.missingStatuses) { + missingData = true; + } + if (null != queryResponse.missingExitLists + && queryResponse.missingExitLists) { + missingData = true; + } if (null != queryResponse.matches) { for (QueryResponse.Match match : queryResponse.matches) { StringBuilder sb = new StringBuilder(); @@ -210,15 +219,18 @@ public class ExoneraTorServlet extends HttpServlet { /* Print out result. */ } else { if (!statusEntries.isEmpty()) { - this.writeSummaryPositive(out, rb, relayIp, requestedDate.asString); + this.writeSummaryPositive(out, rb, relayIp, requestedDate.asString, + missingData); this.writeTechnicalDetails(out, rb, relayIp, requestedDate.asString, statusEntries); } else if (addressesInSameNetwork != null && !addressesInSameNetwork.isEmpty()) { this.writeSummaryAddressesInSameNetwork(out, rb, requestUrl, relayIp, - requestedDate.asString, langStr, addressesInSameNetwork); + requestedDate.asString, langStr, addressesInSameNetwork, + missingData); } else { - this.writeSummaryNegative(out, rb, relayIp, requestedDate.asString); + this.writeSummaryNegative(out, rb, relayIp, requestedDate.asString, + missingData); } this.writePermanentLink(out, rb, requestUrl, relayIp, requestedDate.asString, langStr); @@ -398,13 +410,13 @@ public class ExoneraTorServlet extends HttpServlet { private void writeSummaryNoTimestamp(PrintWriter out, ResourceBundle rb) { this.writeSummary(out, rb.getString("summary.heading"), "panel-danger", - rb.getString("summary.invalidparams.notimestamp.title"), null, + rb.getString("summary.invalidparams.notimestamp.title"), null, null, rb.getString("summary.invalidparams.notimestamp.body")); } private void writeSummaryNoIp(PrintWriter out, ResourceBundle rb) { this.writeSummary(out, rb.getString("summary.heading"), - "panel-danger", rb.getString("summary.invalidparams.noip.title"), + "panel-danger", rb.getString("summary.invalidparams.noip.title"), null, null, rb.getString("summary.invalidparams.noip.body")); } @@ -413,7 +425,7 @@ public class ExoneraTorServlet extends HttpServlet { String lastDate) { this.writeSummary(out, rb.getString("summary.heading"), "panel-danger", - rb.getString("summary.invalidparams.timestamprange.title"), null, + rb.getString("summary.invalidparams.timestamprange.title"), null, null, rb.getString("summary.invalidparams.timestamprange.body"), timestampStr, firstDate, lastDate); } @@ -438,7 +450,7 @@ public class ExoneraTorServlet extends HttpServlet { : StringEscapeUtils.escapeHtml4(timestampParameter); this.writeSummary(out, rb.getString("summary.heading"), "panel-danger", - rb.getString("summary.invalidparams.invalidtimestamp.title"), + rb.getString("summary.invalidparams.invalidtimestamp.title"), null, null, rb.getString("summary.invalidparams.invalidtimestamp.body"), escapedTimestampParameter, "\"YYYY-MM-DD\""); } @@ -446,7 +458,7 @@ public class ExoneraTorServlet extends HttpServlet { private void writeSummaryTimestampTooRecent(PrintWriter out, ResourceBundle rb) { this.writeSummary(out, rb.getString("summary.heading"), "panel-danger", - rb.getString("summary.invalidparams.timestamptoorecent.title"), + rb.getString("summary.invalidparams.timestamptoorecent.title"), null, null, rb.getString("summary.invalidparams.timestamptoorecent.body")); } @@ -465,7 +477,8 @@ public class ExoneraTorServlet extends HttpServlet { void writeSummaryAddressesInSameNetwork(PrintWriter out, ResourceBundle rb, String requestUrl, String relayIp, String timestampStr, - String langStr, List<String> addressesInSameNetwork) { + String langStr, List<String> addressesInSameNetwork, + boolean missingData) { Object[][] panelItems = new Object[addressesInSameNetwork.size()][]; for (int i = 0; i < addressesInSameNetwork.size(); i++) { String addressInSameNetwork = addressesInSameNetwork.get(i); @@ -486,33 +499,36 @@ public class ExoneraTorServlet extends HttpServlet { this.writeSummary(out, rb.getString("summary.heading"), "panel-warning", rb.getString("summary.negativesamenetwork.title"), panelItems, + missingData ? rb.getString("summary.missingdata") : null, rb.getString("summary.negativesamenetwork.body"), relayIp, timestampStr, relayIp.contains(":") ? 48 : 24); } private void writeSummaryPositive(PrintWriter out, ResourceBundle rb, - String relayIp, String timestampStr) { + String relayIp, String timestampStr, boolean missingData) { String formattedRelayIp = relayIp.contains(":") ? "[" + relayIp + "]" : relayIp; this.writeSummary(out, rb.getString("summary.heading"), "panel-success", rb.getString("summary.positive.title"), null, + missingData ? rb.getString("summary.missingdata") : null, rb.getString("summary.positive.body"), formattedRelayIp, timestampStr); } private void writeSummaryNegative(PrintWriter out, ResourceBundle rb, - String relayIp, String timestampStr) { + String relayIp, String timestampStr, boolean missingData) { String formattedRelayIp = relayIp.contains(":") ? "[" + relayIp + "]" : relayIp; this.writeSummary(out, rb.getString("summary.heading"), "panel-warning", rb.getString("summary.negative.title"), null, + missingData ? rb.getString("summary.missingdata") : null, rb.getString("summary.negative.body"), formattedRelayIp, timestampStr); } private void writeSummary(PrintWriter out, String heading, String panelContext, String panelTitle, Object[][] panelItems, - String panelBodyTemplate, Object... panelBodyArgs) { + String panelWarning, String panelBodyTemplate, Object... panelBodyArgs) { out.printf(" <div class=\"row\">\n" + " <div class=\"col-xs-12\">\n" + " <h2>%s</h2>\n" @@ -531,6 +547,9 @@ public class ExoneraTorServlet extends HttpServlet { } out.print(" </ul>\n"); } + if (null != panelWarning) { + out.printf(" <p>%s</p>\n", panelWarning); + } out.print(" </div>\n" + " </div>\n" + " </div>\n" diff --git a/src/main/java/org/torproject/metrics/exonerator/QueryResponse.java b/src/main/java/org/torproject/metrics/exonerator/QueryResponse.java index 6a8976a..d148d0c 100644 --- a/src/main/java/org/torproject/metrics/exonerator/QueryResponse.java +++ b/src/main/java/org/torproject/metrics/exonerator/QueryResponse.java @@ -21,7 +21,7 @@ public class QueryResponse { private static Logger logger = LoggerFactory.getLogger(QueryResponse.class); /* Actual version implemented by this class. */ - private static final String VERSION = "1.0"; + private static final String VERSION = "1.1"; /* Don't accept query responses with versions lower than this. */ private static final String FIRSTRECOGNIZEDVERSION = "1.0"; @@ -60,6 +60,20 @@ public class QueryResponse { * a day of the requested date; {@code null} if the database is empty. */ Boolean relevantStatuses; + /** Whether there are at least 18 hours of statuses missing in the database + * within a day of the requested date; {@code null} if the database is + * empty. + * + * @since 1.1 */ + Boolean missingStatuses; + + /** Whether there are at least 18 hours of exit lists missing in the database + * from two days before the requested date until one day after; {@code null} + * if the database is empty. + * + * @since 1.1 */ + Boolean missingExitLists; + /** All matches for the given IP address and date; {@code null} if there * were no matches at all. */ Match[] matches; @@ -70,13 +84,16 @@ public class QueryResponse { /** Constructor for tests. */ QueryResponse(String version, String queryAddress, String queryDate, String firstDateInDatabase, String lastDateInDatabase, - Boolean relevantStatuses, Match[] matches, String[] nearbyAddresses) { + Boolean relevantStatuses, Boolean missingStatuses, + Boolean missingExitLists, Match[] matches, String[] nearbyAddresses) { this.version = version; this.queryAddress = queryAddress; this.queryDate = queryDate; this.firstDateInDatabase = firstDateInDatabase; this.lastDateInDatabase = lastDateInDatabase; this.relevantStatuses = relevantStatuses; + this.missingStatuses = missingStatuses; + this.missingExitLists = missingExitLists; this.matches = matches; this.nearbyAddresses = nearbyAddresses; } diff --git a/src/main/java/org/torproject/metrics/exonerator/QueryServlet.java b/src/main/java/org/torproject/metrics/exonerator/QueryServlet.java index 760e385..17980a9 100644 --- a/src/main/java/org/torproject/metrics/exonerator/QueryServlet.java +++ b/src/main/java/org/torproject/metrics/exonerator/QueryServlet.java @@ -54,6 +54,10 @@ public class QueryServlet extends HttpServlet { = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss") .withZone(ZoneOffset.UTC); + private static final int MISSING_STATUSES_IF_LESS_THAN = 3 * 24 - 17; + + private static final int MISSING_EXIT_LISTS_IF_LESS_THAN = 4 * 24 - 17; + @Override public void init() { this.logger = LoggerFactory.getLogger(QueryServlet.class); @@ -264,6 +268,14 @@ public class QueryServlet extends HttpServlet { * {first|last}_date_in_database and relevant_statuses fields. */ SortedSet<LocalDate> allDates = new TreeSet<>(); + /* Store all hours for which the database contains relevant status + * entries. */ + Set<LocalDateTime> allStatusHours = new HashSet<>(); + + /* Store all hours for which the database contains relevant exit list + * entries. */ + Set<LocalDateTime> allExitListHours = new HashSet<>(); + /* Store all possible matches for the results table by base64-encoded * fingerprint and valid-after time. This map is first populated by going * through the result set and adding or updating map entries, so that @@ -309,12 +321,12 @@ public class QueryServlet extends HttpServlet { String orAddress = rs.getString(8); if (null != date) { allDates.add(date); - } else if (null != scanned) { + } else if (null != scanned && null != fingerprintBase64) { exitAddressesByFingeprintBase64AndScanned.putIfAbsent( fingerprintBase64, new TreeMap<>()); exitAddressesByFingeprintBase64AndScanned.get(fingerprintBase64) .put(scanned, exitAddress); - } else if (null != validAfter) { + } else if (null != validAfter && null != fingerprintBase64) { matchesByFingerprintBase64AndValidAfter.putIfAbsent( fingerprintBase64, new TreeMap<>()); if (!matchesByFingerprintBase64AndValidAfter @@ -338,6 +350,10 @@ public class QueryServlet extends HttpServlet { } matchesByAddress.putIfAbsent(orAddress, new HashSet<>()); matchesByAddress.get(orAddress).add(match); + } else if (null != validAfter) { + allStatusHours.add(validAfter); + } else if (null != scanned) { + allExitListHours.add(scanned); } } } catch (SQLException e) { @@ -394,6 +410,10 @@ public class QueryServlet extends HttpServlet { || allDates.contains(timestamp.minusDays(1L)) || allDates.contains(timestamp.plusDays(1L)); } + response.missingStatuses = allStatusHours.size() + < MISSING_STATUSES_IF_LESS_THAN; + response.missingExitLists = allExitListHours.size() + < MISSING_EXIT_LISTS_IF_LESS_THAN; if (matchesByAddress.containsKey(relayIp)) { List<QueryResponse.Match> matchesList = new ArrayList<>(matchesByAddress.get(relayIp)); diff --git a/src/main/resources/ExoneraTor.properties b/src/main/resources/ExoneraTor.properties index 05bb0cb..a67d880 100644 --- a/src/main/resources/ExoneraTor.properties +++ b/src/main/resources/ExoneraTor.properties @@ -30,6 +30,7 @@ summary.positive.title=Result is positive summary.positive.body=We found one or more Tor relays on IP address %s on or within a day of %s that Tor clients were likely to know. summary.negative.title=Result is negative summary.negative.body=We did not find IP address %s on or within a day of %s. +summary.missingdata=However, the database is missing several hours of data for this specific request, so that this result must be interpreted carefully. technicaldetails.heading=Technical details technicaldetails.pre=Looking up IP address %s on or within one day of %s. Tor clients could have selected this or these Tor relays to build circuits. technicaldetails.colheader.timestamp=Timestamp (UTC) diff --git a/src/main/sql/exonerator2.sql b/src/main/sql/exonerator2.sql index 397d2bf..ef017d3 100755 --- a/src/main/sql/exonerator2.sql +++ b/src/main/sql/exonerator2.sql @@ -279,20 +279,24 @@ CREATE OR REPLACE FUNCTION insert_exitlistentry_exitaddress ( END; $$ LANGUAGE 'plpgsql'; --- Search for (1) status entries with an IPv4 or IPv6 onion routing address in --- the same /24 network as the given hex-encoded IP address prefix and with a --- valid-after date within a day of the given date, (2) exit list entries with --- an IPv4 exit address in the same /24 network and with a scan time not earlier --- than two days before and not later than one day after the given date, and (3) --- the last and first dates in the database as well as the dates for which the --- database contains relevant data within a day of the given date. +-- Search by date and /24 IPv4 or IPv6 prefix for: +-- - exit list entries with an IPv4 exit address in the same /24 network and +-- with a scan time not earlier than two days before and not later than one +-- day after the given date, +-- - status entries with an IPv4 or IPv6 onion routing address in the same /24 +-- network as the given hex-encoded IP address prefix and with a valid-after +-- date within a day of the given date, +-- - the first and last dates in the database as well as the dates for which +-- the database contains relevant data within a day of the given date, +-- - the hours for which the database contains relevant exit list entries, and +-- - the hours for which the database contains relevant status entries. -- -- This function makes heavy use of the date_address24 table in order to reduce -- query response time by first obtaining all relevant fingerprint identifiers. --- In the next step it runs three selects to obtain status entries, exit list --- entries, and relevant dates. Any postprocessing, including filtering by exact --- IP address or matching status entries and exit list entries, needs to happen --- at the caller. +-- In the next step it runs five selects to obtain status entries, exit list +-- entries, relevant dates, and relevant hours. Any postprocessing, including +-- filtering by exact IP address or matching status entries and exit list +-- entries, needs to happen at the caller. CREATE OR REPLACE FUNCTION search_by_date_address24 ( search_date DATE, search_address24 CHARACTER(6)) RETURNS TABLE( @@ -339,7 +343,21 @@ CREATE OR REPLACE FUNCTION search_by_date_address24 ( WHERE date IN (SELECT MIN(date) FROM date_address24 UNION SELECT MAX(date) FROM date_address24 UNION SELECT date FROM date_address24 - WHERE date >= $1 - 1 AND date <= $1 + 1)' + WHERE date >= $1 - 1 AND date <= $1 + 1) + UNION + SELECT NULL AS date, NULL AS fingerprint_base64, + DATE_TRUNC(''hour'', scanned) AS scanned, NULL AS exitaddress, + NULL AS validafter, NULL AS nickname, NULL AS exit, NULL AS oraddress + FROM exitlistentry_exitaddress + WHERE DATE(exitlistentry_exitaddress.scanned) >= $1 - 2 + AND DATE(exitlistentry_exitaddress.scanned) <= $1 + 1 + UNION + SELECT NULL AS date, NULL AS fingerprint_base64, NULL AS scanned, + NULL AS exitaddress, DATE_TRUNC(''hour'', validafter) AS validafter, + NULL AS nickname, NULL AS exit, NULL AS oraddress + FROM statusentry_oraddress + WHERE DATE(statusentry_oraddress.validafter) >= $1 - 1 + AND DATE(statusentry_oraddress.validafter) <= $1 + 1' USING search_date, search_address24; END; $$ LANGUAGE plpgsql; diff --git a/src/test/java/org/torproject/metrics/exonerator/ExoneraTorServletTest.java b/src/test/java/org/torproject/metrics/exonerator/ExoneraTorServletTest.java index 1ede590..64656da 100644 --- a/src/test/java/org/torproject/metrics/exonerator/ExoneraTorServletTest.java +++ b/src/test/java/org/torproject/metrics/exonerator/ExoneraTorServletTest.java @@ -50,7 +50,7 @@ public class ExoneraTorServletTest { StringWriter sw = new StringWriter(); es.writeSummaryAddressesInSameNetwork(new PrintWriter(sw), rb, "http://localhost:8080/", qr.queryAddress, qr.queryDate, "en", - Arrays.asList(qr.nearbyAddresses)); + Arrays.asList(qr.nearbyAddresses), false); String errorMsg = "Test data:" + QueryResponse.toJson(qr) + "\nresult:\n" + sw.toString(); assertTrue(errorMsg, diff --git a/src/test/java/org/torproject/metrics/exonerator/QueryResponseTest.java b/src/test/java/org/torproject/metrics/exonerator/QueryResponseTest.java index 97ae88d..826da1c 100644 --- a/src/test/java/org/torproject/metrics/exonerator/QueryResponseTest.java +++ b/src/test/java/org/torproject/metrics/exonerator/QueryResponseTest.java @@ -42,10 +42,12 @@ public class QueryResponseTest { + "\"exit\":false}],\"nearby_addresses\":[\"12.13.14.15\"," + "\"12.13.14.16\"]}"}, {new QueryResponse("1.1", null, null, null, - null, false, null, null), - "{\"version\":\"1.1\",\"relevant_statuses\":false}"}, + null, false, true, true, null, null), + "{\"version\":\"1.1\",\"relevant_statuses\":false," + + "\"missing_statuses\":true," + + "\"missing_exit_lists\":true}"}, {new QueryResponse("1.0", "12.13.14.15", "2016-12-12", "2016-01-01", - "2016-12-31", true, + "2016-12-31", true, false, false, new QueryResponse.Match[]{new QueryResponse.Match("2016-12-03", new TreeSet<>(Arrays.asList("12.13.14.15", "12.13.14.16")), "fingerprint-not-checked", "some name", true), @@ -59,6 +61,8 @@ public class QueryResponseTest { + "\"first_date_in_database\":\"2016-01-01\"," + "\"last_date_in_database\":\"2016-12-31\"," + "\"relevant_statuses\":true," + + "\"missing_statuses\":false," + + "\"missing_exit_lists\":false," + "\"matches\":[{\"timestamp\":\"2016-12-03\"," + "\"addresses\":[\"12.13.14.15\"," + "\"12.13.14.16\"],\"fingerprint\":\"fingerprint-not-checked\"," @@ -70,7 +74,7 @@ public class QueryResponseTest { + "\"exit\":false}],\"nearby_addresses\":[\"12.13.14.15\"," + "\"12.13.14.16\"]}"}, {new QueryResponse("1.0", "12.13.14.15", "2016-12-12", "2016-01-01", - "2016-12-31", false, + "2016-12-31", false, null, null, new QueryResponse.Match[]{new QueryResponse.Match("2016-12-03", new TreeSet<>(Arrays.asList("12.13.14.15", "12.13.14.16")), "fingerprint-not-checked", "some name", null),

1 0

[exonerator/release] Prepare for 4.2.0 release.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 5bf62c28764083e5fa3b7a3d4368e93c99ea0c51 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sat Nov 9 21:28:37 2019 +0100 Prepare for 4.2.0 release. --- CHANGELOG.md | 3 ++- build.xml | 4 ++-- src/build | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ddbae79..632f622 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 4.2.0 - 2019-0?-?? +# Changes in version 4.2.0 - 2019-11-09 * Medium changes - Use Ivy for resolving external dependencies rather than relying @@ -12,6 +12,7 @@ few exceptions. - Remove Cobertura from the build process. - Update PostgreSQL JDBC driver version to 42.2.5. + - Update to metrics-lib 2.9.1. # Changes in version 4.1.0 - 2019-05-13 diff --git a/build.xml b/build.xml index 9d522fe..38fee16 100644 --- a/build.xml +++ b/build.xml @@ -9,8 +9,8 @@ <property name="javadoc-title" value="ExoneraTor API Documentation"/> <property name="implementation-title" value="ExoneraTor" /> - <property name="release.version" value="4.1.0-dev" /> - <property name="metricslibversion" value="2.6.2" /> + <property name="release.version" value="4.2.0" /> + <property name="metricslibversion" value="2.9.1" /> <property name="jetty.version" value="-9.2.21.v20170120" /> <property name="warfile" value="exonerator-${release.version}.war"/> diff --git a/src/build b/src/build index 62a964a..eb16cb3 160000 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 62a964a7702086ca6ba0e2307809d4f983163b47 +Subproject commit eb16cb359db41722e6089bafb1e26808df4338df

1 0

[exonerator/release] Revert "Print out a warning if we're missing data."
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit a1fc853ce9889e404934b1cc942cc793240bec14 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sat Nov 9 21:25:27 2019 +0100 Revert "Print out a warning if we're missing data." This reverts commit b0165cb03138e933a337ddc8400c3a8bbe889b3d. Turns out that worst case query time goes up from 3 to 58 seconds with the new query, which is unacceptable. We'll have to analyze and improve the query on the production database before merging this change. --- .../metrics/exonerator/ExoneraTorServlet.java | 43 ++++++---------------- .../metrics/exonerator/QueryResponse.java | 21 +---------- .../metrics/exonerator/QueryServlet.java | 24 +----------- src/main/resources/ExoneraTor.properties | 1 - src/main/sql/exonerator2.sql | 42 ++++++--------------- .../metrics/exonerator/ExoneraTorServletTest.java | 2 +- .../metrics/exonerator/QueryResponseTest.java | 12 ++---- 7 files changed, 33 insertions(+), 112 deletions(-) diff --git a/src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java b/src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java index 4b0cf16..a74aa60 100644 --- a/src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java +++ b/src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java @@ -89,7 +89,6 @@ public class ExoneraTorServlet extends HttpServlet { ExoneraTorDate firstDate = ExoneraTorDate.INVALID; ExoneraTorDate lastDate = ExoneraTorDate.INVALID; boolean noRelevantConsensuses = true; - boolean missingData = false; List<String[]> statusEntries = new ArrayList<>(); List<String> addressesInSameNetwork = null; @@ -106,14 +105,6 @@ public class ExoneraTorServlet extends HttpServlet { && queryResponse.relevantStatuses) { noRelevantConsensuses = false; } - if (null != queryResponse.missingStatuses - && queryResponse.missingStatuses) { - missingData = true; - } - if (null != queryResponse.missingExitLists - && queryResponse.missingExitLists) { - missingData = true; - } if (null != queryResponse.matches) { for (QueryResponse.Match match : queryResponse.matches) { StringBuilder sb = new StringBuilder(); @@ -219,18 +210,15 @@ public class ExoneraTorServlet extends HttpServlet { /* Print out result. */ } else { if (!statusEntries.isEmpty()) { - this.writeSummaryPositive(out, rb, relayIp, requestedDate.asString, - missingData); + this.writeSummaryPositive(out, rb, relayIp, requestedDate.asString); this.writeTechnicalDetails(out, rb, relayIp, requestedDate.asString, statusEntries); } else if (addressesInSameNetwork != null && !addressesInSameNetwork.isEmpty()) { this.writeSummaryAddressesInSameNetwork(out, rb, requestUrl, relayIp, - requestedDate.asString, langStr, addressesInSameNetwork, - missingData); + requestedDate.asString, langStr, addressesInSameNetwork); } else { - this.writeSummaryNegative(out, rb, relayIp, requestedDate.asString, - missingData); + this.writeSummaryNegative(out, rb, relayIp, requestedDate.asString); } this.writePermanentLink(out, rb, requestUrl, relayIp, requestedDate.asString, langStr); @@ -410,13 +398,13 @@ public class ExoneraTorServlet extends HttpServlet { private void writeSummaryNoTimestamp(PrintWriter out, ResourceBundle rb) { this.writeSummary(out, rb.getString("summary.heading"), "panel-danger", - rb.getString("summary.invalidparams.notimestamp.title"), null, null, + rb.getString("summary.invalidparams.notimestamp.title"), null, rb.getString("summary.invalidparams.notimestamp.body")); } private void writeSummaryNoIp(PrintWriter out, ResourceBundle rb) { this.writeSummary(out, rb.getString("summary.heading"), - "panel-danger", rb.getString("summary.invalidparams.noip.title"), null, + "panel-danger", rb.getString("summary.invalidparams.noip.title"), null, rb.getString("summary.invalidparams.noip.body")); } @@ -425,7 +413,7 @@ public class ExoneraTorServlet extends HttpServlet { String lastDate) { this.writeSummary(out, rb.getString("summary.heading"), "panel-danger", - rb.getString("summary.invalidparams.timestamprange.title"), null, null, + rb.getString("summary.invalidparams.timestamprange.title"), null, rb.getString("summary.invalidparams.timestamprange.body"), timestampStr, firstDate, lastDate); } @@ -450,7 +438,7 @@ public class ExoneraTorServlet extends HttpServlet { : StringEscapeUtils.escapeHtml4(timestampParameter); this.writeSummary(out, rb.getString("summary.heading"), "panel-danger", - rb.getString("summary.invalidparams.invalidtimestamp.title"), null, + rb.getString("summary.invalidparams.invalidtimestamp.title"), null, rb.getString("summary.invalidparams.invalidtimestamp.body"), escapedTimestampParameter, "\"YYYY-MM-DD\""); } @@ -458,7 +446,7 @@ public class ExoneraTorServlet extends HttpServlet { private void writeSummaryTimestampTooRecent(PrintWriter out, ResourceBundle rb) { this.writeSummary(out, rb.getString("summary.heading"), "panel-danger", - rb.getString("summary.invalidparams.timestamptoorecent.title"), null, + rb.getString("summary.invalidparams.timestamptoorecent.title"), null, rb.getString("summary.invalidparams.timestamptoorecent.body")); } @@ -477,8 +465,7 @@ public class ExoneraTorServlet extends HttpServlet { void writeSummaryAddressesInSameNetwork(PrintWriter out, ResourceBundle rb, String requestUrl, String relayIp, String timestampStr, - String langStr, List<String> addressesInSameNetwork, - boolean missingData) { + String langStr, List<String> addressesInSameNetwork) { Object[][] panelItems = new Object[addressesInSameNetwork.size()][]; for (int i = 0; i < addressesInSameNetwork.size(); i++) { String addressInSameNetwork = addressesInSameNetwork.get(i); @@ -499,36 +486,33 @@ public class ExoneraTorServlet extends HttpServlet { this.writeSummary(out, rb.getString("summary.heading"), "panel-warning", rb.getString("summary.negativesamenetwork.title"), panelItems, - missingData ? rb.getString("summary.missingdata") : null, rb.getString("summary.negativesamenetwork.body"), relayIp, timestampStr, relayIp.contains(":") ? 48 : 24); } private void writeSummaryPositive(PrintWriter out, ResourceBundle rb, - String relayIp, String timestampStr, boolean missingData) { + String relayIp, String timestampStr) { String formattedRelayIp = relayIp.contains(":") ? "[" + relayIp + "]" : relayIp; this.writeSummary(out, rb.getString("summary.heading"), "panel-success", rb.getString("summary.positive.title"), null, - missingData ? rb.getString("summary.missingdata") : null, rb.getString("summary.positive.body"), formattedRelayIp, timestampStr); } private void writeSummaryNegative(PrintWriter out, ResourceBundle rb, - String relayIp, String timestampStr, boolean missingData) { + String relayIp, String timestampStr) { String formattedRelayIp = relayIp.contains(":") ? "[" + relayIp + "]" : relayIp; this.writeSummary(out, rb.getString("summary.heading"), "panel-warning", rb.getString("summary.negative.title"), null, - missingData ? rb.getString("summary.missingdata") : null, rb.getString("summary.negative.body"), formattedRelayIp, timestampStr); } private void writeSummary(PrintWriter out, String heading, String panelContext, String panelTitle, Object[][] panelItems, - String panelWarning, String panelBodyTemplate, Object... panelBodyArgs) { + String panelBodyTemplate, Object... panelBodyArgs) { out.printf(" <div class=\"row\">\n" + " <div class=\"col-xs-12\">\n" + " <h2>%s</h2>\n" @@ -547,9 +531,6 @@ public class ExoneraTorServlet extends HttpServlet { } out.print(" </ul>\n"); } - if (null != panelWarning) { - out.printf(" <p>%s</p>\n", panelWarning); - } out.print(" </div>\n" + " </div>\n" + " </div>\n" diff --git a/src/main/java/org/torproject/metrics/exonerator/QueryResponse.java b/src/main/java/org/torproject/metrics/exonerator/QueryResponse.java index d148d0c..6a8976a 100644 --- a/src/main/java/org/torproject/metrics/exonerator/QueryResponse.java +++ b/src/main/java/org/torproject/metrics/exonerator/QueryResponse.java @@ -21,7 +21,7 @@ public class QueryResponse { private static Logger logger = LoggerFactory.getLogger(QueryResponse.class); /* Actual version implemented by this class. */ - private static final String VERSION = "1.1"; + private static final String VERSION = "1.0"; /* Don't accept query responses with versions lower than this. */ private static final String FIRSTRECOGNIZEDVERSION = "1.0"; @@ -60,20 +60,6 @@ public class QueryResponse { * a day of the requested date; {@code null} if the database is empty. */ Boolean relevantStatuses; - /** Whether there are at least 18 hours of statuses missing in the database - * within a day of the requested date; {@code null} if the database is - * empty. - * - * @since 1.1 */ - Boolean missingStatuses; - - /** Whether there are at least 18 hours of exit lists missing in the database - * from two days before the requested date until one day after; {@code null} - * if the database is empty. - * - * @since 1.1 */ - Boolean missingExitLists; - /** All matches for the given IP address and date; {@code null} if there * were no matches at all. */ Match[] matches; @@ -84,16 +70,13 @@ public class QueryResponse { /** Constructor for tests. */ QueryResponse(String version, String queryAddress, String queryDate, String firstDateInDatabase, String lastDateInDatabase, - Boolean relevantStatuses, Boolean missingStatuses, - Boolean missingExitLists, Match[] matches, String[] nearbyAddresses) { + Boolean relevantStatuses, Match[] matches, String[] nearbyAddresses) { this.version = version; this.queryAddress = queryAddress; this.queryDate = queryDate; this.firstDateInDatabase = firstDateInDatabase; this.lastDateInDatabase = lastDateInDatabase; this.relevantStatuses = relevantStatuses; - this.missingStatuses = missingStatuses; - this.missingExitLists = missingExitLists; this.matches = matches; this.nearbyAddresses = nearbyAddresses; } diff --git a/src/main/java/org/torproject/metrics/exonerator/QueryServlet.java b/src/main/java/org/torproject/metrics/exonerator/QueryServlet.java index 17980a9..760e385 100644 --- a/src/main/java/org/torproject/metrics/exonerator/QueryServlet.java +++ b/src/main/java/org/torproject/metrics/exonerator/QueryServlet.java @@ -54,10 +54,6 @@ public class QueryServlet extends HttpServlet { = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss") .withZone(ZoneOffset.UTC); - private static final int MISSING_STATUSES_IF_LESS_THAN = 3 * 24 - 17; - - private static final int MISSING_EXIT_LISTS_IF_LESS_THAN = 4 * 24 - 17; - @Override public void init() { this.logger = LoggerFactory.getLogger(QueryServlet.class); @@ -268,14 +264,6 @@ public class QueryServlet extends HttpServlet { * {first|last}_date_in_database and relevant_statuses fields. */ SortedSet<LocalDate> allDates = new TreeSet<>(); - /* Store all hours for which the database contains relevant status - * entries. */ - Set<LocalDateTime> allStatusHours = new HashSet<>(); - - /* Store all hours for which the database contains relevant exit list - * entries. */ - Set<LocalDateTime> allExitListHours = new HashSet<>(); - /* Store all possible matches for the results table by base64-encoded * fingerprint and valid-after time. This map is first populated by going * through the result set and adding or updating map entries, so that @@ -321,12 +309,12 @@ public class QueryServlet extends HttpServlet { String orAddress = rs.getString(8); if (null != date) { allDates.add(date); - } else if (null != scanned && null != fingerprintBase64) { + } else if (null != scanned) { exitAddressesByFingeprintBase64AndScanned.putIfAbsent( fingerprintBase64, new TreeMap<>()); exitAddressesByFingeprintBase64AndScanned.get(fingerprintBase64) .put(scanned, exitAddress); - } else if (null != validAfter && null != fingerprintBase64) { + } else if (null != validAfter) { matchesByFingerprintBase64AndValidAfter.putIfAbsent( fingerprintBase64, new TreeMap<>()); if (!matchesByFingerprintBase64AndValidAfter @@ -350,10 +338,6 @@ public class QueryServlet extends HttpServlet { } matchesByAddress.putIfAbsent(orAddress, new HashSet<>()); matchesByAddress.get(orAddress).add(match); - } else if (null != validAfter) { - allStatusHours.add(validAfter); - } else if (null != scanned) { - allExitListHours.add(scanned); } } } catch (SQLException e) { @@ -410,10 +394,6 @@ public class QueryServlet extends HttpServlet { || allDates.contains(timestamp.minusDays(1L)) || allDates.contains(timestamp.plusDays(1L)); } - response.missingStatuses = allStatusHours.size() - < MISSING_STATUSES_IF_LESS_THAN; - response.missingExitLists = allExitListHours.size() - < MISSING_EXIT_LISTS_IF_LESS_THAN; if (matchesByAddress.containsKey(relayIp)) { List<QueryResponse.Match> matchesList = new ArrayList<>(matchesByAddress.get(relayIp)); diff --git a/src/main/resources/ExoneraTor.properties b/src/main/resources/ExoneraTor.properties index a67d880..05bb0cb 100644 --- a/src/main/resources/ExoneraTor.properties +++ b/src/main/resources/ExoneraTor.properties @@ -30,7 +30,6 @@ summary.positive.title=Result is positive summary.positive.body=We found one or more Tor relays on IP address %s on or within a day of %s that Tor clients were likely to know. summary.negative.title=Result is negative summary.negative.body=We did not find IP address %s on or within a day of %s. -summary.missingdata=However, the database is missing several hours of data for this specific request, so that this result must be interpreted carefully. technicaldetails.heading=Technical details technicaldetails.pre=Looking up IP address %s on or within one day of %s. Tor clients could have selected this or these Tor relays to build circuits. technicaldetails.colheader.timestamp=Timestamp (UTC) diff --git a/src/main/sql/exonerator2.sql b/src/main/sql/exonerator2.sql index ef017d3..397d2bf 100755 --- a/src/main/sql/exonerator2.sql +++ b/src/main/sql/exonerator2.sql @@ -279,24 +279,20 @@ CREATE OR REPLACE FUNCTION insert_exitlistentry_exitaddress ( END; $$ LANGUAGE 'plpgsql'; --- Search by date and /24 IPv4 or IPv6 prefix for: --- - exit list entries with an IPv4 exit address in the same /24 network and --- with a scan time not earlier than two days before and not later than one --- day after the given date, --- - status entries with an IPv4 or IPv6 onion routing address in the same /24 --- network as the given hex-encoded IP address prefix and with a valid-after --- date within a day of the given date, --- - the first and last dates in the database as well as the dates for which --- the database contains relevant data within a day of the given date, --- - the hours for which the database contains relevant exit list entries, and --- - the hours for which the database contains relevant status entries. +-- Search for (1) status entries with an IPv4 or IPv6 onion routing address in +-- the same /24 network as the given hex-encoded IP address prefix and with a +-- valid-after date within a day of the given date, (2) exit list entries with +-- an IPv4 exit address in the same /24 network and with a scan time not earlier +-- than two days before and not later than one day after the given date, and (3) +-- the last and first dates in the database as well as the dates for which the +-- database contains relevant data within a day of the given date. -- -- This function makes heavy use of the date_address24 table in order to reduce -- query response time by first obtaining all relevant fingerprint identifiers. --- In the next step it runs five selects to obtain status entries, exit list --- entries, relevant dates, and relevant hours. Any postprocessing, including --- filtering by exact IP address or matching status entries and exit list --- entries, needs to happen at the caller. +-- In the next step it runs three selects to obtain status entries, exit list +-- entries, and relevant dates. Any postprocessing, including filtering by exact +-- IP address or matching status entries and exit list entries, needs to happen +-- at the caller. CREATE OR REPLACE FUNCTION search_by_date_address24 ( search_date DATE, search_address24 CHARACTER(6)) RETURNS TABLE( @@ -343,21 +339,7 @@ CREATE OR REPLACE FUNCTION search_by_date_address24 ( WHERE date IN (SELECT MIN(date) FROM date_address24 UNION SELECT MAX(date) FROM date_address24 UNION SELECT date FROM date_address24 - WHERE date >= $1 - 1 AND date <= $1 + 1) - UNION - SELECT NULL AS date, NULL AS fingerprint_base64, - DATE_TRUNC(''hour'', scanned) AS scanned, NULL AS exitaddress, - NULL AS validafter, NULL AS nickname, NULL AS exit, NULL AS oraddress - FROM exitlistentry_exitaddress - WHERE DATE(exitlistentry_exitaddress.scanned) >= $1 - 2 - AND DATE(exitlistentry_exitaddress.scanned) <= $1 + 1 - UNION - SELECT NULL AS date, NULL AS fingerprint_base64, NULL AS scanned, - NULL AS exitaddress, DATE_TRUNC(''hour'', validafter) AS validafter, - NULL AS nickname, NULL AS exit, NULL AS oraddress - FROM statusentry_oraddress - WHERE DATE(statusentry_oraddress.validafter) >= $1 - 1 - AND DATE(statusentry_oraddress.validafter) <= $1 + 1' + WHERE date >= $1 - 1 AND date <= $1 + 1)' USING search_date, search_address24; END; $$ LANGUAGE plpgsql; diff --git a/src/test/java/org/torproject/metrics/exonerator/ExoneraTorServletTest.java b/src/test/java/org/torproject/metrics/exonerator/ExoneraTorServletTest.java index 64656da..1ede590 100644 --- a/src/test/java/org/torproject/metrics/exonerator/ExoneraTorServletTest.java +++ b/src/test/java/org/torproject/metrics/exonerator/ExoneraTorServletTest.java @@ -50,7 +50,7 @@ public class ExoneraTorServletTest { StringWriter sw = new StringWriter(); es.writeSummaryAddressesInSameNetwork(new PrintWriter(sw), rb, "http://localhost:8080/", qr.queryAddress, qr.queryDate, "en", - Arrays.asList(qr.nearbyAddresses), false); + Arrays.asList(qr.nearbyAddresses)); String errorMsg = "Test data:" + QueryResponse.toJson(qr) + "\nresult:\n" + sw.toString(); assertTrue(errorMsg, diff --git a/src/test/java/org/torproject/metrics/exonerator/QueryResponseTest.java b/src/test/java/org/torproject/metrics/exonerator/QueryResponseTest.java index 826da1c..97ae88d 100644 --- a/src/test/java/org/torproject/metrics/exonerator/QueryResponseTest.java +++ b/src/test/java/org/torproject/metrics/exonerator/QueryResponseTest.java @@ -42,12 +42,10 @@ public class QueryResponseTest { + "\"exit\":false}],\"nearby_addresses\":[\"12.13.14.15\"," + "\"12.13.14.16\"]}"}, {new QueryResponse("1.1", null, null, null, - null, false, true, true, null, null), - "{\"version\":\"1.1\",\"relevant_statuses\":false," - + "\"missing_statuses\":true," - + "\"missing_exit_lists\":true}"}, + null, false, null, null), + "{\"version\":\"1.1\",\"relevant_statuses\":false}"}, {new QueryResponse("1.0", "12.13.14.15", "2016-12-12", "2016-01-01", - "2016-12-31", true, false, false, + "2016-12-31", true, new QueryResponse.Match[]{new QueryResponse.Match("2016-12-03", new TreeSet<>(Arrays.asList("12.13.14.15", "12.13.14.16")), "fingerprint-not-checked", "some name", true), @@ -61,8 +59,6 @@ public class QueryResponseTest { + "\"first_date_in_database\":\"2016-01-01\"," + "\"last_date_in_database\":\"2016-12-31\"," + "\"relevant_statuses\":true," - + "\"missing_statuses\":false," - + "\"missing_exit_lists\":false," + "\"matches\":[{\"timestamp\":\"2016-12-03\"," + "\"addresses\":[\"12.13.14.15\"," + "\"12.13.14.16\"],\"fingerprint\":\"fingerprint-not-checked\"," @@ -74,7 +70,7 @@ public class QueryResponseTest { + "\"exit\":false}],\"nearby_addresses\":[\"12.13.14.15\"," + "\"12.13.14.16\"]}"}, {new QueryResponse("1.0", "12.13.14.15", "2016-12-12", "2016-01-01", - "2016-12-31", false, null, null, + "2016-12-31", false, new QueryResponse.Match[]{new QueryResponse.Match("2016-12-03", new TreeSet<>(Arrays.asList("12.13.14.15", "12.13.14.16")), "fingerprint-not-checked", "some name", null),

1 0