February 2018 - tor-commits - lists.torproject.org

[metrics-lib/master] Reduce memory footprint of log lines.
by karsten＠torproject.org 05 Feb '18

05 Feb '18

commit 8578de6d64569b410678b3c847e7b82b267501ad Author: iwakeh <iwakeh(a)torproject.org> Date: Wed Jan 31 12:35:31 2018 +0000 Reduce memory footprint of log lines. Also make validation optional for internal web log constructor. CollecTor uses this for storing freshly sanitized logs, which don't need to be validated a second time. --- .../descriptor/log/WebServerAccessLogImpl.java | 19 +++++++-- .../descriptor/log/WebServerAccessLogLine.java | 46 ++++++++++++++++------ 2 files changed, 49 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/torproject/descriptor/log/WebServerAccessLogImpl.java b/src/main/java/org/torproject/descriptor/log/WebServerAccessLogImpl.java index 6708c3a..f02b1d7 100644 --- a/src/main/java/org/torproject/descriptor/log/WebServerAccessLogImpl.java +++ b/src/main/java/org/torproject/descriptor/log/WebServerAccessLogImpl.java @@ -44,6 +44,8 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl private final LocalDate logDate; + private boolean validate = true; + /** * Creates a WebServerAccessLog from the given bytes and filename. * @@ -65,13 +67,20 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl } /** For internal use only. */ - public WebServerAccessLogImpl(Collection<String> lines, String filename) - throws DescriptorParseException { - this(LogDescriptorImpl.collectionToBytes(lines), new File(filename)); + public WebServerAccessLogImpl(Collection<String> lines, String filename, + boolean validate) throws DescriptorParseException { + this(LogDescriptorImpl.collectionToBytes(lines), new File(filename), + FileType.XZ, validate); } private WebServerAccessLogImpl(byte[] logBytes, File file, FileType defaultCompression) throws DescriptorParseException { + this(logBytes, file, defaultCompression, true); + } + + private WebServerAccessLogImpl(byte[] logBytes, File file, + FileType defaultCompression, boolean validate) + throws DescriptorParseException { super(logBytes, file, defaultCompression); try { String fn = file.toPath().getFileName().toString(); @@ -91,7 +100,9 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl this.logDate = LocalDate.parse(ymd, DateTimeFormatter.BASIC_ISO_DATE); this.setValidator((line) -> WebServerAccessLogLine.makeLine(line).isValid()); - this.validate(); + if (validate) { + this.validate(); + } } catch (DescriptorParseException dpe) { throw dpe; // escalate } catch (Exception pe) { diff --git a/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java b/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java index fdbf5c1..2f27441 100644 --- a/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java +++ b/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java @@ -10,6 +10,9 @@ import java.time.LocalDate; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.Optional; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -31,15 +34,23 @@ public class WebServerAccessLogLine { + "\"([A-Z]+) ([^\"]+) ([A-Z]+/\\d\\.\\d)\" " + "(\\d{3}) (\\d+|-)(.*)"); + private static Map<String, String> ipMap + = Collections.synchronizedMap(new HashMap<>()); + private static Map<LocalDate, LocalDate> dateMap + = Collections.synchronizedMap(new HashMap<>()); + private static Map<String, String> protocolMap + = Collections.synchronizedMap(new HashMap<>()); + private static Map<String, String> requestMap + = Collections.synchronizedMap(new HashMap<>()); + private String ip; private int response; private String request; private Method method; private LocalDate date; - private String protocol; private int size = -1; private boolean valid = false; - private String type; + private String protocol; /** Returns a log line string. Possibly empty. */ public String toLogString() { @@ -53,10 +64,11 @@ public class WebServerAccessLogLine { public String toString() { return String.format("%s - - [%s:00:00:00 +0000] \"%s %s %s\" %d %s", this.ip, this.getDateString(), this.method.name(), this.request, - this.type, this.response, this.size < 0 ? DASH : this.size); + this.protocol, this.response, this.size < 0 ? DASH : this.size); } - /** Returns the string of the date using 'yyyymmdd' format. */ + /** Only used internally during sanitization. + * Returns the string of the date using 'dd/MMM/yyyy' format. */ public String getDateString() { return this.date.format(DateTimeFormatter.ofPattern(DATE_PATTERN)); } @@ -68,7 +80,7 @@ public class WebServerAccessLogLine { /** Only used internally during sanitization. */ public void setIp(String ip) { - this.ip = ip; + this.ip = fromMap(ip, ipMap); } public Method getMethod() { @@ -93,7 +105,7 @@ public class WebServerAccessLogLine { /** Only used internally during sanitization. */ public void setRequest(String request) { - this.request = request; + this.request = fromMap(request, requestMap); } public LocalDate getDate() { @@ -112,14 +124,13 @@ public class WebServerAccessLogLine { if (mat.find()) { res.response = Integer.valueOf(mat.group(10)); res.method = Method.valueOf(mat.group(7)); - res.protocol = mat.group(9); String dateTimeString = mat.group(4) + mat.group(5) + mat.group(6); - res.date = ZonedDateTime.parse(dateTimeString, + res.date = fromMap(ZonedDateTime.parse(dateTimeString, dateTimeFormatter).withZoneSameInstant(ZoneOffset.UTC) - .toLocalDate(); - res.ip = mat.group(1); - res.request = mat.group(8); - res.type = mat.group(9); + .toLocalDate(), dateMap); + res.ip = fromMap(mat.group(1), ipMap); + res.request = fromMap(mat.group(8), requestMap); + res.protocol = fromMap(mat.group(9), protocolMap); if (DASH.equals(mat.group(11))) { res.size = -1; } else { @@ -134,5 +145,16 @@ public class WebServerAccessLogLine { return res; } + private static <T> T fromMap(T val, Map<T, T> map) { + synchronized (map) { + T reference = map.get(val); + if (null == reference) { + map.put(val, val); + reference = map.get(val); + } + return reference; + } + } + }

1 0

[metrics-lib/master] Use enum for request method.
by karsten＠torproject.org 05 Feb '18

05 Feb '18

commit 86c47d3764b1d563bb81eb683da39c3ca31c13ba Author: iwakeh <iwakeh(a)torproject.org> Date: Wed Jan 31 12:35:28 2018 +0000 Use enum for request method. --- src/main/java/org/torproject/descriptor/log/Method.java | 8 ++++++++ .../org/torproject/descriptor/log/WebServerAccessLogLine.java | 11 ++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/torproject/descriptor/log/Method.java b/src/main/java/org/torproject/descriptor/log/Method.java new file mode 100644 index 0000000..c29d495 --- /dev/null +++ b/src/main/java/org/torproject/descriptor/log/Method.java @@ -0,0 +1,8 @@ +/* Copyright 2018 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.descriptor.log; + +public enum Method { + GET, HEAD, POST; +} diff --git a/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java b/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java index 4eda73b..fdbf5c1 100644 --- a/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java +++ b/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java @@ -34,7 +34,7 @@ public class WebServerAccessLogLine { private String ip; private int response; private String request; - private String method; + private Method method; private LocalDate date; private String protocol; private int size = -1; @@ -52,8 +52,8 @@ public class WebServerAccessLogLine { @Override public String toString() { return String.format("%s - - [%s:00:00:00 +0000] \"%s %s %s\" %d %s", - this.ip, this.getDateString(), this.method, this.request, this.type, - this.response, this.size < 0 ? DASH : this.size); + this.ip, this.getDateString(), this.method.name(), this.request, + this.type, this.response, this.size < 0 ? DASH : this.size); } /** Returns the string of the date using 'yyyymmdd' format. */ @@ -71,7 +71,7 @@ public class WebServerAccessLogLine { this.ip = ip; } - public String getMethod() { + public Method getMethod() { return this.method; } @@ -111,7 +111,7 @@ public class WebServerAccessLogLine { Matcher mat = logLinePattern.matcher(line); if (mat.find()) { res.response = Integer.valueOf(mat.group(10)); - res.method = mat.group(7); + res.method = Method.valueOf(mat.group(7)); res.protocol = mat.group(9); String dateTimeString = mat.group(4) + mat.group(5) + mat.group(6); res.date = ZonedDateTime.parse(dateTimeString, @@ -135,3 +135,4 @@ public class WebServerAccessLogLine { } } +

1 0

[metrics-lib/master] Provide 'equals' and 'hashCode' implementations.
by karsten＠torproject.org 05 Feb '18

05 Feb '18

commit dc019152886f018f39ecbcba24c3490e2eed8057 Author: iwakeh <iwakeh(a)torproject.org> Date: Wed Jan 31 12:35:36 2018 +0000 Provide 'equals' and 'hashCode' implementations. --- .../torproject/descriptor/log/WebServerAccessLogLine.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java b/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java index 2f27441..a4e3ce6 100644 --- a/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java +++ b/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java @@ -156,5 +156,19 @@ public class WebServerAccessLogLine { } } + @Override + public boolean equals(Object other) { + if (null != other && other instanceof WebServerAccessLogLine) { + return this.toLogString() + .equals(((WebServerAccessLogLine)other).toLogString()); + } + return false; + } + + @Override + public int hashCode() { + return this.toLogString().hashCode(); + } + }

1 0

[metrics-lib/master] Store size as simple int.
by karsten＠torproject.org 05 Feb '18

05 Feb '18

commit bbae5617335d19b2fe1244999da1ddd4dc5b214b Author: iwakeh <iwakeh(a)torproject.org> Date: Wed Jan 31 12:35:27 2018 +0000 Store size as simple int. --- .../torproject/descriptor/log/WebServerAccessLogLine.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java b/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java index 09cd06a..4eda73b 100644 --- a/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java +++ b/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java @@ -20,6 +20,7 @@ public class WebServerAccessLogLine { .getLogger(WebServerAccessLogLine.class); private static final String DATE_PATTERN = "dd/MMM/yyyy"; + private static final String DASH = "-"; private static final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(DATE_PATTERN + ":HH:mm:ss xxxx"); @@ -36,7 +37,7 @@ public class WebServerAccessLogLine { private String method; private LocalDate date; private String protocol; - private Optional<Integer> size; + private int size = -1; private boolean valid = false; private String type; @@ -52,7 +53,7 @@ public class WebServerAccessLogLine { public String toString() { return String.format("%s - - [%s:00:00:00 +0000] \"%s %s %s\" %d %s", this.ip, this.getDateString(), this.method, this.request, this.type, - this.response, this.size.isPresent() ? this.size.get() : "-"); + this.response, this.size < 0 ? DASH : this.size); } /** Returns the string of the date using 'yyyymmdd' format. */ @@ -82,6 +83,10 @@ public class WebServerAccessLogLine { return this.request; } + public Optional<Integer> getSize() { + return this.size < 0 ? Optional.empty() : Optional.of(this.size); + } + public int getResponse() { return this.response; } @@ -115,10 +120,10 @@ public class WebServerAccessLogLine { res.ip = mat.group(1); res.request = mat.group(8); res.type = mat.group(9); - if ("-".equals(mat.group(11))) { - res.size = Optional.empty(); + if (DASH.equals(mat.group(11))) { + res.size = -1; } else { - res.size = Optional.of(Integer.valueOf(mat.group(11))); + res.size = Integer.valueOf(mat.group(11)); } res.valid = true; }

1 0

[metrics-lib/master] Use 'putIfAbsent'.
by karsten＠torproject.org 05 Feb '18

05 Feb '18

commit 3efd91716902e7592304e78bd78ab95d961869ff Author: iwakeh <iwakeh(a)torproject.org> Date: Sat Feb 3 08:19:55 2018 +0000 Use 'putIfAbsent'. --- .../org/torproject/descriptor/log/WebServerAccessLogLine.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java b/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java index a4e3ce6..4884733 100644 --- a/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java +++ b/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java @@ -147,12 +147,8 @@ public class WebServerAccessLogLine { private static <T> T fromMap(T val, Map<T, T> map) { synchronized (map) { - T reference = map.get(val); - if (null == reference) { - map.put(val, val); - reference = map.get(val); - } - return reference; + map.putIfAbsent(val, val); + return map.get(val); } }

1 0

[metrics-lib/master] Fail early when 'null' values are added.
by karsten＠torproject.org 05 Feb '18

05 Feb '18

commit 6dd7cb9676b1bb4c1e76a12afbc37da951a96209 Author: iwakeh <iwakeh(a)torproject.org> Date: Sat Feb 3 08:19:56 2018 +0000 Fail early when 'null' values are added. And, allow for roughly a years worth of entries in dateMap and requestMap before re-hashing by setting initial capacity. Defaults are fine for the small ipMap and protocolMap. --- .../java/org/torproject/descriptor/log/WebServerAccessLogLine.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java b/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java index 4884733..c9d73cc 100644 --- a/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java +++ b/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java @@ -13,6 +13,7 @@ import java.time.format.DateTimeFormatter; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -37,11 +38,11 @@ public class WebServerAccessLogLine { private static Map<String, String> ipMap = Collections.synchronizedMap(new HashMap<>()); private static Map<LocalDate, LocalDate> dateMap - = Collections.synchronizedMap(new HashMap<>()); + = Collections.synchronizedMap(new HashMap<>(500)); private static Map<String, String> protocolMap = Collections.synchronizedMap(new HashMap<>()); private static Map<String, String> requestMap - = Collections.synchronizedMap(new HashMap<>()); + = Collections.synchronizedMap(new HashMap<>(50_000)); private String ip; private int response; @@ -147,7 +148,7 @@ public class WebServerAccessLogLine { private static <T> T fromMap(T val, Map<T, T> map) { synchronized (map) { - map.putIfAbsent(val, val); + map.putIfAbsent(Objects.requireNonNull(val), val); return map.get(val); } }

1 0

[metrics-lib/master] Prepare for 2.2.0 release.
by karsten＠torproject.org 05 Feb '18

05 Feb '18

commit 678ddfd166c5a7bb9bfde2f5005c5b7e27139ec5 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Jan 31 13:35:24 2018 +0100 Prepare for 2.2.0 release. --- CHANGELOG.md | 2 +- build.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42e0e09..5939e38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 2.2.0 - 2018-01-?? +# Changes in version 2.2.0 - 2018-01-31 * Major changes - Add new descriptor type WebServerAccessLog to parse web server diff --git a/build.xml b/build.xml index 72c1de1..3f081f7 100644 --- a/build.xml +++ b/build.xml @@ -6,7 +6,7 @@ <project default="usage" name="metrics-lib" basedir="."> - <property name="release.version" value="2.1.1-dev" /> + <property name="release.version" value="2.2.0" /> <property name="javadoc-title" value="Tor Metrics Library API Documentation"/> <property name="javadoc-excludes" value="**/impl/** **/index/** **/internal/** **/log/**" /> <property name="implementation-title" value="Tor Metrics Library" />

1 0

[tor/master] Use tt_u64_op() for uint64_t inputs.
by nickm＠torproject.org 05 Feb '18

05 Feb '18

commit 3bed8fdb91599b5e7c7946978c6221ba5db85463 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Feb 2 15:23:55 2018 -0500 Use tt_u64_op() for uint64_t inputs. --- src/test/test_dos.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/test_dos.c b/src/test/test_dos.c index 071926c28..9496b0735 100644 --- a/src/test/test_dos.c +++ b/src/test/test_dos.c @@ -179,8 +179,8 @@ test_dos_bucket_refill(void *arg) uint64_t circ_rate = get_circuit_rate_per_second(); /* Check that the circuit rate is a positive number and smaller than the max * circuit count */ - tt_int_op(circ_rate, OP_GT, 1); - tt_int_op(circ_rate, OP_LT, max_circuit_count); + tt_u64_op(circ_rate, OP_GT, 1); + tt_u64_op(circ_rate, OP_LT, max_circuit_count); /* Register this client */ geoip_note_client_seen(GEOIP_CLIENT_CONNECT, addr, NULL, now);

1 0

[tor/master] dos: We can put less token than the current amount
by nickm＠torproject.org 05 Feb '18

05 Feb '18

commit 78d6cb58707ff46464c591e45d81e83388427e2c Author: David Goulet <dgoulet(a)torproject.org> Date: Fri Feb 2 17:04:12 2018 -0500 dos: We can put less token than the current amount Becasue the circuit creation burst and rate can change at runtime it is possible that between two refill of a bucket, we end up setting the bucket value to less than there currently is. Fixes #25128 Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- src/or/dos.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/or/dos.c b/src/or/dos.c index c221e5ecd..88f1351a3 100644 --- a/src/or/dos.c +++ b/src/or/dos.c @@ -308,8 +308,6 @@ cc_stats_refill_bucket(cc_client_stats_t *stats, const tor_addr_t *addr) new_circuit_bucket_count = MIN(stats->circuit_bucket + (uint32_t)num_token, dos_cc_circuit_burst); } - /* This function is not allowed to make the bucket count smaller */ - tor_assert_nonfatal(new_circuit_bucket_count >= stats->circuit_bucket); log_debug(LD_DOS, "DoS address %s has its circuit bucket value: %" PRIu32 ". Filling it to %" PRIu32 ". Circuit rate is %" PRIu64 ". Elapsed time is %" PRIi64,

1 0

[tor/master] Merge remote-tracking branch 'dgoulet/ticket24902_029_05'
by nickm＠torproject.org 05 Feb '18

05 Feb '18

commit f0d7905bc9f20ecee43da3db369073ffb73b4997 Merge: 953c769a8 78d6cb587 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Feb 5 11:38:58 2018 -0500 Merge remote-tracking branch 'dgoulet/ticket24902_029_05' src/or/dos.c | 2 -- src/test/test_dos.c | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-)

1 0