tor-commits February 2018

tor-commits@lists.torproject.org

19 participants
1579 discussions

[metrics-lib/release] Prepare for 2.2.0 release, again.
by karsten＠torproject.org 26 Feb '18

26 Feb '18

commit 72f62fbe48d7cbb2de6d7fd5fd38b867fb0b185e Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Feb 26 14:12:31 2018 +0100 Prepare for 2.2.0 release, again. --- CERT | 20 ++++++++++---------- CHANGELOG.md | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/CERT b/CERT index 0f90f7a..43be56e 100644 --- a/CERT +++ b/CERT @@ -1,8 +1,8 @@ -----BEGIN CERTIFICATE----- -MIIDaTCCAlGgAwIBAgIEM/uNRzANBgkqhkiG9w0BAQsFADBlMQswCQYDVQQGEwJV +MIIDaTCCAlGgAwIBAgIEIk6NnzANBgkqhkiG9w0BAQsFADBlMQswCQYDVQQGEwJV UzELMAkGA1UECBMCV0ExEDAOBgNVBAcTB1NlYXR0bGUxHTAbBgNVBAoTFFRoZSBU -b3IgUHJvamVjdCwgSW5jMRgwFgYDVQQDEw9LYXJzdGVuIExvZXNpbmcwHhcNMTcw -NTE2MTQ0NDAwWhcNMTcwODE0MTQ0NDAwWjBlMQswCQYDVQQGEwJVUzELMAkGA1UE +b3IgUHJvamVjdCwgSW5jMRgwFgYDVQQDEw9LYXJzdGVuIExvZXNpbmcwHhcNMTgw +MjI2MTQwMzUzWhcNMTgwNTI3MTQwMzUzWjBlMQswCQYDVQQGEwJVUzELMAkGA1UE CBMCV0ExEDAOBgNVBAcTB1NlYXR0bGUxHTAbBgNVBAoTFFRoZSBUb3IgUHJvamVj dCwgSW5jMRgwFgYDVQQDEw9LYXJzdGVuIExvZXNpbmcwggEiMA0GCSqGSIb3DQEB AQUAA4IBDwAwggEKAoIBAQChXn+IUp+o6G+k4ffxk3TkxZb3iXfiG7byNsG63olU @@ -11,11 +11,11 @@ Qw+VAhKTcEIv4yiR0BWapQyR07pgmKirYVjN6s6ef8NJzUptpxLlaYJ3ZfQfc4aE MXzScgaccwDFIWQ661lzLGCfeSxxa3Xy4wWsGwzNzLITYrrABcbg7yogLo2btNvD oEwGL3/baQdhl0dra6biVCZr9ydn3Hg57S55pUU0rBY25id78zUO8xrfNHw54wwX lOblGt75OOkahP/ZZSBxxoiknJ6y5VQV8y+noA4vigXFAgMBAAGjITAfMB0GA1Ud -DgQWBBSeh60M+/wMYyYhlxtuff2Hk9n7bzANBgkqhkiG9w0BAQsFAAOCAQEACxUk -6eZkayjJGZjw02bWjrkwwe/N5iOnOz9XcpGQY5lUEk12uIYCYHDB66SsZngsbwNQ -qZ8T6Avm9XSrKAKMxBcPsjTUy0Fb7yG3jLRypPlbgoe61YoliYYaWiaV1OQY1nsZ -ujt6f7T5bwsLeh7La7+kB1SGDu+hKH7Bi3RtVN09iL7x+BJVuJOKHKYoK+UO2fB3 -ltFenkGpiDzGQQxJRzA1WbCQsSPrE6Qi3NmWXsTAc+24Y9gTO8qlmfi6zp1W7TGO -+cUmHIO0u3+ewkb8oIj8jjzGTmEbuJrwgBOohQr6CXCicQHhIrBF6Opi7mF/UHSO -J8d+DLvT41EL9JrgCg== +DgQWBBSeh60M+/wMYyYhlxtuff2Hk9n7bzANBgkqhkiG9w0BAQsFAAOCAQEAlUkU +qqf+4yfXwAWFr2q6iijr54NDDEQwybCblIzVnsuGHPUDuie3ZWSHirtblBs/uJ9x +RxmwkBrJr9IGMmGhN2GKXIPeUH0EZBYo7bsgo5d+E61OCnd/O+1JZzdG9dK+0kfq +MLfo6ltFZZouHIIXfvOm8sLLRrdkXPrLQ/E8fTHB7dL6T8Hqg6pHRrRZDtuSM9CO +zSYropxqlFzzlzciOdTU05D8Cnx2j/RtaycxHxFS7QtriDB0uOfqvyiVeqpr72wG +qetlu3h46fXj3ALGVSXy+YZpYxcRNZsQyiBXdlXbgY0OfOVPFOH3HiZuv3zhfRJW +2DiJiA8BLxZToe2XDA== -----END CERTIFICATE----- diff --git a/CHANGELOG.md b/CHANGELOG.md index 91f00b8..4d87ba4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 2.2.0 - 2018-01-31 +# Changes in version 2.2.0 - 2018-02-26 * Major changes - Add new descriptor type WebServerAccessLog to parse web server

1 0

[metrics-lib/release] Enable handling of larger (> 2G) log files.
by karsten＠torproject.org 26 Feb '18

26 Feb '18

commit c01dfbdc4e5d817fb10fb45f79d8223853cdeac0 Author: iwakeh <iwakeh(a)torproject.org> Date: Fri Feb 16 09:05:46 2018 +0000 Enable handling of larger (> 2G) log files. As log files can be compressed very efficiently log descriptor raw bytes contain the compressed bytes. Added methods for accessing uncompressed log content, as well as stream based methods for decompression and compression in class FileType. Adapted all tests to the changes. Implements task-25329. --- .../org/torproject/descriptor/LogDescriptor.java | 26 ++++++++++++-- .../torproject/descriptor/internal/FileType.java | 18 ++++++++++ .../descriptor/log/LogDescriptorImpl.java | 25 ++++++++------ .../descriptor/log/WebServerAccessLogImpl.java | 12 +++---- .../descriptor/log/LogDescriptorTest.java | 40 +++++++++++----------- .../descriptor/log/WebServerModuleTest.java | 4 ++- 6 files changed, 82 insertions(+), 43 deletions(-) diff --git a/src/main/java/org/torproject/descriptor/LogDescriptor.java b/src/main/java/org/torproject/descriptor/LogDescriptor.java index 6a6bf84..826fcda 100644 --- a/src/main/java/org/torproject/descriptor/LogDescriptor.java +++ b/src/main/java/org/torproject/descriptor/LogDescriptor.java @@ -3,21 +3,34 @@ package org.torproject.descriptor; +import java.io.InputStream; import java.util.List; /** * Contains a log file. * * Unlike other descriptors, logs can get very large and are typically stored - * on disk in compressed form. However, all access to log contents through this - * interface and its subinterfaces is made available in uncompressed form. + * on disk in compressed form. Access to log contents through this + * interface and its subinterfaces is made available in compressed and + * decompressed form: + * <ul> + * <li>The raw descriptor bytes are compressed, because logs contain + * often redundant information that can achieve high compression rates. + * For example, a 500kB compressed log file might be deflated to 3GB.</li> + * <li>The uncompressed log contents can be accessed as a stream of bytes.</li> + * <li>A list of log lines (decompressed) can be retrieved.</li> + * </ul> + * * * @since 2.2.0 */ public interface LogDescriptor extends Descriptor { /** - * Returns the decompressed raw descriptor bytes of the log. + * Returns the raw compressed descriptor bytes of the log. + * + * For access to the log's decompressed bytes of + * use method {@code decompressedByteStream}. * * @since 2.2.0 */ @@ -25,6 +38,13 @@ public interface LogDescriptor extends Descriptor { public byte[] getRawDescriptorBytes(); /** + * Returns the decompressed raw descriptor bytes of the log as stream. + * + * @since 2.2.0 + */ + public InputStream decompressedByteStream() throws DescriptorParseException; + + /** * Returns annotations found in the log file, which may be an empty List if a * log format does not support adding annotations. * diff --git a/src/main/java/org/torproject/descriptor/internal/FileType.java b/src/main/java/org/torproject/descriptor/internal/FileType.java index 353f0bb..2c07df6 100644 --- a/src/main/java/org/torproject/descriptor/internal/FileType.java +++ b/src/main/java/org/torproject/descriptor/internal/FileType.java @@ -93,6 +93,24 @@ public enum FileType { } /** + * Compresses the given InputStream and returns an OutputStream. + * + * @since 2.2.0 + */ + public OutputStream compress(OutputStream os) throws Exception { + return this.outputStream(os); + } + + /** + * Decompresses the given InputStream and returns an OutputStream. + * + * @since 2.2.0 + */ + public InputStream decompress(InputStream is) throws Exception { + return this.inputStream(is); + } + + /** * Decompresses the given bytes in memory and returns the decompressed bytes. * * @since 2.2.0 diff --git a/src/main/java/org/torproject/descriptor/log/LogDescriptorImpl.java b/src/main/java/org/torproject/descriptor/log/LogDescriptorImpl.java index 97854e4..3583d26 100644 --- a/src/main/java/org/torproject/descriptor/log/LogDescriptorImpl.java +++ b/src/main/java/org/torproject/descriptor/log/LogDescriptorImpl.java @@ -14,10 +14,10 @@ import org.slf4j.LoggerFactory; import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.File; +import java.io.InputStream; import java.io.InputStreamReader; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.regex.Matcher; @@ -76,8 +76,7 @@ public abstract class LogDescriptorImpl this.fileType = FileType.findType(mat.group(1).toUpperCase()); if (FileType.PLAIN == this.fileType) { this.fileType = defaultCompression; - } else { - this.logBytes = this.fileType.decompress(this.logBytes); + this.logBytes = this.fileType.compress(this.logBytes); } } catch (Exception ex) { throw new DescriptorParseException("Cannot parse file " @@ -86,10 +85,19 @@ public abstract class LogDescriptorImpl } @Override + public InputStream decompressedByteStream() throws DescriptorParseException { + try { + return this.fileType.decompress(new ByteArrayInputStream(this.logBytes)); + } catch (Exception ex) { + throw new DescriptorParseException("Cannot provide deflated stream of " + + this.descriptorFile + ".", ex); + } + } + + @Override public void validate() throws DescriptorParseException { - try (BufferedReader br - = new BufferedReader(new InputStreamReader(new ByteArrayInputStream( - this.logBytes)))) { + try (BufferedReader br = new BufferedReader( + new InputStreamReader(decompressedByteStream()))) { this.unrecognizedLines.addAll(br.lines().parallel().filter((line) -> null != line && !line.isEmpty() && !validator.validate(line)) .limit(unrecognizedLinesLimit).collect(Collectors.toList())); @@ -114,11 +122,6 @@ public abstract class LogDescriptorImpl } } - public static byte[] collectionToBytes(Collection<String> lines) { - return lines.stream().collect(Collectors.joining("\n", "", "\n")) - .getBytes(); - } - @Override public void setValidator(Validator validator) { this.validator = validator; diff --git a/src/main/java/org/torproject/descriptor/log/WebServerAccessLogImpl.java b/src/main/java/org/torproject/descriptor/log/WebServerAccessLogImpl.java index 7b56528..e48a262 100644 --- a/src/main/java/org/torproject/descriptor/log/WebServerAccessLogImpl.java +++ b/src/main/java/org/torproject/descriptor/log/WebServerAccessLogImpl.java @@ -11,12 +11,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.BufferedReader; -import java.io.ByteArrayInputStream; import java.io.File; import java.io.InputStreamReader; import java.time.LocalDate; import java.time.format.DateTimeFormatter; -import java.util.Collection; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -72,10 +70,9 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl } /** For internal use only. */ - public WebServerAccessLogImpl(Collection<String> lines, String filename, + public WebServerAccessLogImpl(byte[] bytes, String filename, boolean validate) throws DescriptorParseException { - this(LogDescriptorImpl.collectionToBytes(lines), new File(filename), - FileType.XZ, validate); + this(bytes, new File(filename), FileType.XZ, validate); } private WebServerAccessLogImpl(byte[] logBytes, File file, @@ -135,9 +132,8 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl @Override public List<WebServerAccessLog.Line> logLines() throws DescriptorParseException { - try (BufferedReader br - = new BufferedReader(new InputStreamReader(new ByteArrayInputStream( - this.getRawDescriptorBytes())))) { + try (BufferedReader br = new BufferedReader(new InputStreamReader( + this.decompressedByteStream()))) { return br.lines().map(line -> (WebServerAccessLog.Line) WebServerAccessLogLine.makeLine(line)) .filter(line -> line.isValid()).collect(Collectors.toList()); diff --git a/src/test/java/org/torproject/descriptor/log/LogDescriptorTest.java b/src/test/java/org/torproject/descriptor/log/LogDescriptorTest.java index a871791..67ba638 100644 --- a/src/test/java/org/torproject/descriptor/log/LogDescriptorTest.java +++ b/src/test/java/org/torproject/descriptor/log/LogDescriptorTest.java @@ -21,6 +21,7 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; import org.junit.runners.Parameterized.Parameters; import java.io.File; @@ -46,46 +47,45 @@ public class LogDescriptorTest { protected DescriptorReader reader = DescriptorSourceFactory.createDescriptorReader(); - protected int size; - protected String[] pan; - protected Class<LogDescriptor> type; - protected boolean isDecompressionTest; - protected int lineCount; + @Parameter(0) + public boolean isDecompressedLog; + + @Parameter(1) + public int size; + + @Parameter(2) + public String[] pan; + + @Parameter(3) + public Class<LogDescriptor> type; + + @Parameter(4) + public int lineCount; /** All types of data that can be encountered during sync. */ @Parameters public static Collection<Object[]> pathAndName() { return Arrays.asList(new Object[][] { - {Boolean.TRUE, 1878, new String[]{"meronense.torproject.org", + {Boolean.FALSE, 1878, new String[]{"meronense.torproject.org", "metrics.torproject.org_meronense.torproject.org_access.log" + "_20170530.gz", "metrics.torproject.org", "20170530", "gz"}, WebServerAccessLog.class, 24}, - {Boolean.FALSE, 1878, new String[]{"meronense.torproject.org", + {Boolean.TRUE, 1878, new String[]{"meronense.torproject.org", "xy.host.org_meronense.torproject.org_access.log_20170530.log", "metrics.torproject.org", "20170530", "xz"}, WebServerAccessLog.class, 24}, - {Boolean.TRUE, 70730, new String[]{"archeotrichon.torproject.org", + {Boolean.FALSE, 70730, new String[]{"archeotrichon.torproject.org", "archive.torproject.org_archeotrichon.torproject.org_access.log_" + "20151007.xz", "archive.torproject.org", "20151007", "xz"}, WebServerAccessLog.class, 655}, - {Boolean.TRUE, 0, new String[]{"dummy.host.net", + {Boolean.FALSE, 0, new String[]{"dummy.host.net", "nix.server.org_dummy.host.net_access.log_20111111.bz2", "nix.server.org", "20111111", "bz2"}, WebServerAccessLog.class, 0}}); } - /** This constructor receives the above defined data for each run. */ - public LogDescriptorTest(boolean decompression, int size, String[] pan, - Class<LogDescriptor> type, int lineCount) { - this.pan = pan; - this.size = size; - this.type = type; - this.isDecompressionTest = decompression; - this.lineCount = lineCount; - } - /** Prepares the temporary folder and writes files to it for this test. */ private void createTemporaryFolderAndContents() throws IOException { this.indir = this.temp.newFolder(); @@ -157,7 +157,7 @@ public class LogDescriptorTest { @Test public void testCompressionInvalid() throws Exception { - if (!isDecompressionTest) { + if (isDecompressedLog) { return; } assertEquals(1, this.reader.getParsedFiles().size()); diff --git a/src/test/java/org/torproject/descriptor/log/WebServerModuleTest.java b/src/test/java/org/torproject/descriptor/log/WebServerModuleTest.java index a11bc30..66c8f54 100644 --- a/src/test/java/org/torproject/descriptor/log/WebServerModuleTest.java +++ b/src/test/java/org/torproject/descriptor/log/WebServerModuleTest.java @@ -6,6 +6,7 @@ package org.torproject.descriptor.log; import static org.junit.Assert.assertEquals; import org.torproject.descriptor.DescriptorParseException; +import org.torproject.descriptor.internal.FileType; import org.hamcrest.Matchers; @@ -104,7 +105,8 @@ public class WebServerModuleTest { WebServerAccessLogImpl wsal = new WebServerAccessLogImpl(logText.getBytes(), new File("vhost_host7_access.log_20170530")); assertEquals(wsal.getAnnotations().size(), 0); - assertEquals(logText, new String(wsal.getRawDescriptorBytes())); + assertEquals(logText, + new String(FileType.XZ.decompress(wsal.getRawDescriptorBytes()))); assertEquals("host7", wsal.getPhysicalHost()); assertEquals("vhost", wsal.getVirtualHost()); }

1 0

[metrics-lib/master] Prepare for 2.2.0 release, again.
by karsten＠torproject.org 26 Feb '18

26 Feb '18

1 0

[metrics-lib/master] Bump version to 2.2.0-dev.
by karsten＠torproject.org 26 Feb '18

26 Feb '18

commit 7648780844f809f3068d3e625973fe43c78ccbb3 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Feb 26 16:24:25 2018 +0100 Bump version to 2.2.0-dev. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index 3f081f7..199ba7a 100644 --- a/build.xml +++ b/build.xml @@ -6,7 +6,7 @@ <project default="usage" name="metrics-lib" basedir="."> - <property name="release.version" value="2.2.0" /> + <property name="release.version" value="2.2.0-dev" /> <property name="javadoc-title" value="Tor Metrics Library API Documentation"/> <property name="javadoc-excludes" value="**/impl/** **/index/** **/internal/** **/log/**" /> <property name="implementation-title" value="Tor Metrics Library" />

1 0

[collector/master] Bump version to 1.5.0-dev.
by karsten＠torproject.org 26 Feb '18

26 Feb '18

commit ef1dfb6d32d5bb42a95052cd44790b6c586f38c9 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Feb 26 16:24:49 2018 +0100 Bump version to 1.5.0-dev. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index ff8302b..fb219fa 100644 --- a/build.xml +++ b/build.xml @@ -8,7 +8,7 @@ <property name="javadoc-title" value="CollecTor API Documentation"/> <property name="implementation-title" value="CollecTor" /> - <property name="release.version" value="1.5.0" /> + <property name="release.version" value="1.5.0-dev" /> <property name="project-main-class" value="org.torproject.collector.Main" /> <property name="name" value="collector"/> <property name="metricslibversion" value="2.2.0" />

1 0

[collector/master] Prepare for 1.5.0 release.
by karsten＠torproject.org 26 Feb '18

26 Feb '18

commit ddfa7bad243dc8e0a3105b14418794c87786e88f Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Feb 26 14:23:05 2018 +0100 Prepare for 1.5.0 release. --- CERT | 20 ++++++++++---------- CHANGELOG.md | 2 +- build.xml | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/CERT b/CERT index d94c0c3..43be56e 100644 --- a/CERT +++ b/CERT @@ -1,8 +1,8 @@ -----BEGIN CERTIFICATE----- -MIIDaTCCAlGgAwIBAgIEZTniETANBgkqhkiG9w0BAQsFADBlMQswCQYDVQQGEwJV +MIIDaTCCAlGgAwIBAgIEIk6NnzANBgkqhkiG9w0BAQsFADBlMQswCQYDVQQGEwJV UzELMAkGA1UECBMCV0ExEDAOBgNVBAcTB1NlYXR0bGUxHTAbBgNVBAoTFFRoZSBU -b3IgUHJvamVjdCwgSW5jMRgwFgYDVQQDEw9LYXJzdGVuIExvZXNpbmcwHhcNMTcw -ODE3MTg1MDQ0WhcNMTcxMTE1MTg1MDQ0WjBlMQswCQYDVQQGEwJVUzELMAkGA1UE +b3IgUHJvamVjdCwgSW5jMRgwFgYDVQQDEw9LYXJzdGVuIExvZXNpbmcwHhcNMTgw +MjI2MTQwMzUzWhcNMTgwNTI3MTQwMzUzWjBlMQswCQYDVQQGEwJVUzELMAkGA1UE CBMCV0ExEDAOBgNVBAcTB1NlYXR0bGUxHTAbBgNVBAoTFFRoZSBUb3IgUHJvamVj dCwgSW5jMRgwFgYDVQQDEw9LYXJzdGVuIExvZXNpbmcwggEiMA0GCSqGSIb3DQEB AQUAA4IBDwAwggEKAoIBAQChXn+IUp+o6G+k4ffxk3TkxZb3iXfiG7byNsG63olU @@ -11,11 +11,11 @@ Qw+VAhKTcEIv4yiR0BWapQyR07pgmKirYVjN6s6ef8NJzUptpxLlaYJ3ZfQfc4aE MXzScgaccwDFIWQ661lzLGCfeSxxa3Xy4wWsGwzNzLITYrrABcbg7yogLo2btNvD oEwGL3/baQdhl0dra6biVCZr9ydn3Hg57S55pUU0rBY25id78zUO8xrfNHw54wwX lOblGt75OOkahP/ZZSBxxoiknJ6y5VQV8y+noA4vigXFAgMBAAGjITAfMB0GA1Ud -DgQWBBSeh60M+/wMYyYhlxtuff2Hk9n7bzANBgkqhkiG9w0BAQsFAAOCAQEAbsAc -gwl5KJH3pVKw4b+ACCOMgW+27MisCFbT1Izq2Wx+JcLMt3N//MoIpYOZWhsIeazW -/NE0fNbkLi0IYA0F1nUC9pHl44Hd8Gjfqa/YQUi9ALtgsY7l6W0sceW8WnZ8bu8J -DfrqnmB0bD2xc9ZjOn58al8dVjVWs95M87D9WCRU6LiaKFj5c45wciABQsTmC0qD -pyHYOaSGtXxXKDw5pAntdtHkCbowV5tDi/QQ8Tg7i5O7xwSh71Q7TZiNFMpLomBL -QllHfTZryFmoHyGn5MfngBUVCVHig5nXmk0dUMGuLiK4789dkgiPRz0vpB5Yf8Yy -CCE2jB6VBi2g5fMx0w== +DgQWBBSeh60M+/wMYyYhlxtuff2Hk9n7bzANBgkqhkiG9w0BAQsFAAOCAQEAlUkU +qqf+4yfXwAWFr2q6iijr54NDDEQwybCblIzVnsuGHPUDuie3ZWSHirtblBs/uJ9x +RxmwkBrJr9IGMmGhN2GKXIPeUH0EZBYo7bsgo5d+E61OCnd/O+1JZzdG9dK+0kfq +MLfo6ltFZZouHIIXfvOm8sLLRrdkXPrLQ/E8fTHB7dL6T8Hqg6pHRrRZDtuSM9CO +zSYropxqlFzzlzciOdTU05D8Cnx2j/RtaycxHxFS7QtriDB0uOfqvyiVeqpr72wG +qetlu3h46fXj3ALGVSXy+YZpYxcRNZsQyiBXdlXbgY0OfOVPFOH3HiZuv3zhfRJW +2DiJiA8BLxZToe2XDA== -----END CERTIFICATE----- diff --git a/CHANGELOG.md b/CHANGELOG.md index a0b5d1f..eb14839 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 1.5.0 - 2018-01-31 +# Changes in version 1.5.0 - 2018-02-26 * Major changes - Update to metrics-lib 2.2.0. diff --git a/build.xml b/build.xml index 48f6e33..ff8302b 100644 --- a/build.xml +++ b/build.xml @@ -8,7 +8,7 @@ <property name="javadoc-title" value="CollecTor API Documentation"/> <property name="implementation-title" value="CollecTor" /> - <property name="release.version" value="1.4.1-dev" /> + <property name="release.version" value="1.5.0" /> <property name="project-main-class" value="org.torproject.collector.Main" /> <property name="name" value="collector"/> <property name="metricslibversion" value="2.2.0" />

1 0

[collector/master] Adapt CollecTor to latest metrics-lib master branch.
by karsten＠torproject.org 26 Feb '18

26 Feb '18

commit fbb35f75da022a23912b937b1825d8f216abad07 Author: iwakeh <iwakeh(a)torproject.org> Date: Tue Feb 20 16:30:08 2018 +0000 Adapt CollecTor to latest metrics-lib master branch. --- src/main/java/org/torproject/collector/webstats/SanitizeWeblogs.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/torproject/collector/webstats/SanitizeWeblogs.java b/src/main/java/org/torproject/collector/webstats/SanitizeWeblogs.java index 53ad118..7601898 100644 --- a/src/main/java/org/torproject/collector/webstats/SanitizeWeblogs.java +++ b/src/main/java/org/torproject/collector/webstats/SanitizeWeblogs.java @@ -15,10 +15,10 @@ import org.torproject.collector.cron.CollecTorMain; import org.torproject.collector.persist.PersistenceUtils; import org.torproject.collector.persist.WebServerAccessLogPersistence; import org.torproject.descriptor.DescriptorParseException; +import org.torproject.descriptor.Method; import org.torproject.descriptor.WebServerAccessLog; import org.torproject.descriptor.log.InternalLogDescriptor; import org.torproject.descriptor.log.InternalWebServerAccessLog; -import org.torproject.descriptor.log.Method; import org.torproject.descriptor.log.WebServerAccessLogImpl; import org.torproject.descriptor.log.WebServerAccessLogLine;

1 0

[collector/master] Reduce memory footprint and wall time.
by karsten＠torproject.org 26 Feb '18

26 Feb '18

commit 8557bf6255e6e3745088033e8e7bad7801421686 Author: iwakeh <iwakeh(a)torproject.org> Date: Tue Feb 20 16:30:09 2018 +0000 Reduce memory footprint and wall time. Adapt to latest changes of metrics-lib (task-25329) and make use of the high redundancy of logs (e.g. a 3G file might only contain 350 different lines). This avoids OOM and array out of bounds exceptions for large files (>2G) and gives a speed-up of roughly 50%. (The earlier 66min are down to 34min for meronense&weschniakowii files plus two larger files.) There is a BATCH constant, which could be tuned for processing speed. It is logged for each webstats module run. Currently, it is set to 100k. This was more or less arbitrarily chosen and used for all the tests. A test run using 500k didn't show significant differences. --- .../persist/WebServerAccessLogPersistence.java | 8 --- .../collector/webstats/SanitizeWeblogs.java | 61 ++++++++++++++++++---- 2 files changed, 51 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/torproject/collector/persist/WebServerAccessLogPersistence.java b/src/main/java/org/torproject/collector/persist/WebServerAccessLogPersistence.java index 792d3a9..dab4112 100644 --- a/src/main/java/org/torproject/collector/persist/WebServerAccessLogPersistence.java +++ b/src/main/java/org/torproject/collector/persist/WebServerAccessLogPersistence.java @@ -5,7 +5,6 @@ package org.torproject.collector.persist; import org.torproject.descriptor.WebServerAccessLog; import org.torproject.descriptor.internal.FileType; -import org.torproject.descriptor.log.InternalLogDescriptor; import org.torproject.descriptor.log.InternalWebServerAccessLog; import org.slf4j.Logger; @@ -30,13 +29,6 @@ public class WebServerAccessLogPersistence /** Prepare storing the given descriptor. */ public WebServerAccessLogPersistence(WebServerAccessLog desc) { super(desc, new byte[0]); - byte[] compressedBytes = null; - try { // The descriptor bytes have to be stored compressed. - compressedBytes = COMPRESSION.compress(desc.getRawDescriptorBytes()); - ((InternalLogDescriptor)desc).setRawDescriptorBytes(compressedBytes); - } catch (Exception ex) { - log.warn("Cannot compress ’{}’. Storing uncompressed.", ex); - } calculatePaths(); } diff --git a/src/main/java/org/torproject/collector/webstats/SanitizeWeblogs.java b/src/main/java/org/torproject/collector/webstats/SanitizeWeblogs.java index 7601898..1f2e922 100644 --- a/src/main/java/org/torproject/collector/webstats/SanitizeWeblogs.java +++ b/src/main/java/org/torproject/collector/webstats/SanitizeWeblogs.java @@ -3,6 +3,7 @@ package org.torproject.collector.webstats; +import static java.util.stream.Collectors.counting; import static java.util.stream.Collectors.groupingByConcurrent; import static java.util.stream.Collectors.toList; @@ -17,6 +18,7 @@ import org.torproject.collector.persist.WebServerAccessLogPersistence; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.Method; import org.torproject.descriptor.WebServerAccessLog; +import org.torproject.descriptor.internal.FileType; import org.torproject.descriptor.log.InternalLogDescriptor; import org.torproject.descriptor.log.InternalWebServerAccessLog; import org.torproject.descriptor.log.WebServerAccessLogImpl; @@ -26,8 +28,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.BufferedReader; -import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.InputStreamReader; +import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; import java.time.LocalDate; @@ -40,6 +43,7 @@ import java.util.SortedSet; import java.util.StringJoiner; import java.util.TreeMap; import java.util.TreeSet; +import java.util.stream.Collectors; import java.util.stream.Stream; /** @@ -87,6 +91,7 @@ public class SanitizeWeblogs extends CollecTorMain { Set<SourceType> sources = this.config.getSourceTypeSet( Key.WebstatsSources); if (sources.contains(SourceType.Local)) { + log.info("Processing logs using batch value {}.", BATCH); findCleanWrite(this.config.getPath(Key.WebstatsLocalOrigins)); PersistenceUtils.cleanDirectory(this.config.getPath(Key.RecentPath)); } @@ -126,24 +131,60 @@ public class SanitizeWeblogs extends CollecTorMain { String name = new StringJoiner(InternalLogDescriptor.SEP) .add(virtualHost).add(physicalHost) .add(InternalWebServerAccessLog.MARKER) - .add(date.format(DateTimeFormatter.BASIC_ISO_DATE)).toString(); + .add(date.format(DateTimeFormatter.BASIC_ISO_DATE)) + .toString() + "." + FileType.XZ.name().toLowerCase(); log.debug("Sanitizing {}.", name); - List<String> retainedLines = lines + Map<String, Long> retainedLines = new TreeMap<>(lines .stream().parallel().map((line) -> sanitize(line, date)) - .filter((line) -> line.isPresent()).map((line) -> line.get()) - .collect(toList()); - retainedLines.sort(null); + .filter((line) -> line.isPresent()) + .map((line) -> line.get()) + .collect(groupingByConcurrent(line -> line, counting()))); + lines.clear(); // not needed anymore try { WebServerAccessLogPersistence walp = new WebServerAccessLogPersistence( - new WebServerAccessLogImpl(retainedLines, name, false)); + new WebServerAccessLogImpl(toCompressedBytes(retainedLines), + name, false)); log.debug("Storing {}.", name); walp.storeOut(this.outputPathName); walp.storeRecent(this.recentPathName); } catch (DescriptorParseException dpe) { log.error("Cannot store log desriptor {}.", name, dpe); + } catch (Throwable th) { // catch all else + log.error("Serious problem. Cannot store log desriptor {}.", name, th); } - lines.clear(); + } + + private static final int BATCH = 100_000; + + static byte[] toCompressedBytes(Map<String, Long> lines) + throws DescriptorParseException { + try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); + OutputStream os = FileType.XZ.outputStream(baos)) { + for (Map.Entry<String, Long> entry : lines.entrySet()) { + long count = entry.getValue(); + byte[] batch = bytesFor(entry.getKey(), BATCH); + while (count > 0) { + if (count > BATCH) { + os.write(batch); + count -= BATCH; + } else { + os.write(bytesFor(entry.getKey(), count)); + break; + } + } + } + os.flush(); + os.close(); + return baos.toByteArray(); + } catch (Exception ex) { + throw new DescriptorParseException(ex.getMessage()); + } + } + + private static byte[] bytesFor(String line, long times) { + return Stream.of(line).limit(times) + .collect(Collectors.joining("\n", "", "\n")).getBytes(); } static Optional<String> sanitize(WebServerAccessLogLine logLine, @@ -186,8 +227,8 @@ public class SanitizeWeblogs extends CollecTorMain { private Stream<WebServerAccessLogLine> lineStream(LogMetadata metadata) { log.debug("Processing file {}.", metadata.path); try (BufferedReader br - = new BufferedReader(new InputStreamReader(new ByteArrayInputStream( - metadata.fileType.decompress(Files.readAllBytes(metadata.path)))))) { + = new BufferedReader(new InputStreamReader( + metadata.fileType.decompress(Files.newInputStream(metadata.path))))) { return br.lines() .map((String line) -> WebServerAccessLogLine.makeLine(line)) .collect(toList()).stream();

1 0

[collector/master] Circumvent Collection (integer) size limit.
by karsten＠torproject.org 26 Feb '18

26 Feb '18

commit d05b4e4aee3bc15c3e4d5bac660dfcee5bc26279 Author: iwakeh <iwakeh(a)torproject.org> Date: Tue Feb 20 16:30:14 2018 +0000 Circumvent Collection (integer) size limit. Clean log lines immediately when they are read and also make use of sanitized log's high redundancy immediately, i.e., continue with maps of <LocalDate, <Map<String, Long>>. Rename method(s) to reflect what they do. --- .../collector/webstats/SanitizeWeblogs.java | 89 ++++++++++++++++------ 1 file changed, 65 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/torproject/collector/webstats/SanitizeWeblogs.java b/src/main/java/org/torproject/collector/webstats/SanitizeWeblogs.java index 1f2e922..5a270dd 100644 --- a/src/main/java/org/torproject/collector/webstats/SanitizeWeblogs.java +++ b/src/main/java/org/torproject/collector/webstats/SanitizeWeblogs.java @@ -4,8 +4,10 @@ package org.torproject.collector.webstats; import static java.util.stream.Collectors.counting; +import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.groupingByConcurrent; -import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.reducing; +import static java.util.stream.Collectors.summingLong; import org.torproject.collector.conf.Configuration; import org.torproject.collector.conf.ConfigurationException; @@ -35,6 +37,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.LocalDate; import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -111,35 +115,36 @@ public class SanitizeWeblogs extends CollecTorMain { : virtualEntry.getValue().entrySet()) { String physicalHost = physicalEntry.getKey(); log.info("Processing logs for {} on {}.", virtualHost, physicalHost); - Map<LocalDate, List<WebServerAccessLogLine>> linesByDate + Map<LocalDate, Map<String, Long>> linesByDate = physicalEntry.getValue().values().stream().parallel() - .flatMap((LogMetadata metadata) -> lineStream(metadata) - .filter((line) -> line.isValid())).parallel() - .collect(groupingByConcurrent(WebServerAccessLogLine::getDate)); + .flatMap(metadata -> sanitzedLineStream(metadata).entrySet() + .stream()) + .collect(groupingBy(Map.Entry::getKey, + reducing(Collections.emptyMap(), Map.Entry::getValue, + (e1, e2) -> Stream.concat(e1.entrySet().stream(), e2.entrySet() + .stream()) + .collect(groupingByConcurrent(Map.Entry::getKey, + summingLong(Map.Entry::getValue)))))); LocalDate[] interval = determineInterval(linesByDate.keySet()); linesByDate.entrySet().stream() .filter((entry) -> entry.getKey().isAfter(interval[0]) && entry.getKey().isBefore(interval[1])).parallel() - .forEach((entry) -> storeSanitized(virtualHost, physicalHost, + .forEach((entry) -> storeSortedAndForget(virtualHost, physicalHost, entry.getKey(), entry.getValue())); } } } - private void storeSanitized(String virtualHost, String physicalHost, - LocalDate date, List<WebServerAccessLogLine> lines) { + private void storeSortedAndForget(String virtualHost, String physicalHost, + LocalDate date, Map<String, Long> lineCounts) { String name = new StringJoiner(InternalLogDescriptor.SEP) .add(virtualHost).add(physicalHost) .add(InternalWebServerAccessLog.MARKER) .add(date.format(DateTimeFormatter.BASIC_ISO_DATE)) .toString() + "." + FileType.XZ.name().toLowerCase(); - log.debug("Sanitizing {}.", name); - Map<String, Long> retainedLines = new TreeMap<>(lines - .stream().parallel().map((line) -> sanitize(line, date)) - .filter((line) -> line.isPresent()) - .map((line) -> line.get()) - .collect(groupingByConcurrent(line -> line, counting()))); - lines.clear(); // not needed anymore + log.debug("Storing {}.", name); + Map<String, Long> retainedLines = new TreeMap<>(lineCounts); + lineCounts.clear(); // not needed anymore try { WebServerAccessLogPersistence walp = new WebServerAccessLogPersistence( @@ -187,8 +192,8 @@ public class SanitizeWeblogs extends CollecTorMain { .collect(Collectors.joining("\n", "", "\n")).getBytes(); } - static Optional<String> sanitize(WebServerAccessLogLine logLine, - LocalDate date) { + static Optional<WebServerAccessLogLine> + sanitize(WebServerAccessLogLine logLine) { if (!logLine.isValid() || !(Method.GET == logLine.getMethod() || Method.HEAD == logLine.getMethod()) @@ -203,10 +208,13 @@ public class SanitizeWeblogs extends CollecTorMain { if (queryStart > 0) { logLine.setRequest(logLine.getRequest().substring(0, queryStart)); } - return Optional.of(logLine.toLogString()); + return Optional.of(logLine); } LocalDate[] determineInterval(Set<LocalDate> dates) { + if (dates.isEmpty()) { // return the empty interval + return new LocalDate[]{LocalDate.MAX, LocalDate.MIN}; + } SortedSet<LocalDate> sorted = new TreeSet<>(); sorted.addAll(dates); if (this.limits) { @@ -214,7 +222,7 @@ public class SanitizeWeblogs extends CollecTorMain { sorted.remove(sorted.last()); } } - if (sorted.isEmpty()) { + if (sorted.isEmpty()) { // return the empty interval return new LocalDate[]{LocalDate.MAX, LocalDate.MIN}; } if (!this.limits) { @@ -224,18 +232,51 @@ public class SanitizeWeblogs extends CollecTorMain { return new LocalDate[]{sorted.first(), sorted.last()}; } - private Stream<WebServerAccessLogLine> lineStream(LogMetadata metadata) { + private static final int LISTLIMIT = Integer.MAX_VALUE / 2; + + private Map<LocalDate, Map<String, Long>> + sanitzedLineStream(LogMetadata metadata) { log.debug("Processing file {}.", metadata.path); try (BufferedReader br = new BufferedReader(new InputStreamReader( metadata.fileType.decompress(Files.newInputStream(metadata.path))))) { - return br.lines() - .map((String line) -> WebServerAccessLogLine.makeLine(line)) - .collect(toList()).stream(); + List<List<WebServerAccessLogLine>> lists = new ArrayList<>(); + List<WebServerAccessLogLine> currentList = new ArrayList<>(); + lists.add(currentList); + String lineStr = br.readLine(); + int count = 0; + while (null != lineStr) { + WebServerAccessLogLine wsal = WebServerAccessLogLine.makeLine(lineStr); + if (wsal.isValid()) { + currentList.add(wsal); + count++; + } + if (count >= LISTLIMIT) { + currentList = new ArrayList<>(); + lists.add(currentList); + count = 0; + } + lineStr = br.readLine(); + } + br.close(); + return lists.parallelStream() + .map(list -> list.stream() + .map(line -> sanitize(line)) + .filter(line -> line.isPresent()) + .map(line -> line.get()) + .collect(groupingBy(WebServerAccessLogLine::getDate, + groupingBy(WebServerAccessLogLine::toLogString, counting())))) + .flatMap(map -> map.entrySet().stream()).parallel() + .collect(groupingByConcurrent(Map.Entry::getKey, + reducing(Collections.emptyMap(), Map.Entry::getValue, + (e1, e2) -> Stream.concat(e1.entrySet().stream(), + e2.entrySet().stream()).parallel() + .collect(groupingByConcurrent(Map.Entry::getKey, + summingLong(Map.Entry::getValue)))))); } catch (Exception ex) { log.debug("Skipping log-file {}.", metadata.path, ex); } - return Stream.empty(); + return Collections.emptyMap(); } }

1 0

[metrics-lib/master] Accomodate logs with more than Integer.MAX_VALUE lines.
by karsten＠torproject.org 26 Feb '18

26 Feb '18

commit 35feb816f81f26bcc9dc035a1aaf496c34a86647 Author: iwakeh <iwakeh(a)torproject.org> Date: Fri Feb 16 09:05:46 2018 +0000 Accomodate logs with more than Integer.MAX_VALUE lines. Implements task-23046. --- .../org/torproject/descriptor/LogDescriptor.java | 10 +++++-- .../torproject/descriptor/WebServerAccessLog.java | 6 ++++ .../descriptor/log/WebServerAccessLogImpl.java | 32 ++++++++++++++++++---- .../descriptor/log/LogDescriptorTest.java | 5 +++- 4 files changed, 43 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/torproject/descriptor/LogDescriptor.java b/src/main/java/org/torproject/descriptor/LogDescriptor.java index 826fcda..8dd8460 100644 --- a/src/main/java/org/torproject/descriptor/LogDescriptor.java +++ b/src/main/java/org/torproject/descriptor/LogDescriptor.java @@ -5,6 +5,7 @@ package org.torproject.descriptor; import java.io.InputStream; import java.util.List; +import java.util.stream.Stream; /** * Contains a log file. @@ -64,11 +65,14 @@ public interface LogDescriptor extends Descriptor { public List<String> getUnrecognizedLines(); /** - * Returns a list of all parseable log lines. - * Might require a lot of memory depending on log size. + * Returns a stream of all parseable log lines. + * Depending on log size this might not fit into a collection type. + * + * @since 2.2.0 */ - public List<? extends Line> logLines() throws DescriptorParseException; + public Stream<? extends Line> logLines() throws DescriptorParseException; + /** Base interface for accessing log lines. */ public interface Line { /** Returns a log line string. */ diff --git a/src/main/java/org/torproject/descriptor/WebServerAccessLog.java b/src/main/java/org/torproject/descriptor/WebServerAccessLog.java index b4f1940..5f3ad73 100644 --- a/src/main/java/org/torproject/descriptor/WebServerAccessLog.java +++ b/src/main/java/org/torproject/descriptor/WebServerAccessLog.java @@ -62,6 +62,12 @@ public interface WebServerAccessLog extends LogDescriptor { @Override public List<String> getUnrecognizedLines(); + /** + * Facilitates access to all log line fields that don't only contain + * default values post sanitization. + * + * @since 2.2.0 + */ public interface Line extends LogDescriptor.Line { /** Returns the IP address of the requesting host. */ diff --git a/src/main/java/org/torproject/descriptor/log/WebServerAccessLogImpl.java b/src/main/java/org/torproject/descriptor/log/WebServerAccessLogImpl.java index e48a262..3666d5d 100644 --- a/src/main/java/org/torproject/descriptor/log/WebServerAccessLogImpl.java +++ b/src/main/java/org/torproject/descriptor/log/WebServerAccessLogImpl.java @@ -15,10 +15,11 @@ import java.io.File; import java.io.InputStreamReader; import java.time.LocalDate; import java.time.format.DateTimeFormatter; +import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Implementation of web server access log descriptors. @@ -128,15 +129,34 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl return this.logDate; } - /** Returns a list of all valid log lines. */ + private static final int LISTLIMIT = Integer.MAX_VALUE / 2; + + /** Returns a stream of all valid log lines. */ @Override - public List<WebServerAccessLog.Line> logLines() + public Stream<WebServerAccessLog.Line> logLines() throws DescriptorParseException { try (BufferedReader br = new BufferedReader(new InputStreamReader( this.decompressedByteStream()))) { - return br.lines().map(line - -> (WebServerAccessLog.Line) WebServerAccessLogLine.makeLine(line)) - .filter(line -> line.isValid()).collect(Collectors.toList()); + List<List<WebServerAccessLogLine>> lists = new ArrayList<>(); + List<WebServerAccessLogLine> currentList = new ArrayList<>(); + lists.add(currentList); + String lineStr = br.readLine(); + int count = 0; + while (null != lineStr) { + WebServerAccessLogLine wsal = WebServerAccessLogLine.makeLine(lineStr); + if (wsal.isValid()) { + currentList.add(wsal); + count++; + } + if (count >= LISTLIMIT) { + currentList = new ArrayList<>(); + lists.add(currentList); + count = 0; + } + lineStr = br.readLine(); + } + br.close(); + return lists.stream().flatMap(list -> list.stream()); } catch (Exception ex) { throw new DescriptorParseException("Cannot retrieve log lines.", ex); } diff --git a/src/test/java/org/torproject/descriptor/log/LogDescriptorTest.java b/src/test/java/org/torproject/descriptor/log/LogDescriptorTest.java index 67ba638..0ff3e62 100644 --- a/src/test/java/org/torproject/descriptor/log/LogDescriptorTest.java +++ b/src/test/java/org/torproject/descriptor/log/LogDescriptorTest.java @@ -3,6 +3,8 @@ package org.torproject.descriptor.log; +import static java.util.stream.Collectors.toList; + import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -130,7 +132,8 @@ public class LogDescriptorTest { InternalLogDescriptor ld = (InternalLogDescriptor) descs.get(0); assertEquals("Wrong compression type string. " + dataUsed(), pan[4], ld.getCompressionType()); - List<? extends LogDescriptor.Line> lines = ld.logLines(); + List<? extends LogDescriptor.Line> lines + = ld.logLines().collect(toList()); assertEquals(this.lineCount, lines.size()); }

1 0

← Newer
1
2
3
4
5
6
7
8
...
158
Older →

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits February 2018