December 2019 - tor-commits - lists.torproject.org

[metrics-web/master] Update to latest metrics-lib.
by karsten＠torproject.org 02 Dec '19

02 Dec '19

commit b1b1aa97fc83012e5d0feed516c8a629c45c01b4 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Dec 2 16:46:44 2019 +0100 Update to latest metrics-lib. --- src/submods/metrics-lib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/submods/metrics-lib b/src/submods/metrics-lib index 87d998c..067add6 160000 --- a/src/submods/metrics-lib +++ b/src/submods/metrics-lib @@ -1 +1 @@ -Subproject commit 87d998c542688c7e465486de24ef07c1cbece0b0 +Subproject commit 067add639b8cd260b1530a50c81e0e1d0a63e9b0

1 0

[collector/master] Remove dependency on metrics-lib's log package (1/4).
by karsten＠torproject.org 02 Dec '19

02 Dec '19

commit 859476ecaec2164e0d84bbba4377da11c90034b2 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sat Nov 23 17:13:22 2019 +0100 Remove dependency on metrics-lib's log package (1/4). - Copy types from metrics-lib to this code base. - Update package and import statements. - Copy remaining parts of metrics-lib's FileType. --- .../persist/WebServerAccessLogPersistence.java | 2 +- .../metrics/collector/webstats/FileType.java | 44 +++++ .../… [View More]collector/webstats/InternalLogDescriptor.java | 63 +++++++ .../webstats/InternalWebServerAccessLog.java | 17 ++ .../collector/webstats/LogDescriptorImpl.java | 166 +++++++++++++++++++ .../metrics/collector/webstats/LogMetadata.java | 2 +- .../collector/webstats/SanitizeWeblogs.java | 4 - .../collector/webstats/WebServerAccessLogImpl.java | 173 ++++++++++++++++++++ .../collector/webstats/WebServerAccessLogLine.java | 182 +++++++++++++++++++++ 9 files changed, 647 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java index 848fa2e..0f862b4 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java +++ b/src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java @@ -4,8 +4,8 @@ package org.torproject.metrics.collector.persist; import org.torproject.descriptor.WebServerAccessLog; -import org.torproject.descriptor.log.InternalWebServerAccessLog; import org.torproject.metrics.collector.webstats.FileType; +import org.torproject.metrics.collector.webstats.InternalWebServerAccessLog; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/torproject/metrics/collector/webstats/FileType.java b/src/main/java/org/torproject/metrics/collector/webstats/FileType.java index 79dcf21..15b1e00 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/FileType.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/FileType.java @@ -12,6 +12,8 @@ import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.InputStream; import java.io.OutputStream; @@ -69,10 +71,52 @@ public enum FileType { } /** + * Compresses the given bytes in memory and returns the compressed bytes. + */ + public byte[] compress(byte[] bytes) throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (OutputStream os = this.outputStream(baos)) { + os.write(bytes); + os.flush(); + } + return baos.toByteArray(); + } + + /** + * Compresses the given InputStream and returns an OutputStream. + */ + public OutputStream compress(OutputStream os) throws Exception { + return this.outputStream(os); + } + + /** * Decompresses the given InputStream and returns an OutputStream. */ public InputStream decompress(InputStream is) throws Exception { return this.inputStream(is); } + + /** + * Decompresses the given bytes in memory and returns the decompressed bytes. + * + * @since 2.2.0 + */ + public byte[] decompress(byte[] bytes) throws Exception { + if (0 == bytes.length) { + return bytes; + } + try (InputStream is + = this.inputStream(new ByteArrayInputStream(bytes)); + ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + int readByte = is.read(); + while (readByte > 0) { + baos.write(readByte); + readByte = is.read(); + } + baos.flush(); + return baos.toByteArray(); + } + } + } diff --git a/src/main/java/org/torproject/metrics/collector/webstats/InternalLogDescriptor.java b/src/main/java/org/torproject/metrics/collector/webstats/InternalLogDescriptor.java new file mode 100644 index 0000000..3a8a1f0 --- /dev/null +++ b/src/main/java/org/torproject/metrics/collector/webstats/InternalLogDescriptor.java @@ -0,0 +1,63 @@ +/* Copyright 2017--2018 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.collector.webstats; + +import org.torproject.descriptor.DescriptorParseException; +import org.torproject.descriptor.LogDescriptor; + +/** + * This interface provides methods for internal use only. + * + * @since 2.2.0 + */ +public interface InternalLogDescriptor extends LogDescriptor { + + /** Logfile name parts separator. */ + String SEP = "_"; + + /** + * Validate log lines. + * + * @since 2.2.0 + */ + void validate() throws DescriptorParseException; + + /** + * Set the {@code Validator} that will perform the validation on log + * lines. + * + * Usually set by the implementing class. + * + * @since 2.2.0 + */ + void setValidator(Validator validator); + + /** + * Set the descriptor's bytes. + * + * @since 2.2.0 + */ + void setRawDescriptorBytes(byte[] bytes); + + /** Return the descriptor's preferred compression. */ + String getCompressionType(); + + /** + * Provides a single function for validating a single log line. + * + * @since 2.2.0 + */ + interface Validator { + + /** + * Verifies a log line. + * + * @since 2.2.0 + */ + boolean validate(String line); + + } + +} + diff --git a/src/main/java/org/torproject/metrics/collector/webstats/InternalWebServerAccessLog.java b/src/main/java/org/torproject/metrics/collector/webstats/InternalWebServerAccessLog.java new file mode 100644 index 0000000..817b8d5 --- /dev/null +++ b/src/main/java/org/torproject/metrics/collector/webstats/InternalWebServerAccessLog.java @@ -0,0 +1,17 @@ +/* Copyright 2018 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.collector.webstats; + +/** + * This interface provides methods for internal use only. + * + * @since 2.2.0 + */ +public interface InternalWebServerAccessLog extends InternalLogDescriptor { + + /** The log's name should include this string. */ + String MARKER = "access.log"; + +} + diff --git a/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java b/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java new file mode 100644 index 0000000..d13c85a --- /dev/null +++ b/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java @@ -0,0 +1,166 @@ +/* Copyright 2017--2018 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.collector.webstats; + +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.DescriptorParseException; +import org.torproject.descriptor.LogDescriptor; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * Base class for log descriptors. + * + * @since 2.2.0 + */ +public abstract class LogDescriptorImpl + implements LogDescriptor, InternalLogDescriptor { + + /** The log's file name should contain this string. */ + public static final String MARKER = ".log"; + + private static final int unrecognizedLinesLimit = 3; + + private static final Logger log + = LoggerFactory.getLogger(LogDescriptorImpl.class); + + private static Pattern filenamePattern = Pattern.compile( + "(?:\\S*)" + MARKER + SEP + "(?:[0-9a-zA-Z]*)(?:\\.?)([a-zA-Z2]*)"); + + private final File descriptorFile; + + /** Byte array for plain, i.e. uncompressed, log data. */ + private byte[] logBytes; + + private FileType fileType; + + private List<String> unrecognizedLines = new ArrayList<>(); + + private Validator validator = (String line) -> true; + + /** + * This constructor performs basic operations on the given bytes. + * + * An unknown compression type (see {@link #getCompressionType}) + * is interpreted as missing compression. In this case the bytes + * will be compressed to the given compression type. + * + * @since 2.2.0 + */ + protected LogDescriptorImpl(byte[] logBytes, File descriptorFile, + String logName, FileType defaultCompression) + throws DescriptorParseException { + this.logBytes = logBytes; + this.descriptorFile = descriptorFile; + try { + Matcher mat = filenamePattern.matcher(logName); + if (!mat.find()) { + throw new DescriptorParseException( + "Log file name doesn't comply to standard: " + logName); + } + this.fileType = FileType.findType(mat.group(1).toUpperCase()); + if (FileType.PLAIN == this.fileType) { + this.fileType = defaultCompression; + this.logBytes = this.fileType.compress(this.logBytes); + } + } catch (Exception ex) { + throw new DescriptorParseException("Cannot parse file " + + logName + " from file " + descriptorFile.getName(), ex); + } + } + + @Override + public InputStream decompressedByteStream() throws DescriptorParseException { + try { + return this.fileType.decompress(new ByteArrayInputStream(this.logBytes)); + } catch (Exception ex) { + throw new DescriptorParseException("Cannot provide deflated stream of " + + this.descriptorFile + ".", ex); + } + } + + @Override + public void validate() throws DescriptorParseException { + try (BufferedReader br = new BufferedReader( + new InputStreamReader(decompressedByteStream()))) { + this.unrecognizedLines.addAll(br.lines().parallel().filter((line) + -> null != line && !line.isEmpty() && !validator.validate(line)) + .limit(unrecognizedLinesLimit).collect(Collectors.toList())); + } catch (Exception ex) { + throw new DescriptorParseException("Cannot validate log lines.", ex); + } + } + + /** + * Assemble a LogDescriptor. + * + * @since 2.2.0 + */ + public static List<Descriptor> parse(byte[] logBytes, + File descriptorFile, String logName) throws DescriptorParseException { + if (logName.contains(InternalWebServerAccessLog.MARKER)) { + return Arrays.asList(new Descriptor[]{ + new WebServerAccessLogImpl(logBytes, descriptorFile, logName)}); + } else { + throw new DescriptorParseException("Cannot parse file " + logName + + " from file " + descriptorFile.getName()); + } + } + + @Override + public void setValidator(Validator validator) { + this.validator = validator; + } + + @Override + public String getCompressionType() { + return this.fileType.name().toLowerCase(); + } + + @Override + public byte[] getRawDescriptorBytes() { + return this.logBytes; + } + + @Override + public void setRawDescriptorBytes(byte[] bytes) { + this.logBytes = bytes; + } + + @Override + public int getRawDescriptorLength() { + return this.logBytes.length; + } + + @Override + public List<String> getAnnotations() { + return Collections.emptyList(); + } + + @Override + public List<String> getUnrecognizedLines() { + return this.unrecognizedLines; + } + + @Override + public File getDescriptorFile() { + return descriptorFile; + } + +} + diff --git a/src/main/java/org/torproject/metrics/collector/webstats/LogMetadata.java b/src/main/java/org/torproject/metrics/collector/webstats/LogMetadata.java index b04fcb5..879e8d7 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/LogMetadata.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/LogMetadata.java @@ -3,7 +3,7 @@ package org.torproject.metrics.collector.webstats; -import static org.torproject.descriptor.log.WebServerAccessLogImpl.MARKER; +import static org.torproject.metrics.collector.webstats.WebServerAccessLogImpl.MARKER; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java index 5c001b6..74bd741 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java @@ -12,10 +12,6 @@ import static java.util.stream.Collectors.summingLong; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.Method; import org.torproject.descriptor.WebServerAccessLog; -import org.torproject.descriptor.log.InternalLogDescriptor; -import org.torproject.descriptor.log.InternalWebServerAccessLog; -import org.torproject.descriptor.log.WebServerAccessLogImpl; -import org.torproject.descriptor.log.WebServerAccessLogLine; import org.torproject.metrics.collector.conf.Configuration; import org.torproject.metrics.collector.conf.Key; import org.torproject.metrics.collector.conf.SourceType; diff --git a/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java b/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java new file mode 100644 index 0000000..2ca608a --- /dev/null +++ b/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java @@ -0,0 +1,173 @@ +/* Copyright 2017--2018 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.collector.webstats; + +import org.torproject.descriptor.DescriptorParseException; +import org.torproject.descriptor.WebServerAccessLog; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.File; +import java.io.InputStreamReader; +import java.nio.file.Paths; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +/** + * Implementation of web server access log descriptors. + * + * Defines sanitization and validation for web server access logs. + * + * @since 2.2.0 + */ +public class WebServerAccessLogImpl extends LogDescriptorImpl + implements InternalWebServerAccessLog, WebServerAccessLog { + + private static final Logger log + = LoggerFactory.getLogger(WebServerAccessLogImpl.class); + + /** The log's name should include this string. */ + public static final String MARKER = InternalWebServerAccessLog.MARKER; + + /** The mandatory web server log descriptor file name pattern. */ + public static final Pattern filenamePattern + = Pattern.compile("(\\S*)" + SEP + "(\\S*)" + SEP + "" + MARKER + + SEP + "(\\d*)(?:\\.?)([a-zA-Z]*)"); + + private final String physicalHost; + + private final String virtualHost; + + private final LocalDate logDate; + + private boolean validate = true; + + /** + * Creates a WebServerAccessLog from the given bytes and filename. + * + * The given bytes are read, whereas the file is not read. + * + * The path of the given file has to be compliant to the following + * naming pattern + * {@code + * <virtualHost>-<physicalHost>-access.log-<yyyymmdd>.<compression>}, + * where an unknown compression type (see {@link #getCompressionType}) + * is interpreted as missing compression. In this case the bytes + * will be compressed to the default compression type. + * The immediate parent name is taken to be the physical host collecting the + * logs. + */ + protected WebServerAccessLogImpl(byte[] logBytes, File file, String logName) + throws DescriptorParseException { + this(logBytes, file, logName, FileType.XZ); + } + + /** For internal use only. */ + public WebServerAccessLogImpl(byte[] bytes, String filename, + boolean validate) throws DescriptorParseException { + this(bytes, null, filename, FileType.XZ, validate); + } + + /** For internal use only. */ + public WebServerAccessLogImpl(byte[] bytes, File sourceFile, String filename, + boolean validate) throws DescriptorParseException { + this(bytes, sourceFile, filename, FileType.XZ, validate); + } + + private WebServerAccessLogImpl(byte[] logBytes, File file, String logName, + FileType defaultCompression) throws DescriptorParseException { + this(logBytes, file, logName, defaultCompression, true); + } + + private WebServerAccessLogImpl(byte[] logBytes, File file, String logName, + FileType defaultCompression, boolean validate) + throws DescriptorParseException { + super(logBytes, file, logName, defaultCompression); + try { + String fn = Paths.get(logName).getFileName().toString(); + Matcher mat = filenamePattern.matcher(fn); + if (!mat.find()) { + throw new DescriptorParseException( + "WebServerAccessLog file name doesn't comply to standard: " + fn); + } + this.virtualHost = mat.group(1); + this.physicalHost = mat.group(2); + if (null == this.virtualHost || null == this.physicalHost + || this.virtualHost.isEmpty() || this.physicalHost.isEmpty()) { + throw new DescriptorParseException( + "WebServerAccessLog file name doesn't comply to standard: " + fn); + } + String ymd = mat.group(3); + this.logDate = LocalDate.parse(ymd, DateTimeFormatter.BASIC_ISO_DATE); + this.setValidator((line) + -> WebServerAccessLogLine.makeLine(line).isValid()); + if (validate) { + this.validate(); + } + } catch (DescriptorParseException dpe) { + throw dpe; // escalate + } catch (Exception pe) { + throw new DescriptorParseException( + "Cannot parse WebServerAccessLog file: " + logName, pe); + } + } + + @Override + public String getPhysicalHost() { + return this.physicalHost; + } + + @Override + public String getVirtualHost() { + return this.virtualHost; + } + + @Override + public LocalDate getLogDate() { + return this.logDate; + } + + private static final int LISTLIMIT = Integer.MAX_VALUE / 2; + + /** Returns a stream of all valid log lines. */ + @Override + public Stream<WebServerAccessLog.Line> logLines() + throws DescriptorParseException { + try (BufferedReader br = new BufferedReader(new InputStreamReader( + this.decompressedByteStream()))) { + List<List<WebServerAccessLogLine>> lists = new ArrayList<>(); + List<WebServerAccessLogLine> currentList = new ArrayList<>(); + lists.add(currentList); + String lineStr = br.readLine(); + int count = 0; + while (null != lineStr) { + WebServerAccessLogLine wsal = WebServerAccessLogLine.makeLine(lineStr); + if (wsal.isValid()) { + currentList.add(wsal); + count++; + } + if (count >= LISTLIMIT) { + currentList = new ArrayList<>(); + lists.add(currentList); + count = 0; + } + lineStr = br.readLine(); + } + br.close(); + return lists.stream().flatMap(Collection::stream); + } catch (Exception ex) { + throw new DescriptorParseException("Cannot retrieve log lines.", ex); + } + } + +} + diff --git a/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogLine.java b/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogLine.java new file mode 100644 index 0000000..bc03c0a --- /dev/null +++ b/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogLine.java @@ -0,0 +1,182 @@ +/* Copyright 2018 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.collector.webstats; + +import org.torproject.descriptor.Method; +import org.torproject.descriptor.WebServerAccessLog; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class WebServerAccessLogLine implements WebServerAccessLog.Line { + + private static final Logger log = LoggerFactory + .getLogger(WebServerAccessLogLine.class); + + private static final String DATE_PATTERN = "dd/MMM/yyyy"; + private static final String DASH = "-"; + + private static final DateTimeFormatter dateTimeFormatter + = DateTimeFormatter.ofPattern(DATE_PATTERN + ":HH:mm:ss xxxx"); + + private static Pattern logLinePattern = Pattern.compile( + "^((?:\\d{1,3}\\.){3}\\d{1,3}) (\\S+) (\\S+) " + + "\\[([\\w/]+)([\\w:]+)(\\s[+\\-]\\d{4})\\] " + + "\"([A-Z]+) ([^\"]+) ([A-Z]+/\\d\\.\\d)\" " + + "(\\d{3}) (\\d+|-)(.*)"); + + private static Map<String, String> ipMap + = Collections.synchronizedMap(new HashMap<>()); + private static Map<LocalDate, LocalDate> dateMap + = Collections.synchronizedMap(new HashMap<>(500)); + private static Map<String, String> protocolMap + = Collections.synchronizedMap(new HashMap<>()); + private static Map<String, String> requestMap + = Collections.synchronizedMap(new HashMap<>(50_000)); + + private String ip; + private int response; + private String request; + private Method method; + private LocalDate date; + private int size = -1; + private boolean valid = false; + private String protocol; + + /** Returns a log line string. Possibly empty. */ + @Override + public String toLogString() { + if (!this.valid) { + return ""; + } + return toString(); + } + + @Override + public String toString() { + return String.format("%s - - [%s:00:00:00 +0000] \"%s %s %s\" %d %s", + this.ip, this.getDateString(), this.method.name(), this.request, + this.protocol, this.response, this.size < 0 ? DASH : this.size); + } + + /** Only used internally during sanitization. + * Returns the string of the date using 'dd/MMM/yyyy' format. */ + public String getDateString() { + return this.date.format(DateTimeFormatter.ofPattern(DATE_PATTERN)); + } + + @Override + public String getIp() { + return this.ip; + } + + /** Only used internally during sanitization. */ + public void setIp(String ip) { + this.ip = fromMap(ip, ipMap); + } + + @Override + public Method getMethod() { + return this.method; + } + + @Override + public String getProtocol() { + return this.protocol; + } + + @Override + public String getRequest() { + return this.request; + } + + @Override + public Optional<Integer> getSize() { + return this.size < 0 ? Optional.empty() : Optional.of(this.size); + } + + @Override + public int getResponse() { + return this.response; + } + + /** Only used internally during sanitization. */ + public void setRequest(String request) { + this.request = fromMap(request, requestMap); + } + + @Override + public LocalDate getDate() { + return this.date; + } + + @Override + public boolean isValid() { + return this.valid; + } + + /** Creates a Line from a string. */ + public static WebServerAccessLogLine makeLine(String line) { + WebServerAccessLogLine res = new WebServerAccessLogLine(); + try { + Matcher mat = logLinePattern.matcher(line); + if (mat.find()) { + res.response = Integer.valueOf(mat.group(10)); + res.method = Method.valueOf(mat.group(7)); + String dateTimeString = mat.group(4) + mat.group(5) + mat.group(6); + res.date = fromMap(ZonedDateTime.parse(dateTimeString, + dateTimeFormatter).withZoneSameInstant(ZoneOffset.UTC) + .toLocalDate(), dateMap); + res.ip = fromMap(mat.group(1), ipMap); + res.request = fromMap(mat.group(8), requestMap); + res.protocol = fromMap(mat.group(9), protocolMap); + if (DASH.equals(mat.group(11))) { + res.size = -1; + } else { + res.size = Integer.valueOf(mat.group(11)); + } + res.valid = true; + } + } catch (Throwable th) { + log.debug("Unmatchable line: '{}'.", line, th); + return new WebServerAccessLogLine(); + } + return res; + } + + private static <T> T fromMap(T val, Map<T, T> map) { + synchronized (map) { + map.putIfAbsent(Objects.requireNonNull(val), val); + return map.get(val); + } + } + + @Override + public boolean equals(Object other) { + if (other instanceof WebServerAccessLogLine) { + return this.toLogString() + .equals(((WebServerAccessLogLine)other).toLogString()); + } + return false; + } + + @Override + public int hashCode() { + return this.toLogString().hashCode(); + } + +} + [View Less]

1 0

[collector/master] Remove dependency on metrics-lib's log package (2/4).
by karsten＠torproject.org 02 Dec '19

02 Dec '19

commit ea1b1b4f6ab11e7ac933b0e00d5f8c040e4cc11e Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sat Nov 23 17:43:45 2019 +0100 Remove dependency on metrics-lib's log package (2/4). - Remove unused code. --- .../persist/WebServerAccessLogPersistence.java | 5 -- .../collector/webstats/InternalLogDescriptor.java | 35 -------------- .../collector/webstats/LogDescriptorImpl.java | 53 +--------------------- .../metrics/collector/webstats/LogFileMap.java … [View More] | 26 ----------- .../collector/webstats/SanitizeWeblogs.java | 2 +- .../collector/webstats/WebServerAccessLogImpl.java | 38 +--------------- 6 files changed, 4 insertions(+), 155 deletions(-) diff --git a/src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java index 0f862b4..6afb97e 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java +++ b/src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java @@ -7,9 +7,6 @@ import org.torproject.descriptor.WebServerAccessLog; import org.torproject.metrics.collector.webstats.FileType; import org.torproject.metrics.collector.webstats.InternalWebServerAccessLog; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.nio.file.Paths; import java.nio.file.StandardOpenOption; import java.time.format.DateTimeFormatter; @@ -19,8 +16,6 @@ public class WebServerAccessLogPersistence public static final String SEP = InternalWebServerAccessLog.SEP; public static final FileType COMPRESSION = FileType.XZ; - private static final Logger log - = LoggerFactory.getLogger(WebServerAccessLogPersistence.class); private DateTimeFormatter yearPattern = DateTimeFormatter.ofPattern("yyyy"); private DateTimeFormatter monthPattern = DateTimeFormatter.ofPattern("MM"); diff --git a/src/main/java/org/torproject/metrics/collector/webstats/InternalLogDescriptor.java b/src/main/java/org/torproject/metrics/collector/webstats/InternalLogDescriptor.java index 3a8a1f0..ac7d689 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/InternalLogDescriptor.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/InternalLogDescriptor.java @@ -3,7 +3,6 @@ package org.torproject.metrics.collector.webstats; -import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.LogDescriptor; /** @@ -17,23 +16,6 @@ public interface InternalLogDescriptor extends LogDescriptor { String SEP = "_"; /** - * Validate log lines. - * - * @since 2.2.0 - */ - void validate() throws DescriptorParseException; - - /** - * Set the {@code Validator} that will perform the validation on log - * lines. - * - * Usually set by the implementing class. - * - * @since 2.2.0 - */ - void setValidator(Validator validator); - - /** * Set the descriptor's bytes. * * @since 2.2.0 @@ -42,22 +24,5 @@ public interface InternalLogDescriptor extends LogDescriptor { /** Return the descriptor's preferred compression. */ String getCompressionType(); - - /** - * Provides a single function for validating a single log line. - * - * @since 2.2.0 - */ - interface Validator { - - /** - * Verifies a log line. - * - * @since 2.2.0 - */ - boolean validate(String line); - - } - } diff --git a/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java b/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java index d13c85a..c0b0c69 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java @@ -3,25 +3,17 @@ package org.torproject.metrics.collector.webstats; -import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.LogDescriptor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.File; import java.io.InputStream; -import java.io.InputStreamReader; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.stream.Collectors; /** * Base class for log descriptors. @@ -34,11 +26,6 @@ public abstract class LogDescriptorImpl /** The log's file name should contain this string. */ public static final String MARKER = ".log"; - private static final int unrecognizedLinesLimit = 3; - - private static final Logger log - = LoggerFactory.getLogger(LogDescriptorImpl.class); - private static Pattern filenamePattern = Pattern.compile( "(?:\\S*)" + MARKER + SEP + "(?:[0-9a-zA-Z]*)(?:\\.?)([a-zA-Z2]*)"); @@ -51,8 +38,6 @@ public abstract class LogDescriptorImpl private List<String> unrecognizedLines = new ArrayList<>(); - private Validator validator = (String line) -> true; - /** * This constructor performs basic operations on the given bytes. * @@ -63,8 +48,7 @@ public abstract class LogDescriptorImpl * @since 2.2.0 */ protected LogDescriptorImpl(byte[] logBytes, File descriptorFile, - String logName, FileType defaultCompression) - throws DescriptorParseException { + String logName) throws DescriptorParseException { this.logBytes = logBytes; this.descriptorFile = descriptorFile; try { @@ -75,7 +59,7 @@ public abstract class LogDescriptorImpl } this.fileType = FileType.findType(mat.group(1).toUpperCase()); if (FileType.PLAIN == this.fileType) { - this.fileType = defaultCompression; + this.fileType = FileType.XZ; this.logBytes = this.fileType.compress(this.logBytes); } } catch (Exception ex) { @@ -95,39 +79,6 @@ public abstract class LogDescriptorImpl } @Override - public void validate() throws DescriptorParseException { - try (BufferedReader br = new BufferedReader( - new InputStreamReader(decompressedByteStream()))) { - this.unrecognizedLines.addAll(br.lines().parallel().filter((line) - -> null != line && !line.isEmpty() && !validator.validate(line)) - .limit(unrecognizedLinesLimit).collect(Collectors.toList())); - } catch (Exception ex) { - throw new DescriptorParseException("Cannot validate log lines.", ex); - } - } - - /** - * Assemble a LogDescriptor. - * - * @since 2.2.0 - */ - public static List<Descriptor> parse(byte[] logBytes, - File descriptorFile, String logName) throws DescriptorParseException { - if (logName.contains(InternalWebServerAccessLog.MARKER)) { - return Arrays.asList(new Descriptor[]{ - new WebServerAccessLogImpl(logBytes, descriptorFile, logName)}); - } else { - throw new DescriptorParseException("Cannot parse file " + logName - + " from file " + descriptorFile.getName()); - } - } - - @Override - public void setValidator(Validator validator) { - this.validator = validator; - } - - @Override public String getCompressionType() { return this.fileType.name().toLowerCase(); } diff --git a/src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java b/src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java index 2036b1f..d3e7dcd 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java @@ -15,7 +15,6 @@ import java.nio.file.attribute.BasicFileAttributes; import java.time.LocalDate; import java.util.Optional; import java.util.TreeMap; -import java.util.stream.Stream; /** * Processes the given path and stores metadata for log files. @@ -74,30 +73,5 @@ public class LogFileMap k -> new TreeMap<>()); physicalHosts.put(metadata.date, metadata); } - - /** - * Takes the given metadata and returns the LogMetadata for the entry - * of the next day. - */ - public Optional<LogMetadata> nextDayLogFor(LogMetadata metadata) { - TreeMap<String, TreeMap<LocalDate, LogMetadata>> virtualHosts - = this.get(metadata.virtualHost); - if (null == virtualHosts) { - return Optional.empty(); - } - TreeMap<LocalDate, LogMetadata> physicalHosts - = virtualHosts.get(metadata.physicalHost); - if (null == physicalHosts) { - return Optional.empty(); - } - return Optional.ofNullable(physicalHosts.get(metadata.date.plusDays(1))); - } - - /** Returns a stream of all contained log metadata. */ - public Stream<LogMetadata> metadataStream() { - return this.values().stream() - .flatMap((virtualHosts) -> virtualHosts.values().stream()) - .flatMap((physicalHosts) -> physicalHosts.values().stream()); - } } diff --git a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java index 74bd741..b2b4c49 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java @@ -147,7 +147,7 @@ public class SanitizeWeblogs extends CollecTorMain { WebServerAccessLogPersistence walp = new WebServerAccessLogPersistence( new WebServerAccessLogImpl(toCompressedBytes(retainedLines), - new File(name), name, false)); + new File(name), name)); log.debug("Storing {}.", name); walp.storeOut(this.outputPathName); walp.storeRecent(this.recentPathName); diff --git a/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java b/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java index 2ca608a..548fbf6 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java @@ -6,9 +6,6 @@ package org.torproject.metrics.collector.webstats; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.WebServerAccessLog; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.BufferedReader; import java.io.File; import java.io.InputStreamReader; @@ -32,9 +29,6 @@ import java.util.stream.Stream; public class WebServerAccessLogImpl extends LogDescriptorImpl implements InternalWebServerAccessLog, WebServerAccessLog { - private static final Logger log - = LoggerFactory.getLogger(WebServerAccessLogImpl.class); - /** The log's name should include this string. */ public static final String MARKER = InternalWebServerAccessLog.MARKER; @@ -49,8 +43,6 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl private final LocalDate logDate; - private boolean validate = true; - /** * Creates a WebServerAccessLog from the given bytes and filename. * @@ -68,30 +60,7 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl */ protected WebServerAccessLogImpl(byte[] logBytes, File file, String logName) throws DescriptorParseException { - this(logBytes, file, logName, FileType.XZ); - } - - /** For internal use only. */ - public WebServerAccessLogImpl(byte[] bytes, String filename, - boolean validate) throws DescriptorParseException { - this(bytes, null, filename, FileType.XZ, validate); - } - - /** For internal use only. */ - public WebServerAccessLogImpl(byte[] bytes, File sourceFile, String filename, - boolean validate) throws DescriptorParseException { - this(bytes, sourceFile, filename, FileType.XZ, validate); - } - - private WebServerAccessLogImpl(byte[] logBytes, File file, String logName, - FileType defaultCompression) throws DescriptorParseException { - this(logBytes, file, logName, defaultCompression, true); - } - - private WebServerAccessLogImpl(byte[] logBytes, File file, String logName, - FileType defaultCompression, boolean validate) - throws DescriptorParseException { - super(logBytes, file, logName, defaultCompression); + super(logBytes, file, logName); try { String fn = Paths.get(logName).getFileName().toString(); Matcher mat = filenamePattern.matcher(fn); @@ -108,11 +77,6 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl } String ymd = mat.group(3); this.logDate = LocalDate.parse(ymd, DateTimeFormatter.BASIC_ISO_DATE); - this.setValidator((line) - -> WebServerAccessLogLine.makeLine(line).isValid()); - if (validate) { - this.validate(); - } } catch (DescriptorParseException dpe) { throw dpe; // escalate } catch (Exception pe) { [View Less]

1 0

[collector/master] Remove dependency on metrics-lib's log package (4/4).
by karsten＠torproject.org 02 Dec '19

02 Dec '19

commit 8263cc7bdbb0a632f12a84fb2051dd9a25c28142 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sat Nov 23 18:07:41 2019 +0100 Remove dependency on metrics-lib's log package (4/4). - Remove package-internal abstract class. --- .../collector/webstats/LogDescriptorImpl.java | 117 --------------------- .../collector/webstats/SanitizeWeblogs.java | 2 +- .../collector/webstats/WebServerAccessLogImpl.java | 82 +++++++++++++-- 3 files changed, 73 … [View More]insertions(+), 128 deletions(-) diff --git a/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java b/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java deleted file mode 100644 index e20be27..0000000 --- a/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java +++ /dev/null @@ -1,117 +0,0 @@ -/* Copyright 2017--2018 The Tor Project - * See LICENSE for licensing information */ - -package org.torproject.metrics.collector.webstats; - -import org.torproject.descriptor.DescriptorParseException; -import org.torproject.descriptor.LogDescriptor; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Base class for log descriptors. - * - * @since 2.2.0 - */ -public abstract class LogDescriptorImpl implements LogDescriptor { - - /** Logfile name parts separator. */ - public static final String SEP = "_"; - - /** The log's file name should contain this string. */ - public static final String MARKER = ".log"; - - private static Pattern filenamePattern = Pattern.compile( - "(?:\\S*)" + MARKER + SEP + "(?:[0-9a-zA-Z]*)(?:\\.?)([a-zA-Z2]*)"); - - private final File descriptorFile; - - /** Byte array for plain, i.e. uncompressed, log data. */ - private byte[] logBytes; - - private FileType fileType; - - private List<String> unrecognizedLines = new ArrayList<>(); - - /** - * This constructor performs basic operations on the given bytes. - * - * An unknown compression type (see {@link #getCompressionType}) - * is interpreted as missing compression. In this case the bytes - * will be compressed to the given compression type. - * - * @since 2.2.0 - */ - protected LogDescriptorImpl(byte[] logBytes, File descriptorFile, - String logName) throws DescriptorParseException { - this.logBytes = logBytes; - this.descriptorFile = descriptorFile; - try { - Matcher mat = filenamePattern.matcher(logName); - if (!mat.find()) { - throw new DescriptorParseException( - "Log file name doesn't comply to standard: " + logName); - } - this.fileType = FileType.findType(mat.group(1).toUpperCase()); - if (FileType.PLAIN == this.fileType) { - this.fileType = FileType.XZ; - this.logBytes = this.fileType.compress(this.logBytes); - } - } catch (Exception ex) { - throw new DescriptorParseException("Cannot parse file " - + logName + " from file " + descriptorFile.getName(), ex); - } - } - - @Override - public InputStream decompressedByteStream() throws DescriptorParseException { - try { - return this.fileType.decompress(new ByteArrayInputStream(this.logBytes)); - } catch (Exception ex) { - throw new DescriptorParseException("Cannot provide deflated stream of " - + this.descriptorFile + ".", ex); - } - } - - public String getCompressionType() { - return this.fileType.name().toLowerCase(); - } - - @Override - public byte[] getRawDescriptorBytes() { - return this.logBytes; - } - - public void setRawDescriptorBytes(byte[] bytes) { - this.logBytes = bytes; - } - - @Override - public int getRawDescriptorLength() { - return this.logBytes.length; - } - - @Override - public List<String> getAnnotations() { - return Collections.emptyList(); - } - - @Override - public List<String> getUnrecognizedLines() { - return this.unrecognizedLines; - } - - @Override - public File getDescriptorFile() { - return descriptorFile; - } - -} - diff --git a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java index 51318f8..fc7c64f 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java @@ -135,7 +135,7 @@ public class SanitizeWeblogs extends CollecTorMain { private void storeSortedAndForget(String virtualHost, String physicalHost, LocalDate date, Map<String, Long> lineCounts) { - String name = new StringJoiner(LogDescriptorImpl.SEP) + String name = new StringJoiner(WebServerAccessLogImpl.SEP) .add(virtualHost).add(physicalHost) .add(WebServerAccessLogImpl.MARKER) .add(date.format(DateTimeFormatter.BASIC_ISO_DATE)) diff --git a/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java b/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java index 4ec9e39..af77c94 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java @@ -7,13 +7,15 @@ import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.WebServerAccessLog; import java.io.BufferedReader; +import java.io.ByteArrayInputStream; import java.io.File; +import java.io.InputStream; import java.io.InputStreamReader; -import java.nio.file.Paths; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -26,8 +28,10 @@ import java.util.stream.Stream; * * @since 2.2.0 */ -public class WebServerAccessLogImpl extends LogDescriptorImpl - implements WebServerAccessLog { +public class WebServerAccessLogImpl implements WebServerAccessLog { + + /** Logfile name parts separator. */ + public static final String SEP = "_"; /** The log's name should include this string. */ public static final String MARKER = "access.log"; @@ -37,6 +41,15 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl = Pattern.compile("(\\S*)" + SEP + "(\\S*)" + SEP + "" + MARKER + SEP + "(\\d*)(?:\\.?)([a-zA-Z]*)"); + private final File descriptorFile; + + /** Byte array for plain, i.e. uncompressed, log data. */ + private byte[] logBytes; + + private FileType fileType; + + private List<String> unrecognizedLines = new ArrayList<>(); + private final String physicalHost; private final String virtualHost; @@ -58,25 +71,31 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl * The immediate parent name is taken to be the physical host collecting the * logs. */ - protected WebServerAccessLogImpl(byte[] logBytes, File file, String logName) - throws DescriptorParseException { - super(logBytes, file, logName); + protected WebServerAccessLogImpl(byte[] logBytes, File descriptorFile, + String logName) throws DescriptorParseException { + this.logBytes = logBytes; + this.descriptorFile = descriptorFile; try { - String fn = Paths.get(logName).getFileName().toString(); - Matcher mat = filenamePattern.matcher(fn); + Matcher mat = filenamePattern.matcher(logName); if (!mat.find()) { throw new DescriptorParseException( - "WebServerAccessLog file name doesn't comply to standard: " + fn); + "Log file name doesn't comply to standard: " + logName); } this.virtualHost = mat.group(1); this.physicalHost = mat.group(2); if (null == this.virtualHost || null == this.physicalHost || this.virtualHost.isEmpty() || this.physicalHost.isEmpty()) { throw new DescriptorParseException( - "WebServerAccessLog file name doesn't comply to standard: " + fn); + "WebServerAccessLog file name doesn't comply to standard: " + + logName); } String ymd = mat.group(3); this.logDate = LocalDate.parse(ymd, DateTimeFormatter.BASIC_ISO_DATE); + this.fileType = FileType.findType(mat.group(4).toUpperCase()); + if (FileType.PLAIN == this.fileType) { + this.fileType = FileType.XZ; + this.logBytes = this.fileType.compress(this.logBytes); + } } catch (DescriptorParseException dpe) { throw dpe; // escalate } catch (Exception pe) { @@ -86,6 +105,49 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl } @Override + public InputStream decompressedByteStream() throws DescriptorParseException { + try { + return this.fileType.decompress(new ByteArrayInputStream(this.logBytes)); + } catch (Exception ex) { + throw new DescriptorParseException("Cannot provide deflated stream of " + + this.descriptorFile + ".", ex); + } + } + + public String getCompressionType() { + return this.fileType.name().toLowerCase(); + } + + @Override + public byte[] getRawDescriptorBytes() { + return this.logBytes; + } + + public void setRawDescriptorBytes(byte[] bytes) { + this.logBytes = bytes; + } + + @Override + public int getRawDescriptorLength() { + return this.logBytes.length; + } + + @Override + public List<String> getAnnotations() { + return Collections.emptyList(); + } + + @Override + public List<String> getUnrecognizedLines() { + return this.unrecognizedLines; + } + + @Override + public File getDescriptorFile() { + return descriptorFile; + } + + @Override public String getPhysicalHost() { return this.physicalHost; } [View Less]

1 0

[collector/master] Remove dependency on metrics-lib's log package (3/4).
by karsten＠torproject.org 02 Dec '19

02 Dec '19

commit c11b61465a644940559b97c93a769fda84287970 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sat Nov 23 17:55:17 2019 +0100 Remove dependency on metrics-lib's log package (3/4). - Remove package-internal interfaces InternalLogDescriptor and InternalWebServerAccessLog. --- .../persist/WebServerAccessLogPersistence.java | 4 ++-- .../collector/webstats/InternalLogDescriptor.java | 28 ---------------------- .../webstats/InternalWebServerAccessLog.… [View More]java | 17 ------------- .../collector/webstats/LogDescriptorImpl.java | 8 +++---- .../collector/webstats/SanitizeWeblogs.java | 4 ++-- .../collector/webstats/WebServerAccessLogImpl.java | 4 ++-- 6 files changed, 10 insertions(+), 55 deletions(-) diff --git a/src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java index 6afb97e..0001957 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java +++ b/src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java @@ -5,7 +5,7 @@ package org.torproject.metrics.collector.persist; import org.torproject.descriptor.WebServerAccessLog; import org.torproject.metrics.collector.webstats.FileType; -import org.torproject.metrics.collector.webstats.InternalWebServerAccessLog; +import org.torproject.metrics.collector.webstats.WebServerAccessLogImpl; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; @@ -14,7 +14,7 @@ import java.time.format.DateTimeFormatter; public class WebServerAccessLogPersistence extends DescriptorPersistence<WebServerAccessLog> { - public static final String SEP = InternalWebServerAccessLog.SEP; + public static final String SEP = WebServerAccessLogImpl.SEP; public static final FileType COMPRESSION = FileType.XZ; private DateTimeFormatter yearPattern = DateTimeFormatter.ofPattern("yyyy"); diff --git a/src/main/java/org/torproject/metrics/collector/webstats/InternalLogDescriptor.java b/src/main/java/org/torproject/metrics/collector/webstats/InternalLogDescriptor.java deleted file mode 100644 index ac7d689..0000000 --- a/src/main/java/org/torproject/metrics/collector/webstats/InternalLogDescriptor.java +++ /dev/null @@ -1,28 +0,0 @@ -/* Copyright 2017--2018 The Tor Project - * See LICENSE for licensing information */ - -package org.torproject.metrics.collector.webstats; - -import org.torproject.descriptor.LogDescriptor; - -/** - * This interface provides methods for internal use only. - * - * @since 2.2.0 - */ -public interface InternalLogDescriptor extends LogDescriptor { - - /** Logfile name parts separator. */ - String SEP = "_"; - - /** - * Set the descriptor's bytes. - * - * @since 2.2.0 - */ - void setRawDescriptorBytes(byte[] bytes); - - /** Return the descriptor's preferred compression. */ - String getCompressionType(); -} - diff --git a/src/main/java/org/torproject/metrics/collector/webstats/InternalWebServerAccessLog.java b/src/main/java/org/torproject/metrics/collector/webstats/InternalWebServerAccessLog.java deleted file mode 100644 index 817b8d5..0000000 --- a/src/main/java/org/torproject/metrics/collector/webstats/InternalWebServerAccessLog.java +++ /dev/null @@ -1,17 +0,0 @@ -/* Copyright 2018 The Tor Project - * See LICENSE for licensing information */ - -package org.torproject.metrics.collector.webstats; - -/** - * This interface provides methods for internal use only. - * - * @since 2.2.0 - */ -public interface InternalWebServerAccessLog extends InternalLogDescriptor { - - /** The log's name should include this string. */ - String MARKER = "access.log"; - -} - diff --git a/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java b/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java index c0b0c69..e20be27 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java @@ -20,8 +20,10 @@ import java.util.regex.Pattern; * * @since 2.2.0 */ -public abstract class LogDescriptorImpl - implements LogDescriptor, InternalLogDescriptor { +public abstract class LogDescriptorImpl implements LogDescriptor { + + /** Logfile name parts separator. */ + public static final String SEP = "_"; /** The log's file name should contain this string. */ public static final String MARKER = ".log"; @@ -78,7 +80,6 @@ public abstract class LogDescriptorImpl } } - @Override public String getCompressionType() { return this.fileType.name().toLowerCase(); } @@ -88,7 +89,6 @@ public abstract class LogDescriptorImpl return this.logBytes; } - @Override public void setRawDescriptorBytes(byte[] bytes) { this.logBytes = bytes; } diff --git a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java index b2b4c49..51318f8 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java @@ -135,9 +135,9 @@ public class SanitizeWeblogs extends CollecTorMain { private void storeSortedAndForget(String virtualHost, String physicalHost, LocalDate date, Map<String, Long> lineCounts) { - String name = new StringJoiner(InternalLogDescriptor.SEP) + String name = new StringJoiner(LogDescriptorImpl.SEP) .add(virtualHost).add(physicalHost) - .add(InternalWebServerAccessLog.MARKER) + .add(WebServerAccessLogImpl.MARKER) .add(date.format(DateTimeFormatter.BASIC_ISO_DATE)) .toString() + "." + FileType.XZ.name().toLowerCase(); log.debug("Storing {}.", name); diff --git a/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java b/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java index 548fbf6..4ec9e39 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java @@ -27,10 +27,10 @@ import java.util.stream.Stream; * @since 2.2.0 */ public class WebServerAccessLogImpl extends LogDescriptorImpl - implements InternalWebServerAccessLog, WebServerAccessLog { + implements WebServerAccessLog { /** The log's name should include this string. */ - public static final String MARKER = InternalWebServerAccessLog.MARKER; + public static final String MARKER = "access.log"; /** The mandatory web server log descriptor file name pattern. */ public static final Pattern filenamePattern [View Less]

1 0

[metrics-web/master] Remove dependency on metrics-lib's index package.
by karsten＠torproject.org 02 Dec '19

02 Dec '19

commit 267678c5742379ecbdfdef18ca362d6950fb09d2 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Nov 25 16:34:54 2019 +0100 Remove dependency on metrics-lib's index package. We shouldn't depend on metrics-lib's implementation classes but only on its provided interfaces. In this case, downloading and parsing an index.json file is something that we can easily build ourselves using a POJO and Jackson. Removing this dependency will make it much easier … [View More] to refactor metrics-lib. --- .../metrics/web/CollectorDirectoryProvider.java | 9 +- .../torproject/metrics/web/DirectoryListing.java | 135 +++++++++++++++++++-- .../metrics/web/DirectoryListingTest.java | 6 +- 3 files changed, 130 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/torproject/metrics/web/CollectorDirectoryProvider.java b/src/main/java/org/torproject/metrics/web/CollectorDirectoryProvider.java index 2960599..4e20ec5 100644 --- a/src/main/java/org/torproject/metrics/web/CollectorDirectoryProvider.java +++ b/src/main/java/org/torproject/metrics/web/CollectorDirectoryProvider.java @@ -3,8 +3,6 @@ package org.torproject.metrics.web; -import org.torproject.descriptor.index.IndexNode; - import java.util.List; import java.util.Map; import java.util.concurrent.Executors; @@ -54,9 +52,10 @@ public class CollectorDirectoryProvider implements Runnable { * produce directory listings as requested. */ @Override public void run() { - IndexNode indexNode; try { - indexNode = IndexNode.fetchIndex(this.host + "/index/index.json.gz"); + DirectoryListing directoryListing + = DirectoryListing.ofHostString(this.host); + this.index.set(directoryListing); } catch (Exception e) { /* If we failed to fetch the remote index.json this time, abort the * update and don't override what we possibly fetched last time. If this @@ -64,9 +63,7 @@ public class CollectorDirectoryProvider implements Runnable { * it's a permanent problem, we'll at least serve the last known files. * Unless it's a permanent problem right from when we started in which * case there's nothing we can do other than return 500. */ - return; } - this.index.set(new DirectoryListing(indexNode)); } } diff --git a/src/main/java/org/torproject/metrics/web/DirectoryListing.java b/src/main/java/org/torproject/metrics/web/DirectoryListing.java index ceeb2fe..1bb09c2 100644 --- a/src/main/java/org/torproject/metrics/web/DirectoryListing.java +++ b/src/main/java/org/torproject/metrics/web/DirectoryListing.java @@ -3,10 +3,18 @@ package org.torproject.metrics.web; -import org.torproject.descriptor.index.DirectoryNode; -import org.torproject.descriptor.index.FileNode; -import org.torproject.descriptor.index.IndexNode; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.net.URLConnection; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -27,20 +35,127 @@ public class DirectoryListing extends HashMap<String, List<String[]>> extractDirectoryListings(); } + /** + * Parsed {@code index.json} file, which can be the root node ("index node"), + * an inner node ("directory node"), or a leaf node ("file node"). + */ + private static class IndexNode implements Comparable<IndexNode> { + + /** + * Relative path from this node's parent node, or the CollecTor host's base + * URL if this is the root node. + */ + String path; + + /** + * List of file nodes available in this directory, or {@code null} if this + * is a leaf node. + */ + SortedSet<IndexNode> files; + + /** + * List of directory nodes in this directory, or {@code null} if this is a + * leaf node. + */ + SortedSet<IndexNode> directories; + + /** + * Size of the file in bytes if this is a leaf node, or {@code null} + * otherwise. + */ + Long size; + + /** + * Timestamp when this file was last modified using pattern + * {@code "YYYY-MM-DD HH:MM"} in the UTC timezone if this is a leaf node, or + * {@code null} otherwise. + */ + String lastModified; + + /** + * Compare two index nodes by their (relative) path in alphabetic order. + * + * @param other The other index node to compare to. + * @return Comparison result of the two node's paths. + */ + @Override + public int compareTo(IndexNode other) { + return this.path.compareTo(other.path); + } + } + + /** + * Timeout in milliseconds for reading from remote CollecTor host. + */ + private static final int READ_TIMEOUT = Integer.parseInt(System + .getProperty("sun.net.client.defaultReadTimeout", "60000")); + + /** + * Timeout in milliseconds for connecting to remote CollecTor host. + */ + private static final int CONNECT_TIMEOUT = Integer.parseInt(System + .getProperty("sun.net.client.defaultConnectTimeout", "60000")); + + /** + * Object mapper for parsing {@code index.json} files. + */ + private static ObjectMapper objectMapper = new ObjectMapper() + .setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE) + .setSerializationInclusion(JsonInclude.Include.NON_EMPTY) + .setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE) + .setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + /** + * Create a new instance by downloading an {@code index.json} file from the + * given CollecTor host and parsing its contents. + * + * @param host CollecTor host to download the {@code index.json} file from. + * @return Parsed directory listings. + * @throws IOException Thrown if downloading or parsing the {@code index.json} + * file fails. + */ + public static DirectoryListing ofHostString(String host) throws IOException { + String urlString = host + "/index/index.json.gz"; + URLConnection connection = new URL(urlString).openConnection(); + connection.setReadTimeout(READ_TIMEOUT); + connection.setConnectTimeout(CONNECT_TIMEOUT); + connection.connect(); + try (InputStream inputStream + = new GzipCompressorInputStream(connection.getInputStream())) { + return ofInputStream(inputStream); + } + } + + /** + * Create a new instance by parsing an {@code index.json} file from the + * given stream. + * + * @param inputStream Input stream providing (uncompressed) {@code index.json} + * file contents. + * @return Parsed directory listings. + * @throws IOException Thrown if parsing the {@code index.json} file fails. + */ + public static DirectoryListing ofInputStream(InputStream inputStream) + throws IOException { + IndexNode index = objectMapper.readValue(inputStream, IndexNode.class); + return new DirectoryListing(index); + } + /** Extracts directory listing from an index node by visiting all nodes. */ private void extractDirectoryListings() { - Map<DirectoryNode, String> directoryNodes = new HashMap<>(); + Map<IndexNode, String> directoryNodes = new HashMap<>(); this.put("/collector/", formatTableEntries("", "/", this.index.directories, this.index.files)); - for (DirectoryNode directory : this.index.directories) { + for (IndexNode directory : this.index.directories) { directoryNodes.put(directory, "/"); } while (!directoryNodes.isEmpty()) { - DirectoryNode currentDirectoryNode = + IndexNode currentDirectoryNode = directoryNodes.keySet().iterator().next(); String parentPath = directoryNodes.remove(currentDirectoryNode); if (null != currentDirectoryNode.directories) { - for (DirectoryNode subDirectoryNode + for (IndexNode subDirectoryNode : currentDirectoryNode.directories) { directoryNodes.put(subDirectoryNode, String.format("%s%s/", parentPath, currentDirectoryNode.path)); @@ -55,20 +170,20 @@ public class DirectoryListing extends HashMap<String, List<String[]>> /** Formats table entries for a given directory. */ private List<String[]> formatTableEntries(String parentPath, String path, - SortedSet<DirectoryNode> directories, SortedSet<FileNode> files) { + SortedSet<IndexNode> directories, SortedSet<IndexNode> files) { List<String[]> tableEntries = new ArrayList<>(); tableEntries.add(new String[] { "Parent Directory", String.format("/collector%s", parentPath.isEmpty() ? ".html" : parentPath), "", "" }); if (null != directories) { - for (DirectoryNode subDirectoryNode : directories) { + for (IndexNode subDirectoryNode : directories) { tableEntries.add(new String[] { subDirectoryNode.path, String.format("/collector%s%s%s/", parentPath, path, subDirectoryNode.path), "", "" }); } } if (null != files) { - for (FileNode fileNode : new TreeSet<>(files).descendingSet()) { + for (IndexNode fileNode : new TreeSet<>(files).descendingSet()) { tableEntries.add(new String[] { fileNode.path, String.format("%s%s%s%s", this.index.path, parentPath, path, fileNode.path), fileNode.lastModified, diff --git a/src/test/java/org/torproject/metrics/web/DirectoryListingTest.java b/src/test/java/org/torproject/metrics/web/DirectoryListingTest.java index a74df81..a3ef6fa 100644 --- a/src/test/java/org/torproject/metrics/web/DirectoryListingTest.java +++ b/src/test/java/org/torproject/metrics/web/DirectoryListingTest.java @@ -6,8 +6,6 @@ package org.torproject.metrics.web; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import org.torproject.descriptor.index.IndexNode; - import org.junit.Test; import java.io.ByteArrayInputStream; @@ -49,8 +47,8 @@ public class DirectoryListingTest { @Test public void testListing() throws Exception { - DirectoryListing dl = new DirectoryListing(IndexNode - .fetchIndex(new ByteArrayInputStream(jsonIndex.getBytes()))); + DirectoryListing dl = DirectoryListing.ofInputStream( + new ByteArrayInputStream(jsonIndex.getBytes())); assertEquals(4, dl.size()); for (String key : new String[]{"/collector/a1/", "/collector/", "/collector/a1/p2/", "/collector/a1/p1/"}) { [View Less]

1 0

[translation/tbmanual-contentspot_completed] https://gitweb.torproject.org/translation.git/commit/?h=tbmanual-contentspot_completed
by translation＠torproject.org 02 Dec '19

02 Dec '19

commit 4304f381cf18fded6775a2b2c8decb27a29c58fe Author: Translation commit bot <translation(a)torproject.org> Date: Mon Dec 2 13:50:19 2019 +0000 https://gitweb.torproject.org/translation.git/commit/?h=tbmanual-contentspo… --- contents+pt-BR.po | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contents+pt-BR.po b/contents+pt-BR.po index d54c7d4268..c26f6fa5bd 100644 --- a/contents+pt-BR.po +++ b/contents+pt-BR.po @@ -10,6 +10,7 @@ # m tk, 2019 # Caio Volpato … [View More]

1 0

[translation/tbmanual-contentspot] https://gitweb.torproject.org/translation.git/commit/?h=tbmanual-contentspot
by translation＠torproject.org 02 Dec '19

02 Dec '19

commit 15e902800a1bfd86cb30114c4376255206d9b431 Author: Translation commit bot <translation(a)torproject.org> Date: Mon Dec 2 13:50:10 2019 +0000 https://gitweb.torproject.org/translation.git/commit/?h=tbmanual-contentspot --- contents+pt-BR.po | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/contents+pt-BR.po b/contents+pt-BR.po index 7dbe7a7c1e..6b0791f5c3 100644 --- a/contents+pt-BR.po +++ b/contents+pt-BR.po @@ -10,6 +10,7 @@ # m tk, 2019 # Caio … [View More]

1 0

[translation/bridgedb] https://gitweb.torproject.org/translation.git/commit/?h=bridgedb
by translation＠torproject.org 02 Dec '19

02 Dec '19

commit 0fdf43e5817b0d77c9aa2fdac38af267d7a749bf Author: Translation commit bot <translation(a)torproject.org> Date: Mon Dec 2 12:15:16 2019 +0000 https://gitweb.torproject.org/translation.git/commit/?h=bridgedb --- ar/LC_MESSAGES/bridgedb.po | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ar/LC_MESSAGES/bridgedb.po b/ar/LC_MESSAGES/bridgedb.po index 769782953b..d498585202 100644 --- a/ar/LC_MESSAGES/bridgedb.po +++ b/ar/LC_MESSAGES/bridgedb.po @@ -3,… [View More]

1 0

[translation/mat-gui] https://gitweb.torproject.org/translation.git/commit/?h=mat-gui
by translation＠torproject.org 02 Dec '19

02 Dec '19

commit 9bf63fdf91d028789de195f064fb119c4d63cfc7 Author: Translation commit bot <translation(a)torproject.org> Date: Mon Dec 2 11:16:39 2019 +0000 https://gitweb.torproject.org/translation.git/commit/?h=mat-gui --- ia.po | 187 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) diff --git a/ia.po b/ia.po new file mode 100644 index 0000000000..f5ae59d9a4 --- /dev/null +++ b/ia.po @@ -0,0 +1,187 @@ +# SOME DESCRIPTIVE TITLE. +# … [View More]Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# +# Translators: +msgid "" +msgstr "" +"Project-Id-Version: Tor Project\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2016-02-10 23:06+0100\n" +"PO-Revision-Date: 2013-02-04 18:44+0000\n" +"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n" +"Language-Team: Interlingua (http://www.transifex.com/otf/torproject/language/ia/)\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Language: ia\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +#: mat-gui:66 mat-gui:422 mat-gui:445 +msgid "Ready" +msgstr "" + +#: mat-gui:136 +msgid "Choose files" +msgstr "" + +#: mat-gui:144 +msgid "Supported files" +msgstr "" + +#: mat-gui:151 +msgid "All files" +msgstr "" + +#: mat-gui:167 mat-gui:366 mat-gui:417 mat-gui:441 mat-gui:443 +#: data/mat.glade:200 +msgid "Clean" +msgstr "" + +#: mat-gui:168 +msgid "No metadata found" +msgstr "" + +#: mat-gui:170 mat-gui:419 +msgid "Dirty" +msgstr "" + +#: mat-gui:176 +#, python-format +msgid "%s's metadata" +msgstr "" + +#: mat-gui:187 +msgid "Trash your meta, keep your data" +msgstr "" + +#: mat-gui:192 +msgid "Website" +msgstr "" + +#: mat-gui:219 +msgid "Preferences" +msgstr "" + +#: mat-gui:232 +msgid "Reduce PDF quality" +msgstr "" + +#: mat-gui:235 +msgid "Reduce the produced PDF size and quality" +msgstr "" + +#: mat-gui:238 +msgid "Remove unsupported file from archives" +msgstr "" + +#: mat-gui:241 +msgid "Remove non-supported (and so non-anonymised) file from output archive" +msgstr "" + +#: mat-gui:280 +msgid "Unknown" +msgstr "" + +#: mat-gui:325 +msgid "Not-supported" +msgstr "" + +#: mat-gui:339 +msgid "Harmless fileformat" +msgstr "" + +#: mat-gui:341 +msgid "Cant read file" +msgstr "" + +#: mat-gui:343 +msgid "Fileformat not supported" +msgstr "" + +#: mat-gui:346 +msgid "These files can not be processed:" +msgstr "" + +#: mat-gui:351 mat-gui:380 data/mat.glade:239 +msgid "Filename" +msgstr "" + +#: mat-gui:353 +msgid "Reason" +msgstr "" + +#: mat-gui:365 +msgid "Non-supported files in archive" +msgstr "" + +#: mat-gui:379 +msgid "Include" +msgstr "" + +#: mat-gui:397 +#, python-format +msgid "MAT is not able to clean the following files, found in the %s archive" +msgstr "" + +#: mat-gui:413 +#, python-format +msgid "Checking %s" +msgstr "" + +#: mat-gui:428 +#, python-format +msgid "Cleaning %s" +msgstr "" + +#: data/mat.glade:46 +msgid "_File" +msgstr "" + +#: data/mat.glade:95 +msgid "_Edit" +msgstr "" + +#: data/mat.glade:141 +msgid "_Help" +msgstr "" + +#: data/mat.glade:187 +msgid "Add" +msgstr "" + +#: data/mat.glade:256 +msgid "State" +msgstr "" + +#: data/mat.glade:294 data/mat.glade:467 +msgid "Metadata" +msgstr "" + +#: data/mat.glade:354 +msgid "Name" +msgstr "" + +#: data/mat.glade:368 +msgid "Content" +msgstr "" + +#: data/mat.glade:398 +msgid "Supported formats" +msgstr "" + +#: data/mat.glade:456 +msgid "Support" +msgstr "" + +#: data/mat.glade:478 +msgid "Method" +msgstr "" + +#: data/mat.glade:489 +msgid "Remaining" +msgstr "" + +#: data/mat.glade:518 +msgid "Fileformat" +msgstr "" [View Less]

1 0