commit ea1b1b4f6ab11e7ac933b0e00d5f8c040e4cc11e Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Sat Nov 23 17:43:45 2019 +0100 Remove dependency on metrics-lib's log package (2/4). - Remove unused code. --- .../persist/WebServerAccessLogPersistence.java | 5 -- .../collector/webstats/InternalLogDescriptor.java | 35 -------------- .../collector/webstats/LogDescriptorImpl.java | 53 +--------------------- .../metrics/collector/webstats/LogFileMap.java | 26 ----------- .../collector/webstats/SanitizeWeblogs.java | 2 +- .../collector/webstats/WebServerAccessLogImpl.java | 38 +--------------- 6 files changed, 4 insertions(+), 155 deletions(-) diff --git a/src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java index 0f862b4..6afb97e 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java +++ b/src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java @@ -7,9 +7,6 @@ import org.torproject.descriptor.WebServerAccessLog; import org.torproject.metrics.collector.webstats.FileType; import org.torproject.metrics.collector.webstats.InternalWebServerAccessLog; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.nio.file.Paths; import java.nio.file.StandardOpenOption; import java.time.format.DateTimeFormatter; @@ -19,8 +16,6 @@ public class WebServerAccessLogPersistence public static final String SEP = InternalWebServerAccessLog.SEP; public static final FileType COMPRESSION = FileType.XZ; - private static final Logger log - = LoggerFactory.getLogger(WebServerAccessLogPersistence.class); private DateTimeFormatter yearPattern = DateTimeFormatter.ofPattern("yyyy"); private DateTimeFormatter monthPattern = DateTimeFormatter.ofPattern("MM"); diff --git a/src/main/java/org/torproject/metrics/collector/webstats/InternalLogDescriptor.java b/src/main/java/org/torproject/metrics/collector/webstats/InternalLogDescriptor.java index 3a8a1f0..ac7d689 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/InternalLogDescriptor.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/InternalLogDescriptor.java @@ -3,7 +3,6 @@ package org.torproject.metrics.collector.webstats; -import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.LogDescriptor; /** @@ -17,23 +16,6 @@ public interface InternalLogDescriptor extends LogDescriptor { String SEP = "_"; /** - * Validate log lines. - * - * @since 2.2.0 - */ - void validate() throws DescriptorParseException; - - /** - * Set the {@code Validator} that will perform the validation on log - * lines. - * - * <p>Usually set by the implementing class.</p> - * - * @since 2.2.0 - */ - void setValidator(Validator validator); - - /** * Set the descriptor's bytes. * * @since 2.2.0 @@ -42,22 +24,5 @@ public interface InternalLogDescriptor extends LogDescriptor { /** Return the descriptor's preferred compression. */ String getCompressionType(); - - /** - * Provides a single function for validating a single log line. - * - * @since 2.2.0 - */ - interface Validator { - - /** - * Verifies a log line. - * - * @since 2.2.0 - */ - boolean validate(String line); - - } - } diff --git a/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java b/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java index d13c85a..c0b0c69 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java @@ -3,25 +3,17 @@ package org.torproject.metrics.collector.webstats; -import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.LogDescriptor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.File; import java.io.InputStream; -import java.io.InputStreamReader; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.stream.Collectors; /** * Base class for log descriptors. @@ -34,11 +26,6 @@ public abstract class LogDescriptorImpl /** The log's file name should contain this string. */ public static final String MARKER = ".log"; - private static final int unrecognizedLinesLimit = 3; - - private static final Logger log - = LoggerFactory.getLogger(LogDescriptorImpl.class); - private static Pattern filenamePattern = Pattern.compile( "(?:\\S*)" + MARKER + SEP + "(?:[0-9a-zA-Z]*)(?:\\.?)([a-zA-Z2]*)"); @@ -51,8 +38,6 @@ public abstract class LogDescriptorImpl private List<String> unrecognizedLines = new ArrayList<>(); - private Validator validator = (String line) -> true; - /** * This constructor performs basic operations on the given bytes. * @@ -63,8 +48,7 @@ public abstract class LogDescriptorImpl * @since 2.2.0 */ protected LogDescriptorImpl(byte[] logBytes, File descriptorFile, - String logName, FileType defaultCompression) - throws DescriptorParseException { + String logName) throws DescriptorParseException { this.logBytes = logBytes; this.descriptorFile = descriptorFile; try { @@ -75,7 +59,7 @@ public abstract class LogDescriptorImpl } this.fileType = FileType.findType(mat.group(1).toUpperCase()); if (FileType.PLAIN == this.fileType) { - this.fileType = defaultCompression; + this.fileType = FileType.XZ; this.logBytes = this.fileType.compress(this.logBytes); } } catch (Exception ex) { @@ -95,39 +79,6 @@ public abstract class LogDescriptorImpl } @Override - public void validate() throws DescriptorParseException { - try (BufferedReader br = new BufferedReader( - new InputStreamReader(decompressedByteStream()))) { - this.unrecognizedLines.addAll(br.lines().parallel().filter((line) - -> null != line && !line.isEmpty() && !validator.validate(line)) - .limit(unrecognizedLinesLimit).collect(Collectors.toList())); - } catch (Exception ex) { - throw new DescriptorParseException("Cannot validate log lines.", ex); - } - } - - /** - * Assemble a LogDescriptor. - * - * @since 2.2.0 - */ - public static List<Descriptor> parse(byte[] logBytes, - File descriptorFile, String logName) throws DescriptorParseException { - if (logName.contains(InternalWebServerAccessLog.MARKER)) { - return Arrays.asList(new Descriptor[]{ - new WebServerAccessLogImpl(logBytes, descriptorFile, logName)}); - } else { - throw new DescriptorParseException("Cannot parse file " + logName - + " from file " + descriptorFile.getName()); - } - } - - @Override - public void setValidator(Validator validator) { - this.validator = validator; - } - - @Override public String getCompressionType() { return this.fileType.name().toLowerCase(); } diff --git a/src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java b/src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java index 2036b1f..d3e7dcd 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java @@ -15,7 +15,6 @@ import java.nio.file.attribute.BasicFileAttributes; import java.time.LocalDate; import java.util.Optional; import java.util.TreeMap; -import java.util.stream.Stream; /** * Processes the given path and stores metadata for log files. @@ -74,30 +73,5 @@ public class LogFileMap k -> new TreeMap<>()); physicalHosts.put(metadata.date, metadata); } - - /** - * Takes the given metadata and returns the LogMetadata for the entry - * of the next day. - */ - public Optional<LogMetadata> nextDayLogFor(LogMetadata metadata) { - TreeMap<String, TreeMap<LocalDate, LogMetadata>> virtualHosts - = this.get(metadata.virtualHost); - if (null == virtualHosts) { - return Optional.empty(); - } - TreeMap<LocalDate, LogMetadata> physicalHosts - = virtualHosts.get(metadata.physicalHost); - if (null == physicalHosts) { - return Optional.empty(); - } - return Optional.ofNullable(physicalHosts.get(metadata.date.plusDays(1))); - } - - /** Returns a stream of all contained log metadata. */ - public Stream<LogMetadata> metadataStream() { - return this.values().stream() - .flatMap((virtualHosts) -> virtualHosts.values().stream()) - .flatMap((physicalHosts) -> physicalHosts.values().stream()); - } } diff --git a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java index 74bd741..b2b4c49 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java @@ -147,7 +147,7 @@ public class SanitizeWeblogs extends CollecTorMain { WebServerAccessLogPersistence walp = new WebServerAccessLogPersistence( new WebServerAccessLogImpl(toCompressedBytes(retainedLines), - new File(name), name, false)); + new File(name), name)); log.debug("Storing {}.", name); walp.storeOut(this.outputPathName); walp.storeRecent(this.recentPathName); diff --git a/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java b/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java index 2ca608a..548fbf6 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java @@ -6,9 +6,6 @@ package org.torproject.metrics.collector.webstats; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.WebServerAccessLog; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.BufferedReader; import java.io.File; import java.io.InputStreamReader; @@ -32,9 +29,6 @@ import java.util.stream.Stream; public class WebServerAccessLogImpl extends LogDescriptorImpl implements InternalWebServerAccessLog, WebServerAccessLog { - private static final Logger log - = LoggerFactory.getLogger(WebServerAccessLogImpl.class); - /** The log's name should include this string. */ public static final String MARKER = InternalWebServerAccessLog.MARKER; @@ -49,8 +43,6 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl private final LocalDate logDate; - private boolean validate = true; - /** * Creates a WebServerAccessLog from the given bytes and filename. * @@ -68,30 +60,7 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl */ protected WebServerAccessLogImpl(byte[] logBytes, File file, String logName) throws DescriptorParseException { - this(logBytes, file, logName, FileType.XZ); - } - - /** For internal use only. */ - public WebServerAccessLogImpl(byte[] bytes, String filename, - boolean validate) throws DescriptorParseException { - this(bytes, null, filename, FileType.XZ, validate); - } - - /** For internal use only. */ - public WebServerAccessLogImpl(byte[] bytes, File sourceFile, String filename, - boolean validate) throws DescriptorParseException { - this(bytes, sourceFile, filename, FileType.XZ, validate); - } - - private WebServerAccessLogImpl(byte[] logBytes, File file, String logName, - FileType defaultCompression) throws DescriptorParseException { - this(logBytes, file, logName, defaultCompression, true); - } - - private WebServerAccessLogImpl(byte[] logBytes, File file, String logName, - FileType defaultCompression, boolean validate) - throws DescriptorParseException { - super(logBytes, file, logName, defaultCompression); + super(logBytes, file, logName); try { String fn = Paths.get(logName).getFileName().toString(); Matcher mat = filenamePattern.matcher(fn); @@ -108,11 +77,6 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl } String ymd = mat.group(3); this.logDate = LocalDate.parse(ymd, DateTimeFormatter.BASIC_ISO_DATE); - this.setValidator((line) - -> WebServerAccessLogLine.makeLine(line).isValid()); - if (validate) { - this.validate(); - } } catch (DescriptorParseException dpe) { throw dpe; // escalate } catch (Exception pe) {