[tor-commits] [collector/release] Remove dependency on metrics-lib's log package (2/4).
karsten at torproject.org
karsten at torproject.org
Wed Jan 15 22:25:08 UTC 2020
commit ea1b1b4f6ab11e7ac933b0e00d5f8c040e4cc11e
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Sat Nov 23 17:43:45 2019 +0100
Remove dependency on metrics-lib's log package (2/4).
- Remove unused code.
---
.../persist/WebServerAccessLogPersistence.java | 5 --
.../collector/webstats/InternalLogDescriptor.java | 35 --------------
.../collector/webstats/LogDescriptorImpl.java | 53 +---------------------
.../metrics/collector/webstats/LogFileMap.java | 26 -----------
.../collector/webstats/SanitizeWeblogs.java | 2 +-
.../collector/webstats/WebServerAccessLogImpl.java | 38 +---------------
6 files changed, 4 insertions(+), 155 deletions(-)
diff --git a/src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java
index 0f862b4..6afb97e 100644
--- a/src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java
+++ b/src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java
@@ -7,9 +7,6 @@ import org.torproject.descriptor.WebServerAccessLog;
import org.torproject.metrics.collector.webstats.FileType;
import org.torproject.metrics.collector.webstats.InternalWebServerAccessLog;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.time.format.DateTimeFormatter;
@@ -19,8 +16,6 @@ public class WebServerAccessLogPersistence
public static final String SEP = InternalWebServerAccessLog.SEP;
public static final FileType COMPRESSION = FileType.XZ;
- private static final Logger log
- = LoggerFactory.getLogger(WebServerAccessLogPersistence.class);
private DateTimeFormatter yearPattern = DateTimeFormatter.ofPattern("yyyy");
private DateTimeFormatter monthPattern = DateTimeFormatter.ofPattern("MM");
diff --git a/src/main/java/org/torproject/metrics/collector/webstats/InternalLogDescriptor.java b/src/main/java/org/torproject/metrics/collector/webstats/InternalLogDescriptor.java
index 3a8a1f0..ac7d689 100644
--- a/src/main/java/org/torproject/metrics/collector/webstats/InternalLogDescriptor.java
+++ b/src/main/java/org/torproject/metrics/collector/webstats/InternalLogDescriptor.java
@@ -3,7 +3,6 @@
package org.torproject.metrics.collector.webstats;
-import org.torproject.descriptor.DescriptorParseException;
import org.torproject.descriptor.LogDescriptor;
/**
@@ -17,23 +16,6 @@ public interface InternalLogDescriptor extends LogDescriptor {
String SEP = "_";
/**
- * Validate log lines.
- *
- * @since 2.2.0
- */
- void validate() throws DescriptorParseException;
-
- /**
- * Set the {@code Validator} that will perform the validation on log
- * lines.
- *
- * <p>Usually set by the implementing class.</p>
- *
- * @since 2.2.0
- */
- void setValidator(Validator validator);
-
- /**
* Set the descriptor's bytes.
*
* @since 2.2.0
@@ -42,22 +24,5 @@ public interface InternalLogDescriptor extends LogDescriptor {
/** Return the descriptor's preferred compression. */
String getCompressionType();
-
- /**
- * Provides a single function for validating a single log line.
- *
- * @since 2.2.0
- */
- interface Validator {
-
- /**
- * Verifies a log line.
- *
- * @since 2.2.0
- */
- boolean validate(String line);
-
- }
-
}
diff --git a/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java b/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java
index d13c85a..c0b0c69 100644
--- a/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java
+++ b/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java
@@ -3,25 +3,17 @@
package org.torproject.metrics.collector.webstats;
-import org.torproject.descriptor.Descriptor;
import org.torproject.descriptor.DescriptorParseException;
import org.torproject.descriptor.LogDescriptor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.InputStream;
-import java.io.InputStreamReader;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import java.util.stream.Collectors;
/**
* Base class for log descriptors.
@@ -34,11 +26,6 @@ public abstract class LogDescriptorImpl
/** The log's file name should contain this string. */
public static final String MARKER = ".log";
- private static final int unrecognizedLinesLimit = 3;
-
- private static final Logger log
- = LoggerFactory.getLogger(LogDescriptorImpl.class);
-
private static Pattern filenamePattern = Pattern.compile(
"(?:\\S*)" + MARKER + SEP + "(?:[0-9a-zA-Z]*)(?:\\.?)([a-zA-Z2]*)");
@@ -51,8 +38,6 @@ public abstract class LogDescriptorImpl
private List<String> unrecognizedLines = new ArrayList<>();
- private Validator validator = (String line) -> true;
-
/**
* This constructor performs basic operations on the given bytes.
*
@@ -63,8 +48,7 @@ public abstract class LogDescriptorImpl
* @since 2.2.0
*/
protected LogDescriptorImpl(byte[] logBytes, File descriptorFile,
- String logName, FileType defaultCompression)
- throws DescriptorParseException {
+ String logName) throws DescriptorParseException {
this.logBytes = logBytes;
this.descriptorFile = descriptorFile;
try {
@@ -75,7 +59,7 @@ public abstract class LogDescriptorImpl
}
this.fileType = FileType.findType(mat.group(1).toUpperCase());
if (FileType.PLAIN == this.fileType) {
- this.fileType = defaultCompression;
+ this.fileType = FileType.XZ;
this.logBytes = this.fileType.compress(this.logBytes);
}
} catch (Exception ex) {
@@ -95,39 +79,6 @@ public abstract class LogDescriptorImpl
}
@Override
- public void validate() throws DescriptorParseException {
- try (BufferedReader br = new BufferedReader(
- new InputStreamReader(decompressedByteStream()))) {
- this.unrecognizedLines.addAll(br.lines().parallel().filter((line)
- -> null != line && !line.isEmpty() && !validator.validate(line))
- .limit(unrecognizedLinesLimit).collect(Collectors.toList()));
- } catch (Exception ex) {
- throw new DescriptorParseException("Cannot validate log lines.", ex);
- }
- }
-
- /**
- * Assemble a LogDescriptor.
- *
- * @since 2.2.0
- */
- public static List<Descriptor> parse(byte[] logBytes,
- File descriptorFile, String logName) throws DescriptorParseException {
- if (logName.contains(InternalWebServerAccessLog.MARKER)) {
- return Arrays.asList(new Descriptor[]{
- new WebServerAccessLogImpl(logBytes, descriptorFile, logName)});
- } else {
- throw new DescriptorParseException("Cannot parse file " + logName
- + " from file " + descriptorFile.getName());
- }
- }
-
- @Override
- public void setValidator(Validator validator) {
- this.validator = validator;
- }
-
- @Override
public String getCompressionType() {
return this.fileType.name().toLowerCase();
}
diff --git a/src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java b/src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java
index 2036b1f..d3e7dcd 100644
--- a/src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java
+++ b/src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java
@@ -15,7 +15,6 @@ import java.nio.file.attribute.BasicFileAttributes;
import java.time.LocalDate;
import java.util.Optional;
import java.util.TreeMap;
-import java.util.stream.Stream;
/**
* Processes the given path and stores metadata for log files.
@@ -74,30 +73,5 @@ public class LogFileMap
k -> new TreeMap<>());
physicalHosts.put(metadata.date, metadata);
}
-
- /**
- * Takes the given metadata and returns the LogMetadata for the entry
- * of the next day.
- */
- public Optional<LogMetadata> nextDayLogFor(LogMetadata metadata) {
- TreeMap<String, TreeMap<LocalDate, LogMetadata>> virtualHosts
- = this.get(metadata.virtualHost);
- if (null == virtualHosts) {
- return Optional.empty();
- }
- TreeMap<LocalDate, LogMetadata> physicalHosts
- = virtualHosts.get(metadata.physicalHost);
- if (null == physicalHosts) {
- return Optional.empty();
- }
- return Optional.ofNullable(physicalHosts.get(metadata.date.plusDays(1)));
- }
-
- /** Returns a stream of all contained log metadata. */
- public Stream<LogMetadata> metadataStream() {
- return this.values().stream()
- .flatMap((virtualHosts) -> virtualHosts.values().stream())
- .flatMap((physicalHosts) -> physicalHosts.values().stream());
- }
}
diff --git a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java
index 74bd741..b2b4c49 100644
--- a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java
+++ b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java
@@ -147,7 +147,7 @@ public class SanitizeWeblogs extends CollecTorMain {
WebServerAccessLogPersistence walp
= new WebServerAccessLogPersistence(
new WebServerAccessLogImpl(toCompressedBytes(retainedLines),
- new File(name), name, false));
+ new File(name), name));
log.debug("Storing {}.", name);
walp.storeOut(this.outputPathName);
walp.storeRecent(this.recentPathName);
diff --git a/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java b/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java
index 2ca608a..548fbf6 100644
--- a/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java
+++ b/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java
@@ -6,9 +6,6 @@ package org.torproject.metrics.collector.webstats;
import org.torproject.descriptor.DescriptorParseException;
import org.torproject.descriptor.WebServerAccessLog;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.BufferedReader;
import java.io.File;
import java.io.InputStreamReader;
@@ -32,9 +29,6 @@ import java.util.stream.Stream;
public class WebServerAccessLogImpl extends LogDescriptorImpl
implements InternalWebServerAccessLog, WebServerAccessLog {
- private static final Logger log
- = LoggerFactory.getLogger(WebServerAccessLogImpl.class);
-
/** The log's name should include this string. */
public static final String MARKER = InternalWebServerAccessLog.MARKER;
@@ -49,8 +43,6 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl
private final LocalDate logDate;
- private boolean validate = true;
-
/**
* Creates a WebServerAccessLog from the given bytes and filename.
*
@@ -68,30 +60,7 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl
*/
protected WebServerAccessLogImpl(byte[] logBytes, File file, String logName)
throws DescriptorParseException {
- this(logBytes, file, logName, FileType.XZ);
- }
-
- /** For internal use only. */
- public WebServerAccessLogImpl(byte[] bytes, String filename,
- boolean validate) throws DescriptorParseException {
- this(bytes, null, filename, FileType.XZ, validate);
- }
-
- /** For internal use only. */
- public WebServerAccessLogImpl(byte[] bytes, File sourceFile, String filename,
- boolean validate) throws DescriptorParseException {
- this(bytes, sourceFile, filename, FileType.XZ, validate);
- }
-
- private WebServerAccessLogImpl(byte[] logBytes, File file, String logName,
- FileType defaultCompression) throws DescriptorParseException {
- this(logBytes, file, logName, defaultCompression, true);
- }
-
- private WebServerAccessLogImpl(byte[] logBytes, File file, String logName,
- FileType defaultCompression, boolean validate)
- throws DescriptorParseException {
- super(logBytes, file, logName, defaultCompression);
+ super(logBytes, file, logName);
try {
String fn = Paths.get(logName).getFileName().toString();
Matcher mat = filenamePattern.matcher(fn);
@@ -108,11 +77,6 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl
}
String ymd = mat.group(3);
this.logDate = LocalDate.parse(ymd, DateTimeFormatter.BASIC_ISO_DATE);
- this.setValidator((line)
- -> WebServerAccessLogLine.makeLine(line).isValid());
- if (validate) {
- this.validate();
- }
} catch (DescriptorParseException dpe) {
throw dpe; // escalate
} catch (Exception pe) {
More information about the tor-commits
mailing list