- tor-commits - lists.torproject.org

[collector/release] Archive OnionPerf analysis .json files.
by karsten＠torproject.org 30 Apr '20

30 Apr '20

commit 0f5536ed68c79be50a9b1e326356008f7ffaefff Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Apr 30 17:29:35 2020 +0200 Archive OnionPerf analysis .json files. Implements #34072. --- CHANGELOG.md | 5 + build.xml | 2 +- .../collector/onionperf/OnionPerfDownloader.java | 218 +++++++++++++++++---- src/main/resources/collector.properties | 2 +- src/main/resources/create-tarballs.sh | 7 + 5 files changed, 195 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bbedc4..a55a0fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changes in version 1.1?.? - 2020-0?-?? + * Medium changes + - Update to metrics-lib 2.12.1. + - Download OnionPerf analysis .json files in addition to .tpf + files. + * Minor changes - Simplify logging configuration. - Set default locale `US` and default time zone `UTC` at the diff --git a/build.xml b/build.xml index 748351e..a9988f5 100644 --- a/build.xml +++ b/build.xml @@ -12,7 +12,7 @@ <property name="release.version" value="1.14.1-dev" /> <property name="project-main-class" value="org.torproject.metrics.collector.Main" /> <property name="name" value="collector"/> - <property name="metricslibversion" value="2.10.0" /> + <property name="metricslibversion" value="2.12.1" /> <property name="jarincludes" value="collector.properties logback.xml" /> <patternset id="runtime" > diff --git a/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java b/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java index b651620..d22ac0b 100644 --- a/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java +++ b/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java @@ -13,6 +13,7 @@ import org.torproject.metrics.collector.conf.Key; import org.torproject.metrics.collector.cron.CollecTorMain; import org.torproject.metrics.collector.downloader.Downloader; +import org.apache.commons.compress.utils.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,14 +33,16 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.SortedSet; import java.util.Stack; import java.util.TreeSet; import java.util.regex.Matcher; import java.util.regex.Pattern; -/** Download download .tpf files from OnionPerf hosts. */ +/** Download OnionPerf files from OnionPerf hosts. */ public class OnionPerfDownloader extends CollecTorMain { private static final Logger logger = LoggerFactory.getLogger( @@ -47,6 +50,8 @@ public class OnionPerfDownloader extends CollecTorMain { private static final String TORPERF = "torperf"; + private static final String ONIONPERF = "onionperf"; + /** Instantiate the OnionPerf module using the given configuration. */ public OnionPerfDownloader(Configuration config) { super(config); @@ -54,21 +59,25 @@ public class OnionPerfDownloader extends CollecTorMain { } /** File containing the download history, which is necessary, because - * OnionPerf does not delete older .tpf files, but which enables us to do - * so. */ + * OnionPerf does not delete older files, but which enables us to do so. */ private File onionPerfDownloadedFile; - /** Full URLs of .tpf files downloaded in the current or in past - * executions. */ - private SortedSet<String> downloadedTpfFiles = new TreeSet<>(); + /** Full URLs of files downloaded in the current or in past executions. */ + private SortedSet<String> downloadedFiles = new TreeSet<>(); /** Base URLs of configured OnionPerf hosts. */ private URL[] onionPerfHosts = null; - /** Directory for storing archived .tpf files. */ + /** Relative URLs of available .tpf files by base URL. */ + private Map<URL, List<String>> tpfFileUrls = new HashMap<>(); + + /** Relative URLs of available OnionPerf analysis files by base URL. */ + private Map<URL, List<String>> onionPerfAnalysisFileUrls = new HashMap<>(); + + /** Directory for storing archived files. */ private File archiveDirectory = null; - /** Directory for storing recent .tpf files. */ + /** Directory for storing recent files. */ private File recentDirectory = null; @Override @@ -87,19 +96,17 @@ public class OnionPerfDownloader extends CollecTorMain { new File(config.getPath(Key.StatsPath).toFile(), "onionperf-downloaded"); this.onionPerfHosts = config.getUrlArray(Key.OnionPerfHosts); - this.readDownloadedOnionPerfTpfFiles(); - this.archiveDirectory = new File(config.getPath(Key.OutputPath).toFile(), - TORPERF); - this.recentDirectory = new File(config.getPath(Key.RecentPath).toFile(), - TORPERF); + this.readDownloadedOnionPerfFiles(); + this.archiveDirectory = config.getPath(Key.OutputPath).toFile(); + this.recentDirectory = config.getPath(Key.RecentPath).toFile(); for (URL baseUrl : this.onionPerfHosts) { this.downloadFromOnionPerfHost(baseUrl); } - this.writeDownloadedOnionPerfTpfFiles(); + this.writeDownloadedOnionPerfFiles(); this.cleanUpRsyncDirectory(); } - private void readDownloadedOnionPerfTpfFiles() { + private void readDownloadedOnionPerfFiles() { if (!this.onionPerfDownloadedFile.exists()) { return; } @@ -107,47 +114,69 @@ public class OnionPerfDownloader extends CollecTorMain { this.onionPerfDownloadedFile))) { String line; while ((line = br.readLine()) != null) { - this.downloadedTpfFiles.add(line); + this.downloadedFiles.add(line); } } catch (IOException e) { logger.info("Unable to read download history file '{}'. Ignoring " - + "download history and downloading all available .tpf files.", + + "download history and downloading all available files.", this.onionPerfDownloadedFile.getAbsolutePath()); - this.downloadedTpfFiles.clear(); + this.downloadedFiles.clear(); } } private void downloadFromOnionPerfHost(URL baseUrl) { logger.info("Downloading from OnionPerf host {}", baseUrl); - List<String> tpfFileNames = - this.downloadOnionPerfDirectoryListing(baseUrl); + this.downloadOnionPerfDirectoryListing(baseUrl); String source = baseUrl.getHost().split("\\.")[0]; - for (String tpfFileName : tpfFileNames) { - this.downloadAndParseOnionPerfTpfFile(baseUrl, source, tpfFileName); + if (this.tpfFileUrls.containsKey(baseUrl)) { + for (String tpfFileName : this.tpfFileUrls.get(baseUrl)) { + this.downloadAndParseOnionPerfTpfFile(baseUrl, source, tpfFileName); + } + } + if (this.onionPerfAnalysisFileUrls.containsKey(baseUrl)) { + for (String onionPerfAnalysisFileName + : this.onionPerfAnalysisFileUrls.get(baseUrl)) { + this.downloadAndParseOnionPerfAnalysisFile(baseUrl, source, + onionPerfAnalysisFileName); + } } } - /** Pattern for links contained in directory listings. */ + /** Patterns for links contained in directory listings. */ private static final Pattern TPF_FILE_URL_PATTERN = Pattern.compile(".*<a href=\"([^\"]+\\.tpf)\">.*"); - private List<String> downloadOnionPerfDirectoryListing(URL baseUrl) { - List<String> tpfFileUrls = new ArrayList<>(); + private static final Pattern ONIONPERF_ANALYSIS_FILE_URL_PATTERN = + Pattern.compile( + ".*<a href=\"([0-9-]{10}\\.onionperf\\.analysis\\.json\\.xz)\">.*"); + + private void downloadOnionPerfDirectoryListing(URL baseUrl) { try (BufferedReader br = new BufferedReader(new InputStreamReader( baseUrl.openStream()))) { String line; while ((line = br.readLine()) != null) { - Matcher matcher = TPF_FILE_URL_PATTERN.matcher(line); - if (matcher.matches() && !matcher.group(1).startsWith("/")) { - tpfFileUrls.add(matcher.group(1)); + Matcher tpfFileMatcher = TPF_FILE_URL_PATTERN.matcher(line); + if (tpfFileMatcher.matches() + && !tpfFileMatcher.group(1).startsWith("/")) { + this.tpfFileUrls.putIfAbsent(baseUrl, new ArrayList<>()); + this.tpfFileUrls.get(baseUrl).add(tpfFileMatcher.group(1)); + } + Matcher onionPerfAnalysisFileMatcher + = ONIONPERF_ANALYSIS_FILE_URL_PATTERN.matcher(line); + if (onionPerfAnalysisFileMatcher.matches() + && !onionPerfAnalysisFileMatcher.group(1).startsWith("/")) { + this.onionPerfAnalysisFileUrls.putIfAbsent(baseUrl, + new ArrayList<>()); + this.onionPerfAnalysisFileUrls.get(baseUrl) + .add(onionPerfAnalysisFileMatcher.group(1)); } } } catch (IOException e) { logger.warn("Unable to download directory listing from '{}'. Skipping " + "this OnionPerf host.", baseUrl); - tpfFileUrls.clear(); + this.tpfFileUrls.remove(baseUrl); + this.onionPerfAnalysisFileUrls.remove(baseUrl); } - return tpfFileUrls; } private static final DateFormat DATE_FORMAT; @@ -169,7 +198,7 @@ public class OnionPerfDownloader extends CollecTorMain { } /* Skip if we successfully downloaded this file before. */ - if (this.downloadedTpfFiles.contains(tpfFileUrl.toString())) { + if (this.downloadedFiles.contains(tpfFileUrl.toString())) { return; } @@ -197,7 +226,8 @@ public class OnionPerfDownloader extends CollecTorMain { } /* Download file contents to temporary file. */ - File tempFile = new File(this.recentDirectory, "." + tpfFileName); + File tempFile = new File(this.recentDirectory, + TORPERF + "/." + tpfFileName); byte[] downloadedBytes; try { downloadedBytes = Downloader.downloadFromHttpServer( @@ -263,7 +293,7 @@ public class OnionPerfDownloader extends CollecTorMain { /* Copy/move files in place. */ File archiveFile = new File(this.archiveDirectory, - date.replaceAll("-", "/") + "/" + tpfFileName); + TORPERF + "/" + date.replaceAll("-", "/") + "/" + tpfFileName); archiveFile.getParentFile().mkdirs(); try { Files.copy(tempFile.toPath(), archiveFile.toPath(), @@ -274,18 +304,132 @@ public class OnionPerfDownloader extends CollecTorMain { tempFile.delete(); return; } - File recentFile = new File(this.recentDirectory, tpfFileName); + File recentFile = new File(this.recentDirectory, + TORPERF + "/" + tpfFileName); + tempFile.renameTo(recentFile); + + /* Add to download history to avoid downloading it again. */ + this.downloadedFiles.add(baseUrl + tpfFileName); + } + + + private void downloadAndParseOnionPerfAnalysisFile(URL baseUrl, String source, + String onionPerfAnalysisFileName) { + URL onionPerfAnalysisFileUrl; + try { + onionPerfAnalysisFileUrl = new URL(baseUrl, onionPerfAnalysisFileName); + } catch (MalformedURLException e1) { + logger.warn("Unable to put together base URL '{}' and file path '{}' to " + + "a URL. Skipping.", baseUrl, onionPerfAnalysisFileName); + return; + } + + /* Skip if we successfully downloaded this file before. */ + if (this.downloadedFiles.contains(onionPerfAnalysisFileUrl.toString())) { + return; + } + + /* Parse date from file name: yyyy-MM-dd.onionperf.analysis.json.xz */ + String date; + try { + date = onionPerfAnalysisFileName.substring(0, 10); + DATE_FORMAT.parse(date); + } catch (NumberFormatException | ParseException e) { + logger.warn("Invalid file name '{}{}'. Skipping.", baseUrl, + onionPerfAnalysisFileName, e); + return; + } + + /* Download file contents to temporary file. */ + File tempFile = new File(this.recentDirectory, + ONIONPERF + "/." + onionPerfAnalysisFileName); + byte[] downloadedBytes; + try { + downloadedBytes = Downloader.downloadFromHttpServer( + new URL(baseUrl + onionPerfAnalysisFileName)); + } catch (IOException e) { + logger.warn("Unable to download '{}{}'. Skipping.", baseUrl, + onionPerfAnalysisFileName, e); + return; + } + if (null == downloadedBytes) { + logger.warn("Unable to download '{}{}'. Skipping.", baseUrl, + onionPerfAnalysisFileName); + return; + } + tempFile.getParentFile().mkdirs(); + try { + Files.write(tempFile.toPath(), downloadedBytes); + } catch (IOException e) { + logger.warn("Unable to write previously downloaded '{}{}' to temporary " + + "file '{}'. Skipping.", baseUrl, onionPerfAnalysisFileName, + tempFile, e); + return; + } + + /* Validate contained descriptors. */ + DescriptorParser descriptorParser = + DescriptorSourceFactory.createDescriptorParser(); + byte[] rawDescriptorBytes; + try { + rawDescriptorBytes = IOUtils.toByteArray( + Files.newInputStream(tempFile.toPath())); + } catch (IOException e) { + logger.warn("OnionPerf file '{}{}' could not be read. Skipping.", baseUrl, + onionPerfAnalysisFileName, e); + tempFile.delete(); + return; + } + Iterable<Descriptor> descriptors = descriptorParser.parseDescriptors( + rawDescriptorBytes, null, onionPerfAnalysisFileName); + String message = null; + for (Descriptor descriptor : descriptors) { + if (!(descriptor instanceof TorperfResult)) { + message = "File contains descriptors other than an OnionPerf analysis " + + "document: " + descriptor.getClass(); + break; + } + TorperfResult torperf = (TorperfResult) descriptor; + if (!source.equals(torperf.getSource())) { + message = "File contains transfer from another source: " + + torperf.getSource(); + break; + } + } + if (null != message) { + logger.warn("OnionPerf file '{}{}' was found to be invalid: {}. " + + "Skipping.", baseUrl, onionPerfAnalysisFileName, message); + tempFile.delete(); + return; + } + + /* Copy/move files in place. */ + File archiveFile = new File(this.archiveDirectory, + ONIONPERF + "/" + date.replaceAll("-", "/") + "/" + date + "." + source + + ".onionperf.analysis.json.xz"); + archiveFile.getParentFile().mkdirs(); + try { + Files.copy(tempFile.toPath(), archiveFile.toPath(), + StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + logger.warn("Unable to copy OnionPerf file {} to {}. Skipping.", + tempFile, archiveFile, e); + tempFile.delete(); + return; + } + File recentFile = new File(this.recentDirectory, + ONIONPERF + "/" + date + "." + source + ".onionperf.analysis.json.xz"); tempFile.renameTo(recentFile); /* Add to download history to avoid downloading it again. */ - this.downloadedTpfFiles.add(baseUrl + tpfFileName); + this.downloadedFiles.add(baseUrl + onionPerfAnalysisFileName); } - private void writeDownloadedOnionPerfTpfFiles() { + private void writeDownloadedOnionPerfFiles() { this.onionPerfDownloadedFile.getParentFile().mkdirs(); try (BufferedWriter bw = new BufferedWriter(new FileWriter( this.onionPerfDownloadedFile))) { - for (String line : this.downloadedTpfFiles) { + for (String line : this.downloadedFiles) { bw.write(line); bw.newLine(); } diff --git a/src/main/resources/collector.properties b/src/main/resources/collector.properties index 61baed5..2347021 100644 --- a/src/main/resources/collector.properties +++ b/src/main/resources/collector.properties @@ -175,7 +175,7 @@ ExitlistUrl = https://check.torproject.org/exit-addresses ######## OnionPerf downloader ######## # ## Define descriptor sources -# possible values: Remote,Sync +# possible values: Remote,Sync (.tpf files only!) OnionPerfSources = Remote # Retrieve files from the following CollecTor instances. # List of URLs separated by comma. diff --git a/src/main/resources/create-tarballs.sh b/src/main/resources/create-tarballs.sh index 07952c7..fcac2f3 100755 --- a/src/main/resources/create-tarballs.sh +++ b/src/main/resources/create-tarballs.sh @@ -40,6 +40,8 @@ TARBALLS=( exit-list-$YEARTWO-$MONTHTWO torperf-$YEARONE-$MONTHONE torperf-$YEARTWO-$MONTHTWO + onionperf-$YEARONE-$MONTHONE + onionperf-$YEARTWO-$MONTHTWO certs microdescs-$YEARONE-$MONTHONE microdescs-$YEARTWO-$MONTHTWO @@ -73,6 +75,8 @@ DIRECTORIES=( $OUTDIR/exit-lists/$YEARTWO/$MONTHTWO/ $OUTDIR/torperf/$YEARONE/$MONTHONE/ $OUTDIR/torperf/$YEARTWO/$MONTHTWO/ + $OUTDIR/onionperf/$YEARONE/$MONTHONE/ + $OUTDIR/onionperf/$YEARTWO/$MONTHTWO/ $OUTDIR/relay-descriptors/certs/ $OUTDIR/relay-descriptors/microdesc/$YEARONE/$MONTHONE $OUTDIR/relay-descriptors/microdesc/$YEARTWO/$MONTHTWO @@ -178,6 +182,9 @@ ln -f -s -t $ARCHIVEDIR/relay-descriptors/bandwidths/ $TARBALLTARGETDIR/bandwidt mkdir -p $ARCHIVEDIR/torperf/ ln -f -s -t $ARCHIVEDIR/torperf/ $TARBALLTARGETDIR/torperf-20??-??.tar.xz +mkdir -p $ARCHIVEDIR/onionperf/ +ln -f -s -t $ARCHIVEDIR/onionperf/ $TARBALLTARGETDIR/onionperf-20??-??.tar.xz + mkdir -p $ARCHIVEDIR/webstats/ ln -f -s -t $ARCHIVEDIR/webstats/ $TARBALLTARGETDIR/webstats-20??-??.tar

1 0

[collector/release] Prepare for 1.15.0 release.
by karsten＠torproject.org 30 Apr '20

30 Apr '20

commit 2a0c40f54b51bd734f8db1be1c1026bac9b661cd Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Apr 30 17:42:00 2020 +0200 Prepare for 1.15.0 release. --- CHANGELOG.md | 2 +- build.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a55a0fc..8e58e7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 1.1?.? - 2020-0?-?? +# Changes in version 1.15.0 - 2020-04-30 * Medium changes - Update to metrics-lib 2.12.1. diff --git a/build.xml b/build.xml index a9988f5..1e22fef 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ <property name="javadoc-title" value="CollecTor API Documentation"/> <property name="implementation-title" value="CollecTor" /> - <property name="release.version" value="1.14.1-dev" /> + <property name="release.version" value="1.15.0" /> <property name="project-main-class" value="org.torproject.metrics.collector.Main" /> <property name="name" value="collector"/> <property name="metricslibversion" value="2.12.1" />

1 0

[collector/master] Prepare for 1.15.0 release.
by karsten＠torproject.org 30 Apr '20

30 Apr '20

commit 2a0c40f54b51bd734f8db1be1c1026bac9b661cd Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Apr 30 17:42:00 2020 +0200 Prepare for 1.15.0 release. --- CHANGELOG.md | 2 +- build.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a55a0fc..8e58e7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 1.1?.? - 2020-0?-?? +# Changes in version 1.15.0 - 2020-04-30 * Medium changes - Update to metrics-lib 2.12.1. diff --git a/build.xml b/build.xml index a9988f5..1e22fef 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ <property name="javadoc-title" value="CollecTor API Documentation"/> <property name="implementation-title" value="CollecTor" /> - <property name="release.version" value="1.14.1-dev" /> + <property name="release.version" value="1.15.0" /> <property name="project-main-class" value="org.torproject.metrics.collector.Main" /> <property name="name" value="collector"/> <property name="metricslibversion" value="2.12.1" />

1 0

[collector/master] Archive OnionPerf analysis .json files.
by karsten＠torproject.org 30 Apr '20

30 Apr '20

commit 0f5536ed68c79be50a9b1e326356008f7ffaefff Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Apr 30 17:29:35 2020 +0200 Archive OnionPerf analysis .json files. Implements #34072. --- CHANGELOG.md | 5 + build.xml | 2 +- .../collector/onionperf/OnionPerfDownloader.java | 218 +++++++++++++++++---- src/main/resources/collector.properties | 2 +- src/main/resources/create-tarballs.sh | 7 + 5 files changed, 195 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bbedc4..a55a0fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changes in version 1.1?.? - 2020-0?-?? + * Medium changes + - Update to metrics-lib 2.12.1. + - Download OnionPerf analysis .json files in addition to .tpf + files. + * Minor changes - Simplify logging configuration. - Set default locale `US` and default time zone `UTC` at the diff --git a/build.xml b/build.xml index 748351e..a9988f5 100644 --- a/build.xml +++ b/build.xml @@ -12,7 +12,7 @@ <property name="release.version" value="1.14.1-dev" /> <property name="project-main-class" value="org.torproject.metrics.collector.Main" /> <property name="name" value="collector"/> - <property name="metricslibversion" value="2.10.0" /> + <property name="metricslibversion" value="2.12.1" /> <property name="jarincludes" value="collector.properties logback.xml" /> <patternset id="runtime" > diff --git a/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java b/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java index b651620..d22ac0b 100644 --- a/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java +++ b/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java @@ -13,6 +13,7 @@ import org.torproject.metrics.collector.conf.Key; import org.torproject.metrics.collector.cron.CollecTorMain; import org.torproject.metrics.collector.downloader.Downloader; +import org.apache.commons.compress.utils.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,14 +33,16 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.SortedSet; import java.util.Stack; import java.util.TreeSet; import java.util.regex.Matcher; import java.util.regex.Pattern; -/** Download download .tpf files from OnionPerf hosts. */ +/** Download OnionPerf files from OnionPerf hosts. */ public class OnionPerfDownloader extends CollecTorMain { private static final Logger logger = LoggerFactory.getLogger( @@ -47,6 +50,8 @@ public class OnionPerfDownloader extends CollecTorMain { private static final String TORPERF = "torperf"; + private static final String ONIONPERF = "onionperf"; + /** Instantiate the OnionPerf module using the given configuration. */ public OnionPerfDownloader(Configuration config) { super(config); @@ -54,21 +59,25 @@ public class OnionPerfDownloader extends CollecTorMain { } /** File containing the download history, which is necessary, because - * OnionPerf does not delete older .tpf files, but which enables us to do - * so. */ + * OnionPerf does not delete older files, but which enables us to do so. */ private File onionPerfDownloadedFile; - /** Full URLs of .tpf files downloaded in the current or in past - * executions. */ - private SortedSet<String> downloadedTpfFiles = new TreeSet<>(); + /** Full URLs of files downloaded in the current or in past executions. */ + private SortedSet<String> downloadedFiles = new TreeSet<>(); /** Base URLs of configured OnionPerf hosts. */ private URL[] onionPerfHosts = null; - /** Directory for storing archived .tpf files. */ + /** Relative URLs of available .tpf files by base URL. */ + private Map<URL, List<String>> tpfFileUrls = new HashMap<>(); + + /** Relative URLs of available OnionPerf analysis files by base URL. */ + private Map<URL, List<String>> onionPerfAnalysisFileUrls = new HashMap<>(); + + /** Directory for storing archived files. */ private File archiveDirectory = null; - /** Directory for storing recent .tpf files. */ + /** Directory for storing recent files. */ private File recentDirectory = null; @Override @@ -87,19 +96,17 @@ public class OnionPerfDownloader extends CollecTorMain { new File(config.getPath(Key.StatsPath).toFile(), "onionperf-downloaded"); this.onionPerfHosts = config.getUrlArray(Key.OnionPerfHosts); - this.readDownloadedOnionPerfTpfFiles(); - this.archiveDirectory = new File(config.getPath(Key.OutputPath).toFile(), - TORPERF); - this.recentDirectory = new File(config.getPath(Key.RecentPath).toFile(), - TORPERF); + this.readDownloadedOnionPerfFiles(); + this.archiveDirectory = config.getPath(Key.OutputPath).toFile(); + this.recentDirectory = config.getPath(Key.RecentPath).toFile(); for (URL baseUrl : this.onionPerfHosts) { this.downloadFromOnionPerfHost(baseUrl); } - this.writeDownloadedOnionPerfTpfFiles(); + this.writeDownloadedOnionPerfFiles(); this.cleanUpRsyncDirectory(); } - private void readDownloadedOnionPerfTpfFiles() { + private void readDownloadedOnionPerfFiles() { if (!this.onionPerfDownloadedFile.exists()) { return; } @@ -107,47 +114,69 @@ public class OnionPerfDownloader extends CollecTorMain { this.onionPerfDownloadedFile))) { String line; while ((line = br.readLine()) != null) { - this.downloadedTpfFiles.add(line); + this.downloadedFiles.add(line); } } catch (IOException e) { logger.info("Unable to read download history file '{}'. Ignoring " - + "download history and downloading all available .tpf files.", + + "download history and downloading all available files.", this.onionPerfDownloadedFile.getAbsolutePath()); - this.downloadedTpfFiles.clear(); + this.downloadedFiles.clear(); } } private void downloadFromOnionPerfHost(URL baseUrl) { logger.info("Downloading from OnionPerf host {}", baseUrl); - List<String> tpfFileNames = - this.downloadOnionPerfDirectoryListing(baseUrl); + this.downloadOnionPerfDirectoryListing(baseUrl); String source = baseUrl.getHost().split("\\.")[0]; - for (String tpfFileName : tpfFileNames) { - this.downloadAndParseOnionPerfTpfFile(baseUrl, source, tpfFileName); + if (this.tpfFileUrls.containsKey(baseUrl)) { + for (String tpfFileName : this.tpfFileUrls.get(baseUrl)) { + this.downloadAndParseOnionPerfTpfFile(baseUrl, source, tpfFileName); + } + } + if (this.onionPerfAnalysisFileUrls.containsKey(baseUrl)) { + for (String onionPerfAnalysisFileName + : this.onionPerfAnalysisFileUrls.get(baseUrl)) { + this.downloadAndParseOnionPerfAnalysisFile(baseUrl, source, + onionPerfAnalysisFileName); + } } } - /** Pattern for links contained in directory listings. */ + /** Patterns for links contained in directory listings. */ private static final Pattern TPF_FILE_URL_PATTERN = Pattern.compile(".*<a href=\"([^\"]+\\.tpf)\">.*"); - private List<String> downloadOnionPerfDirectoryListing(URL baseUrl) { - List<String> tpfFileUrls = new ArrayList<>(); + private static final Pattern ONIONPERF_ANALYSIS_FILE_URL_PATTERN = + Pattern.compile( + ".*<a href=\"([0-9-]{10}\\.onionperf\\.analysis\\.json\\.xz)\">.*"); + + private void downloadOnionPerfDirectoryListing(URL baseUrl) { try (BufferedReader br = new BufferedReader(new InputStreamReader( baseUrl.openStream()))) { String line; while ((line = br.readLine()) != null) { - Matcher matcher = TPF_FILE_URL_PATTERN.matcher(line); - if (matcher.matches() && !matcher.group(1).startsWith("/")) { - tpfFileUrls.add(matcher.group(1)); + Matcher tpfFileMatcher = TPF_FILE_URL_PATTERN.matcher(line); + if (tpfFileMatcher.matches() + && !tpfFileMatcher.group(1).startsWith("/")) { + this.tpfFileUrls.putIfAbsent(baseUrl, new ArrayList<>()); + this.tpfFileUrls.get(baseUrl).add(tpfFileMatcher.group(1)); + } + Matcher onionPerfAnalysisFileMatcher + = ONIONPERF_ANALYSIS_FILE_URL_PATTERN.matcher(line); + if (onionPerfAnalysisFileMatcher.matches() + && !onionPerfAnalysisFileMatcher.group(1).startsWith("/")) { + this.onionPerfAnalysisFileUrls.putIfAbsent(baseUrl, + new ArrayList<>()); + this.onionPerfAnalysisFileUrls.get(baseUrl) + .add(onionPerfAnalysisFileMatcher.group(1)); } } } catch (IOException e) { logger.warn("Unable to download directory listing from '{}'. Skipping " + "this OnionPerf host.", baseUrl); - tpfFileUrls.clear(); + this.tpfFileUrls.remove(baseUrl); + this.onionPerfAnalysisFileUrls.remove(baseUrl); } - return tpfFileUrls; } private static final DateFormat DATE_FORMAT; @@ -169,7 +198,7 @@ public class OnionPerfDownloader extends CollecTorMain { } /* Skip if we successfully downloaded this file before. */ - if (this.downloadedTpfFiles.contains(tpfFileUrl.toString())) { + if (this.downloadedFiles.contains(tpfFileUrl.toString())) { return; } @@ -197,7 +226,8 @@ public class OnionPerfDownloader extends CollecTorMain { } /* Download file contents to temporary file. */ - File tempFile = new File(this.recentDirectory, "." + tpfFileName); + File tempFile = new File(this.recentDirectory, + TORPERF + "/." + tpfFileName); byte[] downloadedBytes; try { downloadedBytes = Downloader.downloadFromHttpServer( @@ -263,7 +293,7 @@ public class OnionPerfDownloader extends CollecTorMain { /* Copy/move files in place. */ File archiveFile = new File(this.archiveDirectory, - date.replaceAll("-", "/") + "/" + tpfFileName); + TORPERF + "/" + date.replaceAll("-", "/") + "/" + tpfFileName); archiveFile.getParentFile().mkdirs(); try { Files.copy(tempFile.toPath(), archiveFile.toPath(), @@ -274,18 +304,132 @@ public class OnionPerfDownloader extends CollecTorMain { tempFile.delete(); return; } - File recentFile = new File(this.recentDirectory, tpfFileName); + File recentFile = new File(this.recentDirectory, + TORPERF + "/" + tpfFileName); + tempFile.renameTo(recentFile); + + /* Add to download history to avoid downloading it again. */ + this.downloadedFiles.add(baseUrl + tpfFileName); + } + + + private void downloadAndParseOnionPerfAnalysisFile(URL baseUrl, String source, + String onionPerfAnalysisFileName) { + URL onionPerfAnalysisFileUrl; + try { + onionPerfAnalysisFileUrl = new URL(baseUrl, onionPerfAnalysisFileName); + } catch (MalformedURLException e1) { + logger.warn("Unable to put together base URL '{}' and file path '{}' to " + + "a URL. Skipping.", baseUrl, onionPerfAnalysisFileName); + return; + } + + /* Skip if we successfully downloaded this file before. */ + if (this.downloadedFiles.contains(onionPerfAnalysisFileUrl.toString())) { + return; + } + + /* Parse date from file name: yyyy-MM-dd.onionperf.analysis.json.xz */ + String date; + try { + date = onionPerfAnalysisFileName.substring(0, 10); + DATE_FORMAT.parse(date); + } catch (NumberFormatException | ParseException e) { + logger.warn("Invalid file name '{}{}'. Skipping.", baseUrl, + onionPerfAnalysisFileName, e); + return; + } + + /* Download file contents to temporary file. */ + File tempFile = new File(this.recentDirectory, + ONIONPERF + "/." + onionPerfAnalysisFileName); + byte[] downloadedBytes; + try { + downloadedBytes = Downloader.downloadFromHttpServer( + new URL(baseUrl + onionPerfAnalysisFileName)); + } catch (IOException e) { + logger.warn("Unable to download '{}{}'. Skipping.", baseUrl, + onionPerfAnalysisFileName, e); + return; + } + if (null == downloadedBytes) { + logger.warn("Unable to download '{}{}'. Skipping.", baseUrl, + onionPerfAnalysisFileName); + return; + } + tempFile.getParentFile().mkdirs(); + try { + Files.write(tempFile.toPath(), downloadedBytes); + } catch (IOException e) { + logger.warn("Unable to write previously downloaded '{}{}' to temporary " + + "file '{}'. Skipping.", baseUrl, onionPerfAnalysisFileName, + tempFile, e); + return; + } + + /* Validate contained descriptors. */ + DescriptorParser descriptorParser = + DescriptorSourceFactory.createDescriptorParser(); + byte[] rawDescriptorBytes; + try { + rawDescriptorBytes = IOUtils.toByteArray( + Files.newInputStream(tempFile.toPath())); + } catch (IOException e) { + logger.warn("OnionPerf file '{}{}' could not be read. Skipping.", baseUrl, + onionPerfAnalysisFileName, e); + tempFile.delete(); + return; + } + Iterable<Descriptor> descriptors = descriptorParser.parseDescriptors( + rawDescriptorBytes, null, onionPerfAnalysisFileName); + String message = null; + for (Descriptor descriptor : descriptors) { + if (!(descriptor instanceof TorperfResult)) { + message = "File contains descriptors other than an OnionPerf analysis " + + "document: " + descriptor.getClass(); + break; + } + TorperfResult torperf = (TorperfResult) descriptor; + if (!source.equals(torperf.getSource())) { + message = "File contains transfer from another source: " + + torperf.getSource(); + break; + } + } + if (null != message) { + logger.warn("OnionPerf file '{}{}' was found to be invalid: {}. " + + "Skipping.", baseUrl, onionPerfAnalysisFileName, message); + tempFile.delete(); + return; + } + + /* Copy/move files in place. */ + File archiveFile = new File(this.archiveDirectory, + ONIONPERF + "/" + date.replaceAll("-", "/") + "/" + date + "." + source + + ".onionperf.analysis.json.xz"); + archiveFile.getParentFile().mkdirs(); + try { + Files.copy(tempFile.toPath(), archiveFile.toPath(), + StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + logger.warn("Unable to copy OnionPerf file {} to {}. Skipping.", + tempFile, archiveFile, e); + tempFile.delete(); + return; + } + File recentFile = new File(this.recentDirectory, + ONIONPERF + "/" + date + "." + source + ".onionperf.analysis.json.xz"); tempFile.renameTo(recentFile); /* Add to download history to avoid downloading it again. */ - this.downloadedTpfFiles.add(baseUrl + tpfFileName); + this.downloadedFiles.add(baseUrl + onionPerfAnalysisFileName); } - private void writeDownloadedOnionPerfTpfFiles() { + private void writeDownloadedOnionPerfFiles() { this.onionPerfDownloadedFile.getParentFile().mkdirs(); try (BufferedWriter bw = new BufferedWriter(new FileWriter( this.onionPerfDownloadedFile))) { - for (String line : this.downloadedTpfFiles) { + for (String line : this.downloadedFiles) { bw.write(line); bw.newLine(); } diff --git a/src/main/resources/collector.properties b/src/main/resources/collector.properties index 61baed5..2347021 100644 --- a/src/main/resources/collector.properties +++ b/src/main/resources/collector.properties @@ -175,7 +175,7 @@ ExitlistUrl = https://check.torproject.org/exit-addresses ######## OnionPerf downloader ######## # ## Define descriptor sources -# possible values: Remote,Sync +# possible values: Remote,Sync (.tpf files only!) OnionPerfSources = Remote # Retrieve files from the following CollecTor instances. # List of URLs separated by comma. diff --git a/src/main/resources/create-tarballs.sh b/src/main/resources/create-tarballs.sh index 07952c7..fcac2f3 100755 --- a/src/main/resources/create-tarballs.sh +++ b/src/main/resources/create-tarballs.sh @@ -40,6 +40,8 @@ TARBALLS=( exit-list-$YEARTWO-$MONTHTWO torperf-$YEARONE-$MONTHONE torperf-$YEARTWO-$MONTHTWO + onionperf-$YEARONE-$MONTHONE + onionperf-$YEARTWO-$MONTHTWO certs microdescs-$YEARONE-$MONTHONE microdescs-$YEARTWO-$MONTHTWO @@ -73,6 +75,8 @@ DIRECTORIES=( $OUTDIR/exit-lists/$YEARTWO/$MONTHTWO/ $OUTDIR/torperf/$YEARONE/$MONTHONE/ $OUTDIR/torperf/$YEARTWO/$MONTHTWO/ + $OUTDIR/onionperf/$YEARONE/$MONTHONE/ + $OUTDIR/onionperf/$YEARTWO/$MONTHTWO/ $OUTDIR/relay-descriptors/certs/ $OUTDIR/relay-descriptors/microdesc/$YEARONE/$MONTHONE $OUTDIR/relay-descriptors/microdesc/$YEARTWO/$MONTHTWO @@ -178,6 +182,9 @@ ln -f -s -t $ARCHIVEDIR/relay-descriptors/bandwidths/ $TARBALLTARGETDIR/bandwidt mkdir -p $ARCHIVEDIR/torperf/ ln -f -s -t $ARCHIVEDIR/torperf/ $TARBALLTARGETDIR/torperf-20??-??.tar.xz +mkdir -p $ARCHIVEDIR/onionperf/ +ln -f -s -t $ARCHIVEDIR/onionperf/ $TARBALLTARGETDIR/onionperf-20??-??.tar.xz + mkdir -p $ARCHIVEDIR/webstats/ ln -f -s -t $ARCHIVEDIR/webstats/ $TARBALLTARGETDIR/webstats-20??-??.tar

1 0

[metrics-lib/master] Bump version to 2.12.1-dev.
by karsten＠torproject.org 30 Apr '20

30 Apr '20

commit 037f463761efeb9d7cfb50442c9c3a68a25120f3 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Apr 30 17:16:47 2020 +0200 Bump version to 2.12.1-dev. --- CHANGELOG.md | 3 +++ build.xml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9870962..bb8430d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +# Changes in version 2.??.? - 2020-??-?? + + # Changes in version 2.12.1 - 2020-04-30 * Minor changes diff --git a/build.xml b/build.xml index ab62a88..ac4a3ed 100644 --- a/build.xml +++ b/build.xml @@ -7,7 +7,7 @@ <project default="usage" name="metrics-lib" basedir="." xmlns:ivy="antlib:org.apache.ivy.ant"> - <property name="release.version" value="2.12.1" /> + <property name="release.version" value="2.12.1-dev" /> <property name="javadoc-title" value="Tor Metrics Library API Documentation"/> <property name="javadoc-excludes" value="**/impl/** **/index/** **/internal/** **/log/**" /> <property name="implementation-title" value="Tor Metrics Library" />

1 0

[metrics-lib/master] Prepare for 2.12.1 release.
by karsten＠torproject.org 30 Apr '20

30 Apr '20

commit aa55fd05f01f71b113cbf72b60d6aba77aa04bfd Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Apr 30 17:11:37 2020 +0200 Prepare for 2.12.1 release. --- CHANGELOG.md | 2 +- build.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fbc4bd..9870962 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 2.??.? - 2020-??-?? +# Changes in version 2.12.1 - 2020-04-30 * Minor changes - Change back how we treat xz-compressed files by leaving diff --git a/build.xml b/build.xml index e3ed6e6..ab62a88 100644 --- a/build.xml +++ b/build.xml @@ -7,7 +7,7 @@ <project default="usage" name="metrics-lib" basedir="." xmlns:ivy="antlib:org.apache.ivy.ant"> - <property name="release.version" value="2.12.0-dev" /> + <property name="release.version" value="2.12.1" /> <property name="javadoc-title" value="Tor Metrics Library API Documentation"/> <property name="javadoc-excludes" value="**/impl/** **/index/** **/internal/** **/log/**" /> <property name="implementation-title" value="Tor Metrics Library" />

1 0

[metrics-lib/master] Change back how we treat xz-compressed files.
by karsten＠torproject.org 30 Apr '20

30 Apr '20

commit d4bb28525f550ea72ab396f24d4fc101e9f48eae Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Apr 30 17:08:44 2020 +0200 Change back how we treat xz-compressed files. It seemed like a good idea to decompress xz-compressed files before handing over decompressed bytes to descriptor implementations. However, our web logs expect descriptor bytes to be in compressed form, and now they're unhappy about receiving uncompressed bytes. Changing this back and teaching OnionPerf analysis files to expect compressed bytes, too. Found by running CollecTor's unit tests. --- CHANGELOG.md | 4 ++++ .../org/torproject/descriptor/impl/DescriptorReaderImpl.java | 2 -- .../descriptor/onionperf/OnionPerfAnalysisConverter.java | 12 +++++++++++- .../descriptor/onionperf/OnionPerfAnalysisConverterTest.java | 5 +---- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 682980b..8fbc4bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changes in version 2.??.? - 2020-??-?? + * Minor changes + - Change back how we treat xz-compressed files by leaving + decompression to descriptor implementations. + # Changes in version 2.12.0 - 2020-04-30 diff --git a/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java b/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java index 0da32ad..8ec04a5 100644 --- a/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java @@ -330,8 +330,6 @@ public class DescriptorReaderImpl implements DescriptorReader { InputStream is = fis; if (file.getName().endsWith(".gz")) { is = new GzipCompressorInputStream(fis); - } else if (file.getName().endsWith(".xz")) { - is = new XZCompressorInputStream(fis); } byte[] rawDescriptorBytes = IOUtils.toByteArray(is); if (rawDescriptorBytes.length > 0) { diff --git a/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java b/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java index 1f9b1f8..18adf8c 100644 --- a/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java +++ b/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java @@ -7,8 +7,13 @@ import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.impl.TorperfResultImpl; +import org.apache.commons.compress.compressors.xz.XZCompressorInputStream; +import org.apache.commons.compress.utils.IOUtils; + +import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -66,8 +71,13 @@ public class OnionPerfAnalysisConverter { public List<Descriptor> asTorperfResults() throws DescriptorParseException { ParsedOnionPerfAnalysis parsedOnionPerfAnalysis; try { - parsedOnionPerfAnalysis = ParsedOnionPerfAnalysis.fromBytes( + InputStream compressedInputStream = new ByteArrayInputStream( this.rawDescriptorBytes); + InputStream decompressedInputStream = new XZCompressorInputStream( + compressedInputStream); + byte[] decompressedBytes = IOUtils.toByteArray(decompressedInputStream); + parsedOnionPerfAnalysis = ParsedOnionPerfAnalysis.fromBytes( + decompressedBytes); } catch (IOException ioException) { throw new DescriptorParseException("Ran into an I/O error while " + "attempting to parse an OnionPerf analysis document.", diff --git a/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java b/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java index d840dd2..c921a2d 100644 --- a/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java +++ b/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java @@ -10,7 +10,6 @@ import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.TorperfResult; -import org.apache.commons.compress.compressors.xz.XZCompressorInputStream; import org.apache.commons.compress.utils.IOUtils; import org.junit.Test; @@ -88,9 +87,7 @@ public class OnionPerfAnalysisConverterTest { assertNotNull(resouce); InputStream compressedInputStream = resouce.openStream(); assertNotNull(compressedInputStream); - InputStream uncompressedInputStream = new XZCompressorInputStream( - compressedInputStream); - byte[] rawDescriptorBytes = IOUtils.toByteArray(uncompressedInputStream); + byte[] rawDescriptorBytes = IOUtils.toByteArray(compressedInputStream); OnionPerfAnalysisConverter onionPerfAnalysisConverter = new OnionPerfAnalysisConverter(rawDescriptorBytes, null); for (Descriptor descriptor

1 0

[metrics-lib/release] Bump version to 2.12.0-dev.
by karsten＠torproject.org 30 Apr '20

30 Apr '20

commit 7efeec721f7e76f10c547603ac0bd73dec3be7ef Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Apr 30 16:14:25 2020 +0200 Bump version to 2.12.0-dev. --- CHANGELOG.md | 3 +++ build.xml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 675fb09..682980b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +# Changes in version 2.??.? - 2020-??-?? + + # Changes in version 2.12.0 - 2020-04-30 * Medium changes diff --git a/build.xml b/build.xml index 98b8b69..e3ed6e6 100644 --- a/build.xml +++ b/build.xml @@ -7,7 +7,7 @@ <project default="usage" name="metrics-lib" basedir="." xmlns:ivy="antlib:org.apache.ivy.ant"> - <property name="release.version" value="2.12.0" /> + <property name="release.version" value="2.12.0-dev" /> <property name="javadoc-title" value="Tor Metrics Library API Documentation"/> <property name="javadoc-excludes" value="**/impl/** **/index/** **/internal/** **/log/**" /> <property name="implementation-title" value="Tor Metrics Library" />

1 0

[metrics-lib/release] Prepare for 2.12.1 release.
by karsten＠torproject.org 30 Apr '20

30 Apr '20

commit aa55fd05f01f71b113cbf72b60d6aba77aa04bfd Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Apr 30 17:11:37 2020 +0200 Prepare for 2.12.1 release. --- CHANGELOG.md | 2 +- build.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fbc4bd..9870962 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 2.??.? - 2020-??-?? +# Changes in version 2.12.1 - 2020-04-30 * Minor changes - Change back how we treat xz-compressed files by leaving diff --git a/build.xml b/build.xml index e3ed6e6..ab62a88 100644 --- a/build.xml +++ b/build.xml @@ -7,7 +7,7 @@ <project default="usage" name="metrics-lib" basedir="." xmlns:ivy="antlib:org.apache.ivy.ant"> - <property name="release.version" value="2.12.0-dev" /> + <property name="release.version" value="2.12.1" /> <property name="javadoc-title" value="Tor Metrics Library API Documentation"/> <property name="javadoc-excludes" value="**/impl/** **/index/** **/internal/** **/log/**" /> <property name="implementation-title" value="Tor Metrics Library" />

1 0

[metrics-lib/release] Change back how we treat xz-compressed files.
by karsten＠torproject.org 30 Apr '20

30 Apr '20

commit d4bb28525f550ea72ab396f24d4fc101e9f48eae Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Apr 30 17:08:44 2020 +0200 Change back how we treat xz-compressed files. It seemed like a good idea to decompress xz-compressed files before handing over decompressed bytes to descriptor implementations. However, our web logs expect descriptor bytes to be in compressed form, and now they're unhappy about receiving uncompressed bytes. Changing this back and teaching OnionPerf analysis files to expect compressed bytes, too. Found by running CollecTor's unit tests. --- CHANGELOG.md | 4 ++++ .../org/torproject/descriptor/impl/DescriptorReaderImpl.java | 2 -- .../descriptor/onionperf/OnionPerfAnalysisConverter.java | 12 +++++++++++- .../descriptor/onionperf/OnionPerfAnalysisConverterTest.java | 5 +---- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 682980b..8fbc4bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changes in version 2.??.? - 2020-??-?? + * Minor changes + - Change back how we treat xz-compressed files by leaving + decompression to descriptor implementations. + # Changes in version 2.12.0 - 2020-04-30 diff --git a/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java b/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java index 0da32ad..8ec04a5 100644 --- a/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java @@ -330,8 +330,6 @@ public class DescriptorReaderImpl implements DescriptorReader { InputStream is = fis; if (file.getName().endsWith(".gz")) { is = new GzipCompressorInputStream(fis); - } else if (file.getName().endsWith(".xz")) { - is = new XZCompressorInputStream(fis); } byte[] rawDescriptorBytes = IOUtils.toByteArray(is); if (rawDescriptorBytes.length > 0) { diff --git a/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java b/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java index 1f9b1f8..18adf8c 100644 --- a/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java +++ b/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java @@ -7,8 +7,13 @@ import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.impl.TorperfResultImpl; +import org.apache.commons.compress.compressors.xz.XZCompressorInputStream; +import org.apache.commons.compress.utils.IOUtils; + +import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -66,8 +71,13 @@ public class OnionPerfAnalysisConverter { public List<Descriptor> asTorperfResults() throws DescriptorParseException { ParsedOnionPerfAnalysis parsedOnionPerfAnalysis; try { - parsedOnionPerfAnalysis = ParsedOnionPerfAnalysis.fromBytes( + InputStream compressedInputStream = new ByteArrayInputStream( this.rawDescriptorBytes); + InputStream decompressedInputStream = new XZCompressorInputStream( + compressedInputStream); + byte[] decompressedBytes = IOUtils.toByteArray(decompressedInputStream); + parsedOnionPerfAnalysis = ParsedOnionPerfAnalysis.fromBytes( + decompressedBytes); } catch (IOException ioException) { throw new DescriptorParseException("Ran into an I/O error while " + "attempting to parse an OnionPerf analysis document.", diff --git a/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java b/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java index d840dd2..c921a2d 100644 --- a/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java +++ b/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java @@ -10,7 +10,6 @@ import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.TorperfResult; -import org.apache.commons.compress.compressors.xz.XZCompressorInputStream; import org.apache.commons.compress.utils.IOUtils; import org.junit.Test; @@ -88,9 +87,7 @@ public class OnionPerfAnalysisConverterTest { assertNotNull(resouce); InputStream compressedInputStream = resouce.openStream(); assertNotNull(compressedInputStream); - InputStream uncompressedInputStream = new XZCompressorInputStream( - compressedInputStream); - byte[] rawDescriptorBytes = IOUtils.toByteArray(uncompressedInputStream); + byte[] rawDescriptorBytes = IOUtils.toByteArray(compressedInputStream); OnionPerfAnalysisConverter onionPerfAnalysisConverter = new OnionPerfAnalysisConverter(rawDescriptorBytes, null); for (Descriptor descriptor

1 0