[tor-commits] [collector/master] Remove unused bridge pool assignments code.
karsten at torproject.org
karsten at torproject.org
Thu May 12 19:45:00 UTC 2016
commit 393a8d6d5b8f4d037127bcdddca49e6fc43076d5
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Thu May 12 21:43:59 2016 +0200
Remove unused bridge pool assignments code.
---
bin/run-bridgepools | 3 -
config.template | 10 -
src/org/torproject/collector/Main.java | 2 -
.../BridgePoolAssignmentsProcessor.java | 273 ---------------------
.../torproject/collector/main/Configuration.java | 15 --
5 files changed, 303 deletions(-)
diff --git a/bin/run-bridgepools b/bin/run-bridgepools
deleted file mode 100755
index 62d71f7..0000000
--- a/bin/run-bridgepools
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-`dirname $0`/run-all `basename $0`
-
diff --git a/config.template b/config.template
index cd31802..88407a2 100644
--- a/config.template
+++ b/config.template
@@ -85,16 +85,6 @@
#SanitizedBridgesWriteDirectory out/bridge-descriptors/
#
#
-######## Bridge pool assignments ########
-#
-## Relative path to directory to read bridge pool assignment files from
-#AssignmentsDirectory in/bridge-pool-assignments/
-#
-## Relative path to directory to write sanitized bridge pool assignment
-## files to
-#SanitizedAssignmentsDirectory out/bridge-pool-assignments/
-#
-#
######## Exit lists ########
#
## (No options available)
diff --git a/src/org/torproject/collector/Main.java b/src/org/torproject/collector/Main.java
index 44182ed..9c64696 100644
--- a/src/org/torproject/collector/Main.java
+++ b/src/org/torproject/collector/Main.java
@@ -4,7 +4,6 @@
package org.torproject.collector;
import org.torproject.collector.bridgedescs.SanitizedBridgesWriter;
-import org.torproject.collector.bridgepools.BridgePoolAssignmentsProcessor;
import org.torproject.collector.exitlists.ExitListDownloader;
import org.torproject.collector.index.CreateIndexJson;
import org.torproject.collector.relaydescs.ArchiveWriter;
@@ -33,7 +32,6 @@ public class Main {
static { // add a new main class here
collecTorMains.put("bridgedescs", SanitizedBridgesWriter.class);
- collecTorMains.put("bridgepools", BridgePoolAssignmentsProcessor.class);
collecTorMains.put("exitlists", ExitListDownloader.class);
collecTorMains.put("updateindex", CreateIndexJson.class);
collecTorMains.put("relaydescs", ArchiveWriter.class);
diff --git a/src/org/torproject/collector/bridgepools/BridgePoolAssignmentsProcessor.java b/src/org/torproject/collector/bridgepools/BridgePoolAssignmentsProcessor.java
deleted file mode 100644
index 2b6c262..0000000
--- a/src/org/torproject/collector/bridgepools/BridgePoolAssignmentsProcessor.java
+++ /dev/null
@@ -1,273 +0,0 @@
-/* Copyright 2011--2016 The Tor Project
- * See LICENSE for licensing information */
-
-package org.torproject.collector.bridgepools;
-
-import org.torproject.collector.main.Configuration;
-import org.torproject.collector.main.LockFile;
-
-import org.apache.commons.codec.DecoderException;
-import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.Stack;
-import java.util.TimeZone;
-import java.util.TreeSet;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-public class BridgePoolAssignmentsProcessor extends Thread {
-
- public static void main(String[] args) {
-
- Logger logger = Logger.getLogger(
- BridgePoolAssignmentsProcessor.class.getName());
- logger.info("Starting bridge-pool-assignments module of CollecTor.");
-
- // Initialize configuration
- Configuration config = new Configuration();
-
- // Use lock file to avoid overlapping runs
- LockFile lf = new LockFile("bridge-pool-assignments");
- if (!lf.acquireLock()) {
- logger.severe("Warning: CollecTor is already running or has not exited "
- + "cleanly! Exiting!");
- System.exit(1);
- }
-
- // Process bridge pool assignments
- new BridgePoolAssignmentsProcessor(config).run();
-
- // Remove lock file
- lf.releaseLock();
-
- logger.info("Terminating bridge-pool-assignments module of CollecTor.");
- }
-
- private Configuration config;
-
- public BridgePoolAssignmentsProcessor(Configuration config) {
- this.config = config;
- }
-
- public void run() {
-
- File assignmentsDirectory =
- new File(config.getAssignmentsDirectory());
- File sanitizedAssignmentsDirectory =
- new File(config.getSanitizedAssignmentsDirectory());
-
- Logger logger =
- Logger.getLogger(BridgePoolAssignmentsProcessor.class.getName());
- if (assignmentsDirectory == null
- || sanitizedAssignmentsDirectory == null) {
- IllegalArgumentException e = new IllegalArgumentException("Neither "
- + "assignmentsDirectory nor sanitizedAssignmentsDirectory may "
- + "be null!");
- throw e;
- }
-
- List<File> assignmentFiles = new ArrayList<File>();
- Stack<File> files = new Stack<File>();
- files.add(assignmentsDirectory);
- while (!files.isEmpty()) {
- File file = files.pop();
- if (file.isDirectory()) {
- files.addAll(Arrays.asList(file.listFiles()));
- } else if (file.getName().equals("assignments.log")) {
- assignmentFiles.add(file);
- }
- }
-
- SimpleDateFormat assignmentFormat =
- new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- assignmentFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- SimpleDateFormat filenameFormat =
- new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
- filenameFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- String duplicateFingerprint = null;
- long maxBridgePoolAssignmentTime = 0L;
- for (File assignmentFile : assignmentFiles) {
- logger.info("Processing bridge pool assignment file '"
- + assignmentFile.getAbsolutePath() + "'...");
- try {
- BufferedReader br = null;
- if (assignmentFile.getName().endsWith(".gz")) {
- br = new BufferedReader(new InputStreamReader(
- new GzipCompressorInputStream(new FileInputStream(
- assignmentFile))));
- } else {
- br = new BufferedReader(new FileReader(assignmentFile));
- }
- String line;
- String bridgePoolAssignmentLine = null;
- SortedSet<String> sanitizedAssignments = new TreeSet<String>();
- boolean wroteLastLine = false;
- boolean skipBefore20120504125947 = true;
- Set<String> hashedFingerprints = null;
- while ((line = br.readLine()) != null || !wroteLastLine) {
- if (line != null && line.startsWith("bridge-pool-assignment ")) {
- String[] parts = line.split(" ");
- if (parts.length != 3) {
- continue;
- }
- /* TODO Take out this temporary hack to ignore all assignments
- * coming from ponticum when byblos was still the official
- * BridgeDB host. */
- if (line.compareTo(
- "bridge-pool-assignment 2012-05-04 12:59:47") >= 0) {
- skipBefore20120504125947 = false;
- }
- }
- if (skipBefore20120504125947) {
- if (line == null) {
- break;
- } else {
- continue;
- }
- }
- if (line == null
- || line.startsWith("bridge-pool-assignment ")) {
- if (bridgePoolAssignmentLine != null) {
- try {
- long bridgePoolAssignmentTime = assignmentFormat.parse(
- bridgePoolAssignmentLine.substring(
- "bridge-pool-assignment ".length())).getTime();
- maxBridgePoolAssignmentTime = Math.max(
- maxBridgePoolAssignmentTime,
- bridgePoolAssignmentTime);
- File tarballFile = new File(
- sanitizedAssignmentsDirectory, filenameFormat.format(
- bridgePoolAssignmentTime));
- File rsyncFile = new File(
- "recent/bridge-pool-assignments/"
- + tarballFile.getName());
- File[] outputFiles = new File[] { tarballFile,
- rsyncFile };
- for (File outputFile : outputFiles) {
- if (!outputFile.exists()) {
- outputFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- outputFile));
- bw.write("@type bridge-pool-assignment 1.0\n");
- bw.write(bridgePoolAssignmentLine + "\n");
- for (String assignmentLine : sanitizedAssignments) {
- bw.write(assignmentLine + "\n");
- }
- bw.close();
- }
- }
- } catch (IOException e) {
- logger.log(Level.WARNING, "Could not write sanitized "
- + "bridge pool assignment file for line '"
- + bridgePoolAssignmentLine + "' to disk. Skipping "
- + "bridge pool assignment file '"
- + assignmentFile.getAbsolutePath() + "'.", e);
- break;
- } catch (ParseException e) {
- logger.log(Level.WARNING, "Could not write sanitized "
- + "bridge pool assignment file for line '"
- + bridgePoolAssignmentLine + "' to disk. Skipping "
- + "bridge pool assignment file '"
- + assignmentFile.getAbsolutePath() + "'.", e);
- break;
- }
- sanitizedAssignments.clear();
- }
- if (line == null) {
- wroteLastLine = true;
- } else {
- bridgePoolAssignmentLine = line;
- hashedFingerprints = new HashSet<String>();
- }
- } else {
- String[] parts = line.split(" ");
- if (parts.length < 2 || parts[0].length() < 40) {
- logger.warning("Unrecognized line '" + line
- + "'. Aborting.");
- break;
- }
- String hashedFingerprint = null;
- try {
- hashedFingerprint = DigestUtils.shaHex(Hex.decodeHex(
- line.split(" ")[0].toCharArray())).toLowerCase();
- } catch (DecoderException e) {
- logger.warning("Unable to decode hex fingerprint in line '"
- + line + "'. Aborting.");
- break;
- }
- if (hashedFingerprints.contains(hashedFingerprint)) {
- duplicateFingerprint = bridgePoolAssignmentLine;
- }
- hashedFingerprints.add(hashedFingerprint);
- String assignmentDetails = line.substring(40);
- sanitizedAssignments.add(hashedFingerprint
- + assignmentDetails);
- }
- }
- br.close();
- } catch (IOException e) {
- logger.log(Level.WARNING, "Could not read bridge pool assignment "
- + "file '" + assignmentFile.getAbsolutePath()
- + "'. Skipping.", e);
- }
- }
-
- if (duplicateFingerprint != null) {
- logger.warning("At least one bridge pool assignment list contained "
- + "duplicate fingerprints. Last found in assignment list "
- + "starting with '" + duplicateFingerprint + "'.");
- }
-
- if (maxBridgePoolAssignmentTime > 0L
- && maxBridgePoolAssignmentTime + 330L * 60L * 1000L
- < System.currentTimeMillis()) {
- SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss");
- dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- logger.warning("The last known bridge pool assignment list was "
- + "published at "
- + dateTimeFormat.format(maxBridgePoolAssignmentTime)
- + ", which is more than 5:30 hours in the past.");
- }
-
- this.cleanUpRsyncDirectory();
-
- logger.info("Finished processing bridge pool assignment file(s).");
- }
-
- /* Delete all files from the rsync directory that have not been modified
- * in the last three days. */
- public void cleanUpRsyncDirectory() {
- long cutOffMillis = System.currentTimeMillis()
- - 3L * 24L * 60L * 60L * 1000L;
- Stack<File> allFiles = new Stack<File>();
- allFiles.add(new File("recent/bridge-pool-assignments"));
- while (!allFiles.isEmpty()) {
- File file = allFiles.pop();
- if (file.isDirectory()) {
- allFiles.addAll(Arrays.asList(file.listFiles()));
- } else if (file.lastModified() < cutOffMillis) {
- file.delete();
- }
- }
- }
-}
-
diff --git a/src/org/torproject/collector/main/Configuration.java b/src/org/torproject/collector/main/Configuration.java
index 9a511d3..aee1d02 100644
--- a/src/org/torproject/collector/main/Configuration.java
+++ b/src/org/torproject/collector/main/Configuration.java
@@ -62,9 +62,6 @@ public class Configuration {
private boolean downloadAllServerDescriptors = false;
private boolean downloadAllExtraInfoDescriptors = false;
private boolean compressRelayDescriptorDownloads;
- private String assignmentsDirectory = "in/bridge-pool-assignments/";
- private String sanitizedAssignmentsDirectory =
- "out/bridge-pool-assignments/";
private String torperfOutputDirectory = "out/torperf/";
private SortedMap<String, String> torperfSources = null;
private List<String> torperfFiles = null;
@@ -170,10 +167,6 @@ public class Configuration {
} else if (line.startsWith("CompressRelayDescriptorDownloads")) {
this.compressRelayDescriptorDownloads = Integer.parseInt(
line.split(" ")[1]) != 0;
- } else if (line.startsWith("AssignmentsDirectory")) {
- this.assignmentsDirectory = line.split(" ")[1];
- } else if (line.startsWith("SanitizedAssignmentsDirectory")) {
- this.sanitizedAssignmentsDirectory = line.split(" ")[1];
} else if (line.startsWith("TorperfOutputDirectory")) {
this.torperfOutputDirectory = line.split(" ")[1];
} else if (line.startsWith("TorperfSource")) {
@@ -310,14 +303,6 @@ public class Configuration {
return this.compressRelayDescriptorDownloads;
}
- public String getAssignmentsDirectory() {
- return this.assignmentsDirectory;
- }
-
- public String getSanitizedAssignmentsDirectory() {
- return this.sanitizedAssignmentsDirectory;
- }
-
public String getTorperfOutputDirectory() {
return this.torperfOutputDirectory;
}
More information about the tor-commits
mailing list