commit 393a8d6d5b8f4d037127bcdddca49e6fc43076d5 Author: Karsten Loesing karsten.loesing@gmx.net Date: Thu May 12 21:43:59 2016 +0200
Remove unused bridge pool assignments code. --- bin/run-bridgepools | 3 - config.template | 10 - src/org/torproject/collector/Main.java | 2 - .../BridgePoolAssignmentsProcessor.java | 273 --------------------- .../torproject/collector/main/Configuration.java | 15 -- 5 files changed, 303 deletions(-)
diff --git a/bin/run-bridgepools b/bin/run-bridgepools deleted file mode 100755 index 62d71f7..0000000 --- a/bin/run-bridgepools +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -`dirname $0`/run-all `basename $0` - diff --git a/config.template b/config.template index cd31802..88407a2 100644 --- a/config.template +++ b/config.template @@ -85,16 +85,6 @@ #SanitizedBridgesWriteDirectory out/bridge-descriptors/ # # -######## Bridge pool assignments ######## -# -## Relative path to directory to read bridge pool assignment files from -#AssignmentsDirectory in/bridge-pool-assignments/ -# -## Relative path to directory to write sanitized bridge pool assignment -## files to -#SanitizedAssignmentsDirectory out/bridge-pool-assignments/ -# -# ######## Exit lists ######## # ## (No options available) diff --git a/src/org/torproject/collector/Main.java b/src/org/torproject/collector/Main.java index 44182ed..9c64696 100644 --- a/src/org/torproject/collector/Main.java +++ b/src/org/torproject/collector/Main.java @@ -4,7 +4,6 @@ package org.torproject.collector;
import org.torproject.collector.bridgedescs.SanitizedBridgesWriter; -import org.torproject.collector.bridgepools.BridgePoolAssignmentsProcessor; import org.torproject.collector.exitlists.ExitListDownloader; import org.torproject.collector.index.CreateIndexJson; import org.torproject.collector.relaydescs.ArchiveWriter; @@ -33,7 +32,6 @@ public class Main {
static { // add a new main class here collecTorMains.put("bridgedescs", SanitizedBridgesWriter.class); - collecTorMains.put("bridgepools", BridgePoolAssignmentsProcessor.class); collecTorMains.put("exitlists", ExitListDownloader.class); collecTorMains.put("updateindex", CreateIndexJson.class); collecTorMains.put("relaydescs", ArchiveWriter.class); diff --git a/src/org/torproject/collector/bridgepools/BridgePoolAssignmentsProcessor.java b/src/org/torproject/collector/bridgepools/BridgePoolAssignmentsProcessor.java deleted file mode 100644 index 2b6c262..0000000 --- a/src/org/torproject/collector/bridgepools/BridgePoolAssignmentsProcessor.java +++ /dev/null @@ -1,273 +0,0 @@ -/* Copyright 2011--2016 The Tor Project - * See LICENSE for licensing information */ - -package org.torproject.collector.bridgepools; - -import org.torproject.collector.main.Configuration; -import org.torproject.collector.main.LockFile; - -import org.apache.commons.codec.DecoderException; -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStreamReader; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.SortedSet; -import java.util.Stack; -import java.util.TimeZone; -import java.util.TreeSet; -import java.util.logging.Level; -import java.util.logging.Logger; - -public class BridgePoolAssignmentsProcessor extends Thread { - - public static void main(String[] args) { - - Logger logger = Logger.getLogger( - BridgePoolAssignmentsProcessor.class.getName()); - logger.info("Starting bridge-pool-assignments module of CollecTor."); - - // Initialize configuration - Configuration config = new Configuration(); - - // Use lock file to avoid overlapping runs - LockFile lf = new LockFile("bridge-pool-assignments"); - if (!lf.acquireLock()) { - logger.severe("Warning: CollecTor is already running or has not exited " - + "cleanly! Exiting!"); - System.exit(1); - } - - // Process bridge pool assignments - new BridgePoolAssignmentsProcessor(config).run(); - - // Remove lock file - lf.releaseLock(); - - logger.info("Terminating bridge-pool-assignments module of CollecTor."); - } - - private Configuration config; - - public BridgePoolAssignmentsProcessor(Configuration config) { - this.config = config; - } - - public void run() { - - File assignmentsDirectory = - new File(config.getAssignmentsDirectory()); - File sanitizedAssignmentsDirectory = - new File(config.getSanitizedAssignmentsDirectory()); - - Logger logger = - Logger.getLogger(BridgePoolAssignmentsProcessor.class.getName()); - if (assignmentsDirectory == null - || sanitizedAssignmentsDirectory == null) { - IllegalArgumentException e = new IllegalArgumentException("Neither " - + "assignmentsDirectory nor sanitizedAssignmentsDirectory may " - + "be null!"); - throw e; - } - - List<File> assignmentFiles = new ArrayList<File>(); - Stack<File> files = new Stack<File>(); - files.add(assignmentsDirectory); - while (!files.isEmpty()) { - File file = files.pop(); - if (file.isDirectory()) { - files.addAll(Arrays.asList(file.listFiles())); - } else if (file.getName().equals("assignments.log")) { - assignmentFiles.add(file); - } - } - - SimpleDateFormat assignmentFormat = - new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - assignmentFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - SimpleDateFormat filenameFormat = - new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss"); - filenameFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - String duplicateFingerprint = null; - long maxBridgePoolAssignmentTime = 0L; - for (File assignmentFile : assignmentFiles) { - logger.info("Processing bridge pool assignment file '" - + assignmentFile.getAbsolutePath() + "'..."); - try { - BufferedReader br = null; - if (assignmentFile.getName().endsWith(".gz")) { - br = new BufferedReader(new InputStreamReader( - new GzipCompressorInputStream(new FileInputStream( - assignmentFile)))); - } else { - br = new BufferedReader(new FileReader(assignmentFile)); - } - String line; - String bridgePoolAssignmentLine = null; - SortedSet<String> sanitizedAssignments = new TreeSet<String>(); - boolean wroteLastLine = false; - boolean skipBefore20120504125947 = true; - Set<String> hashedFingerprints = null; - while ((line = br.readLine()) != null || !wroteLastLine) { - if (line != null && line.startsWith("bridge-pool-assignment ")) { - String[] parts = line.split(" "); - if (parts.length != 3) { - continue; - } - /* TODO Take out this temporary hack to ignore all assignments - * coming from ponticum when byblos was still the official - * BridgeDB host. */ - if (line.compareTo( - "bridge-pool-assignment 2012-05-04 12:59:47") >= 0) { - skipBefore20120504125947 = false; - } - } - if (skipBefore20120504125947) { - if (line == null) { - break; - } else { - continue; - } - } - if (line == null - || line.startsWith("bridge-pool-assignment ")) { - if (bridgePoolAssignmentLine != null) { - try { - long bridgePoolAssignmentTime = assignmentFormat.parse( - bridgePoolAssignmentLine.substring( - "bridge-pool-assignment ".length())).getTime(); - maxBridgePoolAssignmentTime = Math.max( - maxBridgePoolAssignmentTime, - bridgePoolAssignmentTime); - File tarballFile = new File( - sanitizedAssignmentsDirectory, filenameFormat.format( - bridgePoolAssignmentTime)); - File rsyncFile = new File( - "recent/bridge-pool-assignments/" - + tarballFile.getName()); - File[] outputFiles = new File[] { tarballFile, - rsyncFile }; - for (File outputFile : outputFiles) { - if (!outputFile.exists()) { - outputFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter( - outputFile)); - bw.write("@type bridge-pool-assignment 1.0\n"); - bw.write(bridgePoolAssignmentLine + "\n"); - for (String assignmentLine : sanitizedAssignments) { - bw.write(assignmentLine + "\n"); - } - bw.close(); - } - } - } catch (IOException e) { - logger.log(Level.WARNING, "Could not write sanitized " - + "bridge pool assignment file for line '" - + bridgePoolAssignmentLine + "' to disk. Skipping " - + "bridge pool assignment file '" - + assignmentFile.getAbsolutePath() + "'.", e); - break; - } catch (ParseException e) { - logger.log(Level.WARNING, "Could not write sanitized " - + "bridge pool assignment file for line '" - + bridgePoolAssignmentLine + "' to disk. Skipping " - + "bridge pool assignment file '" - + assignmentFile.getAbsolutePath() + "'.", e); - break; - } - sanitizedAssignments.clear(); - } - if (line == null) { - wroteLastLine = true; - } else { - bridgePoolAssignmentLine = line; - hashedFingerprints = new HashSet<String>(); - } - } else { - String[] parts = line.split(" "); - if (parts.length < 2 || parts[0].length() < 40) { - logger.warning("Unrecognized line '" + line - + "'. Aborting."); - break; - } - String hashedFingerprint = null; - try { - hashedFingerprint = DigestUtils.shaHex(Hex.decodeHex( - line.split(" ")[0].toCharArray())).toLowerCase(); - } catch (DecoderException e) { - logger.warning("Unable to decode hex fingerprint in line '" - + line + "'. Aborting."); - break; - } - if (hashedFingerprints.contains(hashedFingerprint)) { - duplicateFingerprint = bridgePoolAssignmentLine; - } - hashedFingerprints.add(hashedFingerprint); - String assignmentDetails = line.substring(40); - sanitizedAssignments.add(hashedFingerprint - + assignmentDetails); - } - } - br.close(); - } catch (IOException e) { - logger.log(Level.WARNING, "Could not read bridge pool assignment " - + "file '" + assignmentFile.getAbsolutePath() - + "'. Skipping.", e); - } - } - - if (duplicateFingerprint != null) { - logger.warning("At least one bridge pool assignment list contained " - + "duplicate fingerprints. Last found in assignment list " - + "starting with '" + duplicateFingerprint + "'."); - } - - if (maxBridgePoolAssignmentTime > 0L - && maxBridgePoolAssignmentTime + 330L * 60L * 1000L - < System.currentTimeMillis()) { - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - logger.warning("The last known bridge pool assignment list was " - + "published at " - + dateTimeFormat.format(maxBridgePoolAssignmentTime) - + ", which is more than 5:30 hours in the past."); - } - - this.cleanUpRsyncDirectory(); - - logger.info("Finished processing bridge pool assignment file(s)."); - } - - /* Delete all files from the rsync directory that have not been modified - * in the last three days. */ - public void cleanUpRsyncDirectory() { - long cutOffMillis = System.currentTimeMillis() - - 3L * 24L * 60L * 60L * 1000L; - Stack<File> allFiles = new Stack<File>(); - allFiles.add(new File("recent/bridge-pool-assignments")); - while (!allFiles.isEmpty()) { - File file = allFiles.pop(); - if (file.isDirectory()) { - allFiles.addAll(Arrays.asList(file.listFiles())); - } else if (file.lastModified() < cutOffMillis) { - file.delete(); - } - } - } -} - diff --git a/src/org/torproject/collector/main/Configuration.java b/src/org/torproject/collector/main/Configuration.java index 9a511d3..aee1d02 100644 --- a/src/org/torproject/collector/main/Configuration.java +++ b/src/org/torproject/collector/main/Configuration.java @@ -62,9 +62,6 @@ public class Configuration { private boolean downloadAllServerDescriptors = false; private boolean downloadAllExtraInfoDescriptors = false; private boolean compressRelayDescriptorDownloads; - private String assignmentsDirectory = "in/bridge-pool-assignments/"; - private String sanitizedAssignmentsDirectory = - "out/bridge-pool-assignments/"; private String torperfOutputDirectory = "out/torperf/"; private SortedMap<String, String> torperfSources = null; private List<String> torperfFiles = null; @@ -170,10 +167,6 @@ public class Configuration { } else if (line.startsWith("CompressRelayDescriptorDownloads")) { this.compressRelayDescriptorDownloads = Integer.parseInt( line.split(" ")[1]) != 0; - } else if (line.startsWith("AssignmentsDirectory")) { - this.assignmentsDirectory = line.split(" ")[1]; - } else if (line.startsWith("SanitizedAssignmentsDirectory")) { - this.sanitizedAssignmentsDirectory = line.split(" ")[1]; } else if (line.startsWith("TorperfOutputDirectory")) { this.torperfOutputDirectory = line.split(" ")[1]; } else if (line.startsWith("TorperfSource")) { @@ -310,14 +303,6 @@ public class Configuration { return this.compressRelayDescriptorDownloads; }
- public String getAssignmentsDirectory() { - return this.assignmentsDirectory; - } - - public String getSanitizedAssignmentsDirectory() { - return this.sanitizedAssignmentsDirectory; - } - public String getTorperfOutputDirectory() { return this.torperfOutputDirectory; }