commit e6a0e94535014926c6df53ed728e108a30d6386b Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Mon Jan 9 11:57:11 2012 +0100 Copy recently processed files to an rsync/ directory. --- config.template | 10 ++ src/org/torproject/ernie/db/Configuration.java | 13 ++ src/org/torproject/ernie/db/Main.java | 13 ++ src/org/torproject/ernie/db/RsyncDataProvider.java | 146 ++++++++++++++++++++ 4 files changed, 182 insertions(+), 0 deletions(-) diff --git a/config.template b/config.template index dc5d30e..6cfdce5 100644 --- a/config.template +++ b/config.template @@ -112,4 +112,14 @@ # ## Download exit list and store it to disk #DownloadExitList 0 +# +# +######## Rsync directory ######## +# +## Copy recently published decriptors to another directory to provide them +## via rsync +#ProvideFilesViaRsync 0 +# +## Relative path to the directory that shall be provided via rsync +#RsyncDirectory rsync/ diff --git a/src/org/torproject/ernie/db/Configuration.java b/src/org/torproject/ernie/db/Configuration.java index b4e6f8b..7197c7d 100644 --- a/src/org/torproject/ernie/db/Configuration.java +++ b/src/org/torproject/ernie/db/Configuration.java @@ -49,6 +49,8 @@ public class Configuration { private boolean processBridgePoolAssignments = false; private String assignmentsDirectory = "assignments/"; private String sanitizedAssignmentsDirectory = "sanitized-assignments/"; + private boolean provideFilesViaRsync = false; + private String rsyncDirectory = "rsync"; public Configuration() { /* Initialize logger. */ @@ -168,6 +170,11 @@ public class Configuration { this.assignmentsDirectory = line.split(" ")[1]; } else if (line.startsWith("SanitizedAssignmentsDirectory")) { this.sanitizedAssignmentsDirectory = line.split(" ")[1]; + } else if (line.startsWith("ProvideFilesViaRsync")) { + this.provideFilesViaRsync = Integer.parseInt( + line.split(" ")[1]) != 0; + } else if (line.startsWith("RsyncDirectory")) { + this.rsyncDirectory = line.split(" ")[1]; } else { logger.severe("Configuration file contains unrecognized " + "configuration key in line '" + line + "'! Exiting!"); @@ -317,5 +324,11 @@ public class Configuration { public String getSanitizedAssignmentsDirectory() { return sanitizedAssignmentsDirectory; } + public boolean getProvideFilesViaRsync() { + return this.provideFilesViaRsync; + } + public String getRsyncDirectory() { + return this.rsyncDirectory; + } } diff --git a/src/org/torproject/ernie/db/Main.java b/src/org/torproject/ernie/db/Main.java index 5a06900..a20748a 100644 --- a/src/org/torproject/ernie/db/Main.java +++ b/src/org/torproject/ernie/db/Main.java @@ -136,6 +136,19 @@ public class Main { new File(config.getSanitizedAssignmentsDirectory())); } + // Copy recently published files to a local directory that can then + // be served via rsync. + if (config.getProvideFilesViaRsync()) { + new RsyncDataProvider( + config.getDirectoryArchivesOutputDirectory() == null ? null : + new File(config.getDirectoryArchivesOutputDirectory()), + config.getSanitizedBridgesWriteDirectory() == null ? null : + new File(config.getSanitizedBridgesWriteDirectory()), + config.getSanitizedAssignmentsDirectory() == null ? null : + new File(config.getSanitizedAssignmentsDirectory()), + new File(config.getRsyncDirectory())); + } + // Remove lock file lf.releaseLock(); diff --git a/src/org/torproject/ernie/db/RsyncDataProvider.java b/src/org/torproject/ernie/db/RsyncDataProvider.java new file mode 100644 index 0000000..1f0d68f --- /dev/null +++ b/src/org/torproject/ernie/db/RsyncDataProvider.java @@ -0,0 +1,146 @@ +/* Copyright 2012 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.ernie.db; + +import java.io.*; +import java.util.*; + +/** + * Copy files published in the last 3 days to a local directory that can + * then be served via rsync. + */ +public class RsyncDataProvider { + public RsyncDataProvider(File directoryArchivesOutputDirectory, + File sanitizedBridgesWriteDirectory, + File sanitizedAssignmentsDirectory, File rsyncDirectory) { + + /* Determine the cut-off time for files in rsync/. */ + long cutOffMillis = System.currentTimeMillis() + - 3L * 24L * 60L * 60L * 1000L; + + /* Create rsync/ directory if it doesn't exist. */ + if (!rsyncDirectory.exists()) { + rsyncDirectory.mkdirs(); + } + + /* Make a list of all files in the rsync/ directory to delete those + * that we didn't copy in this run. */ + Set<String> fileNamesInRsync = new HashSet<String>(); + Stack<File> files = new Stack<File>(); + files.add(rsyncDirectory); + while (!files.isEmpty()) { + File pop = files.pop(); + if (pop.isDirectory()) { + files.addAll(Arrays.asList(pop.listFiles())); + } else { + fileNamesInRsync.add(pop.getName()); + } + } + + /* Copy relay descriptors from the last 3 days. */ + if (directoryArchivesOutputDirectory != null) { + files.add(directoryArchivesOutputDirectory); + while (!files.isEmpty()) { + File pop = files.pop(); + if (pop.isDirectory()) { + files.addAll(Arrays.asList(pop.listFiles())); + } else if (pop.lastModified() >= cutOffMillis) { + String fileName = pop.getName(); + if (pop.getAbsolutePath().contains("/consensus/")) { + this.copyFile(pop, new File(rsyncDirectory, + "relay-descriptors/consensuses/" + fileName)); + } else if (pop.getAbsolutePath().contains("/vote/")) { + this.copyFile(pop, new File(rsyncDirectory, + "relay-descriptors/votes/" + fileName)); + } else if (pop.getAbsolutePath().contains( + "/server-descriptor/")) { + this.copyFile(pop, new File(rsyncDirectory, + "relay-descriptors/server-descriptors/" + fileName)); + } else if (pop.getAbsolutePath().contains("/extra-info/")) { + this.copyFile(pop, new File(rsyncDirectory, + "relay-descriptors/extra-infos/" + fileName)); + } else { + continue; + } + fileNamesInRsync.remove(pop.getName()); + } + } + } + + /* Copy sanitized bridge descriptors from the last 3 days. */ + if (sanitizedBridgesWriteDirectory != null) { + files.add(sanitizedBridgesWriteDirectory); + while (!files.isEmpty()) { + File pop = files.pop(); + if (pop.isDirectory()) { + files.addAll(Arrays.asList(pop.listFiles())); + } else if (pop.lastModified() >= cutOffMillis) { + String fileName = pop.getName(); + if (pop.getAbsolutePath().contains("/statuses/")) { + this.copyFile(pop, new File(rsyncDirectory, + "bridge-descriptors/statuses/" + fileName)); + } else if (pop.getAbsolutePath().contains( + "/server-descriptors/")) { + this.copyFile(pop, new File(rsyncDirectory, + "bridge-descriptors/server-descriptors/" + fileName)); + } else if (pop.getAbsolutePath().contains("/extra-infos/")) { + this.copyFile(pop, new File(rsyncDirectory, + "bridge-descriptors/extra-infos/" + fileName)); + } else { + continue; + } + fileNamesInRsync.remove(pop.getName()); + } + } + } + + /* Copy sanitized bridge pool assignments from the last 3 days. */ + if (sanitizedAssignmentsDirectory != null) { + files.add(sanitizedAssignmentsDirectory); + while (!files.isEmpty()) { + File pop = files.pop(); + if (pop.isDirectory()) { + files.addAll(Arrays.asList(pop.listFiles())); + } else if (pop.lastModified() >= cutOffMillis) { + String fileName = pop.getName(); + this.copyFile(pop, new File(rsyncDirectory, + "bridge-pool-assignments/" + fileName)); + fileNamesInRsync.remove(pop.getName()); + } + } + } + + /* Delete all files that we didn't (over-)write in this run. */ + files.add(rsyncDirectory); + while (!files.isEmpty()) { + File pop = files.pop(); + if (pop.isDirectory()) { + files.addAll(Arrays.asList(pop.listFiles())); + } else if (fileNamesInRsync.contains(pop.getName())) { + pop.delete(); + } + } + } + + private void copyFile(File from, File to) { + if (to.exists()) { + return; + } + try { + to.getParentFile().mkdirs(); + FileInputStream fis = new FileInputStream(from); + BufferedInputStream bis = new BufferedInputStream(fis); + FileOutputStream fos = new FileOutputStream(to); + int len; + byte[] data = new byte[1024]; + while ((len = bis.read(data, 0, 1024)) >= 0) { + fos.write(data, 0, len); + } + bis.close(); + fos.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } +} +