[or-cvs] [ernie/master] Parse cached-* files from local Tor client and sort descriptors into directory structure.
karsten at torproject.org
karsten at torproject.org
Mon Feb 22 14:25:35 UTC 2010
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Mon, 22 Feb 2010 15:24:30 +0100
Subject: Parse cached-* files from local Tor client and sort descriptors into directory structure.
Commit: e5289f22eebaf892de58db46442af64b6e58957d
---
config | 3 +
src/ArchiveWriter.java | 89 ++++++++++++++++++++++++++++++++++
src/CachedRelayDescriptorReader.java | 87 +++++++++++++++++++++++++++++++++
src/Configuration.java | 7 +++
src/Main.java | 11 ++++-
src/RelayDescriptorDownloader.java | 3 +-
6 files changed, 197 insertions(+), 3 deletions(-)
create mode 100644 src/ArchiveWriter.java
create mode 100644 src/CachedRelayDescriptorReader.java
diff --git a/config b/config
index a54d71c..e783344 100644
--- a/config
+++ b/config
@@ -39,6 +39,9 @@
## Write directory archives to disk
#WriteDirectoryArchives 1
+## Read cached-* files from a local Tor client
+#ImportCachedRelayDescriptors 1
+
## Import directory archives from disk, if available
#ImportDirectoryArchives 1
diff --git a/src/ArchiveWriter.java b/src/ArchiveWriter.java
new file mode 100644
index 0000000..2ebe423
--- /dev/null
+++ b/src/ArchiveWriter.java
@@ -0,0 +1,89 @@
+import java.io.*;
+import java.text.*;
+import java.util.*;
+import org.apache.commons.compress.archivers.tar.*;
+
+public class ArchiveWriter {
+ public ArchiveWriter() {
+ }
+ public void storeConsensus(BufferedReader br, String validAfterTime)
+ throws IOException, ParseException {
+ SimpleDateFormat parseFormat =
+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ parseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ long validAfter = parseFormat.parse(validAfterTime).getTime();
+ SimpleDateFormat printFormat =
+ new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
+ printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ File consensusFile = new File("directory-archive/consensus/"
+ + printFormat.format(new Date(validAfter)) + "-consensus");
+ consensusFile.getParentFile().mkdirs();
+ if (!consensusFile.exists()) {
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ consensusFile));
+ String line = null;
+ while ((line = br.readLine()) != null) {
+ bw.write(line + "\n");
+ if (line.startsWith("r ")) {
+ // TODO compile list of server descriptors that we might want to
+ // learn about
+ }
+ }
+ bw.close();
+ }
+ }
+ public void storeVote(BufferedReader br, String validAfterTime,
+ String authorityFingerprint) throws IOException {
+ // TODO implement me
+ }
+ public void storeServerDescriptor(BufferedReader br, String digest,
+ String publishedTime, String extraInfoDigest) throws IOException,
+ ParseException {
+ SimpleDateFormat parseFormat =
+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ parseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ long published = parseFormat.parse(publishedTime).getTime();
+ SimpleDateFormat printFormat = new SimpleDateFormat("yyyy/MM/");
+ printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ File descriptorFile = new File("directory-archive/server-descriptor/"
+ + printFormat.format(new Date(published))
+ + digest.substring(0, 1) + "/" + digest.substring(1, 2) + "/"
+ + digest);
+ descriptorFile.getParentFile().mkdirs();
+ if (!descriptorFile.exists()) {
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ descriptorFile));
+ String line = null;
+ while ((line = br.readLine()) != null) {
+ bw.write(line + "\n");
+ }
+ bw.close();
+ }
+ // TODO if extraInfoDigest != null, add digest to extra-info
+ // descriptors we want to download
+ }
+ public void storeExtraInfo(BufferedReader br, String digest,
+ String publishedTime) throws IOException, ParseException {
+ SimpleDateFormat parseFormat =
+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ parseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ long published = parseFormat.parse(publishedTime).getTime();
+ SimpleDateFormat printFormat = new SimpleDateFormat("yyyy/MM/");
+ printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ File descriptorFile = new File("directory-archive/extra-info/"
+ + printFormat.format(new Date(published))
+ + digest.substring(0, 1) + "/" + digest.substring(1, 2) + "/"
+ + digest);
+ descriptorFile.getParentFile().mkdirs();
+ if (!descriptorFile.exists()) {
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ descriptorFile));
+ String line = null, extraInfoDigest = null;
+ while ((line = br.readLine()) != null) {
+ bw.write(line + "\n");
+ }
+ bw.close();
+ }
+ }
+}
+
diff --git a/src/CachedRelayDescriptorReader.java b/src/CachedRelayDescriptorReader.java
new file mode 100644
index 0000000..8a7ed5f
--- /dev/null
+++ b/src/CachedRelayDescriptorReader.java
@@ -0,0 +1,87 @@
+import java.io.*;
+import java.text.*;
+import org.apache.commons.codec.digest.*;
+
+/**
+ * Parses all descriptors in local directory cacheddesc/ and sorts them
+ * into directory structure in directory-archive/.
+ */
+public class CachedRelayDescriptorReader {
+ public CachedRelayDescriptorReader(RelayDescriptorParser rdp,
+ ArchiveWriter aw) {
+ File cachedDescDir = new File("cacheddesc");
+ if (cachedDescDir.exists()) {
+ try {
+ rdp.initialize();
+ } catch (IOException e) {
+ return;
+ }
+ for (File f : cachedDescDir.listFiles()) {
+ try {
+ if (f.getName().equals("cached-consensus") ||
+ f.getName().startsWith("cached-descriptors") ||
+ f.getName().startsWith("cached-extrainfo")) {
+ BufferedReader br = new BufferedReader(new FileReader(f));
+ String line = null, validAfterTime = null, publishedTime = null,
+ extraInfoDigest = null, digest = null;
+ StringBuilder sb = new StringBuilder();
+ while ((line = br.readLine()) != null || sb != null) {
+ if (line == null && sb.length() < 1) {
+ continue; // empty file?
+ }
+ if (line == null || line.startsWith("router ") ||
+ line.startsWith("extra-info ")) {
+ if (sb.length() > 0) {
+ BufferedReader storeBr = new BufferedReader(
+ new StringReader(sb.toString()));
+ if (f.getName().equals("cached-consensus")) {
+ aw.storeConsensus(storeBr, validAfterTime);
+ validAfterTime = null;
+ } else if (f.getName().startsWith("cached-descriptors")) {
+ aw.storeServerDescriptor(storeBr, digest,
+ publishedTime, extraInfoDigest);
+ digest = null;
+ publishedTime = null;
+ extraInfoDigest = null;
+ } else if (f.getName().startsWith("cached-extrainfo")) {
+ aw.storeExtraInfo(storeBr, digest, publishedTime);
+ digest = null;
+ publishedTime = null;
+ }
+ storeBr.close();
+ }
+ if (line == null) {
+ sb = null;
+ break;
+ } else {
+ sb = new StringBuilder();
+ }
+ }
+ if (line.startsWith("valid-after ")) {
+ validAfterTime = line.substring("valid-after ".length());
+ } else if (line.startsWith("published ")) {
+ publishedTime = line.substring("published ".length());
+ } else if (line.startsWith("router-signature")) {
+ digest = DigestUtils.shaHex(sb.toString()
+ + "router-signature\n").toUpperCase();
+ } else if (line.startsWith("opt extra-info-digest ")) {
+ extraInfoDigest = line.split(" ")[2];
+ } else if (line.startsWith("extra-info-digest ")) {
+ extraInfoDigest = line.split(" ")[1];
+ }
+ if (!line.startsWith("@")) {
+ sb.append(line + "\n");
+ }
+ }
+ br.close();
+ }
+ } catch (IOException e) {
+ // TODO handle
+ } catch (ParseException e) {
+ // TODO handle
+ }
+ }
+ }
+ }
+}
+
diff --git a/src/Configuration.java b/src/Configuration.java
index 6bc95fe..6e96636 100644
--- a/src/Configuration.java
+++ b/src/Configuration.java
@@ -18,6 +18,7 @@ public class Configuration {
+ "9695DFC35FFEB861329B9F1AB04C46397020CE31"));
private boolean writeBridgeStats = true;
private boolean writeDirectoryArchives = true;
+ private boolean importCachedRelayDescriptors = true;
private boolean importDirectoryArchives = true;
private boolean importSanitizedBridges = true;
private boolean importBridgeSnapshots = true;
@@ -65,6 +66,9 @@ public class Configuration {
} else if (line.startsWith("WriteDirectoryArchives")) {
this.writeDirectoryArchives = Integer.parseInt(
line.split(" ")[1]) != 0;
+ } else if (line.startsWith("ImportCachedRelayDescriptors")) {
+ this.importCachedRelayDescriptors = Integer.parseInt(
+ line.split(" ")[1]) != 0;
} else if (line.startsWith("ImportDirectoryArchives")) {
this.importDirectoryArchives = Integer.parseInt(
line.split(" ")[1]) != 0;
@@ -147,6 +151,9 @@ public class Configuration {
public boolean getWriteDirectoryArchives() {
return this.writeDirectoryArchives;
}
+ public boolean getImportCachedRelayDescriptors() {
+ return this.importCachedRelayDescriptors;
+ }
public boolean getImportDirectoryArchives() {
return this.importDirectoryArchives;
}
diff --git a/src/Main.java b/src/Main.java
index e4d2f35..98d4658 100644
--- a/src/Main.java
+++ b/src/Main.java
@@ -45,18 +45,25 @@ public class Main {
BridgeDescriptorParser bdp = new BridgeDescriptorParser(csfh, bsfh,
countries);
- // TODO WriteDirectoryArchives
+ // Prepare writing relay descriptor archive to disk
+ ArchiveWriter aw = config.getWriteDirectoryArchives() ?
+ new ArchiveWriter() : null;
+ // TODO handle case aw==NULL below
// import and/or download relay and bridge descriptors
+ if (config.getImportCachedRelayDescriptors()) {
+ new CachedRelayDescriptorReader(rdp, aw);
+ }
if (config.getImportDirectoryArchives()) {
new ArchiveReader(rdp, "archives");
}
if (config.getDownloadRelayDescriptors()) {
// TODO make this smarter by letting rdd ask rdp which descriptors
// are still missing and only download those
+ // TODO move iteration over dirauths from main() to class code
for (String directoryAuthority :
config.getDownloadFromDirectoryAuthorities()) {
- new RelayDescriptorDownloader(rdp, directoryAuthority,
+ new RelayDescriptorDownloader(rdp, aw, directoryAuthority,
directories);
}
}
diff --git a/src/RelayDescriptorDownloader.java b/src/RelayDescriptorDownloader.java
index 24c040a..7f953be 100644
--- a/src/RelayDescriptorDownloader.java
+++ b/src/RelayDescriptorDownloader.java
@@ -9,9 +9,10 @@ import java.util.logging.*;
*/
public class RelayDescriptorDownloader {
public RelayDescriptorDownloader(RelayDescriptorParser rdp,
- String authority, SortedSet<String> directories) {
+ ArchiveWriter aw, String authority, SortedSet<String> directories) {
Logger logger =
Logger.getLogger(RelayDescriptorDownloader.class.getName());
+ // TODO use aw
try {
rdp.initialize();
} catch (IOException e) {
--
1.6.5
More information about the tor-commits
mailing list