[tor-commits] [metrics-web/master] Use metrics-lib for parsing relay descriptors.
karsten at torproject.org
karsten at torproject.org
Thu Mar 22 11:48:14 UTC 2012
commit 72c1aee10e13d199a2e604adb47c670c417ab0a0
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Wed Mar 21 14:44:30 2012 +0100
Use metrics-lib for parsing relay descriptors.
---
config.template | 2 +-
src/org/torproject/ernie/cron/ArchiveReader.java | 573 +++++---------------
src/org/torproject/ernie/cron/Configuration.java | 2 +-
.../cron/RelayDescriptorDatabaseImporter.java | 32 +-
4 files changed, 155 insertions(+), 454 deletions(-)
diff --git a/config.template b/config.template
index 8c56909..ce2dc94 100644
--- a/config.template
+++ b/config.template
@@ -2,7 +2,7 @@
#ImportDirectoryArchives 0
#
## Relative path to directory to import directory archives from
-#DirectoryArchivesDirectory archives/
+#DirectoryArchivesDirectory in/relay-descriptors/
#
## Keep a history of imported directory archive files to know which files
## have been imported before. This history can be useful when importing
diff --git a/src/org/torproject/ernie/cron/ArchiveReader.java b/src/org/torproject/ernie/cron/ArchiveReader.java
index ed1e505..5703661 100644
--- a/src/org/torproject/ernie/cron/ArchiveReader.java
+++ b/src/org/torproject/ernie/cron/ArchiveReader.java
@@ -2,34 +2,29 @@
* See LICENSE for licensing information */
package org.torproject.ernie.cron;
-import java.io.BufferedInputStream;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.ByteArrayOutputStream;
import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.StringReader;
-import java.text.ParseException;
+import java.io.UnsupportedEncodingException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.Stack;
import java.util.TimeZone;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.logging.Level;
import java.util.logging.Logger;
-import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.codec.DecoderException;
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.codec.digest.DigestUtils;
+import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.DescriptorFile;
+import org.torproject.descriptor.DescriptorReader;
+import org.torproject.descriptor.DescriptorSourceFactory;
+import org.torproject.descriptor.ExtraInfoDescriptor;
+import org.torproject.descriptor.NetworkStatusEntry;
+import org.torproject.descriptor.RelayNetworkStatusConsensus;
+import org.torproject.descriptor.RelayNetworkStatusVote;
+import org.torproject.descriptor.ServerDescriptor;
/**
* Read in all files in a given directory and pass buffered readers of
@@ -72,439 +67,159 @@ public class ArchiveReader {
int parsedFiles = 0, ignoredFiles = 0;
this.logger = Logger.getLogger(ArchiveReader.class.getName());
- SortedMap<String, Long>
- lastArchivesImportHistory = new TreeMap<String, Long>(),
- newArchivesImportHistory = new TreeMap<String, Long>();
- File archivesImportHistoryFile = new File(statsDirectory,
- "archives-import-history");
- if (keepImportHistory && archivesImportHistoryFile.exists()) {
- try {
- BufferedReader br = new BufferedReader(new FileReader(
- archivesImportHistoryFile));
- String line = null;
- while ((line = br.readLine()) != null) {
- String[] parts = line.split(",");
- if (parts.length < 2) {
- logger.warning("Archives import history file does not "
- + "contain timestamps. Skipping.");
- break;
- }
- long lastModified = Long.parseLong(parts[0]);
- String filename = parts[1];
- lastArchivesImportHistory.put(filename, lastModified);
- }
- br.close();
- } catch (IOException e) {
- logger.log(Level.WARNING, "Could not read in archives import "
- + "history file. Skipping.");
- }
- }
if (archivesDirectory.exists()) {
logger.fine("Importing files in directory " + archivesDirectory
+ "/...");
- Stack<File> filesInInputDir = new Stack<File>();
- filesInInputDir.add(archivesDirectory);
- List<File> problems = new ArrayList<File>();
- while (!filesInInputDir.isEmpty()) {
- File pop = filesInInputDir.pop();
- if (pop.isDirectory()) {
- for (File f : pop.listFiles()) {
- filesInInputDir.add(f);
- }
- } else {
- try {
- long lastModified = pop.lastModified();
- String filename = pop.getName();
- if (keepImportHistory) {
- newArchivesImportHistory.put(filename, lastModified);
- }
- if (keepImportHistory &&
- lastArchivesImportHistory.containsKey(filename) &&
- lastArchivesImportHistory.get(filename) >= lastModified) {
- ignoredFiles++;
- continue;
- } else if (filename.endsWith(".tar.bz2")) {
- logger.warning("Cannot parse compressed tarball "
- + pop.getAbsolutePath() + ". Skipping.");
- continue;
- }
- FileInputStream fis = new FileInputStream(pop);
- BufferedInputStream bis = new BufferedInputStream(fis);
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- int len;
- byte[] data = new byte[1024];
- while ((len = bis.read(data, 0, 1024)) >= 0) {
- baos.write(data, 0, len);
- }
- bis.close();
- byte[] allData = baos.toByteArray();
- this.parse(allData);
- parsedFiles++;
- } catch (IOException e) {
- problems.add(pop);
- if (problems.size() > 3) {
- break;
- }
- }
- }
+ DescriptorReader reader =
+ DescriptorSourceFactory.createDescriptorReader();
+ reader.addDirectory(archivesDirectory);
+ if (keepImportHistory) {
+ reader.setExcludeFiles(new File(statsDirectory,
+ "relay-descriptor-history"));
}
- if (problems.isEmpty()) {
- logger.fine("Finished importing files in directory "
- + archivesDirectory + "/.");
- } else {
- StringBuilder sb = new StringBuilder("Failed importing files in "
- + "directory " + archivesDirectory + "/:");
- int printed = 0;
- for (File f : problems) {
- sb.append("\n " + f.getAbsolutePath());
- if (++printed >= 3) {
- sb.append("\n ... more");
- break;
+ Iterator<DescriptorFile> descriptorFiles = reader.readDescriptors();
+ while (descriptorFiles.hasNext()) {
+ DescriptorFile descriptorFile = descriptorFiles.next();
+ if (descriptorFile.getDescriptors() != null) {
+ for (Descriptor descriptor : descriptorFile.getDescriptors()) {
+ if (descriptor instanceof RelayNetworkStatusConsensus) {
+ this.addRelayNetworkStatusConsensus(
+ (RelayNetworkStatusConsensus) descriptor);
+ } else if (descriptor instanceof RelayNetworkStatusVote) {
+ this.addRelayNetworkStatusVote(
+ (RelayNetworkStatusVote) descriptor);
+ } else if (descriptor instanceof ServerDescriptor) {
+ this.addServerDescriptor((ServerDescriptor) descriptor);
+ } else if (descriptor instanceof ExtraInfoDescriptor) {
+ this.addExtraInfoDescriptor(
+ (ExtraInfoDescriptor) descriptor);
+ }
}
}
- logger.warning(sb.toString());
- }
- }
- if (keepImportHistory) {
- try {
- archivesImportHistoryFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- archivesImportHistoryFile));
- for (Map.Entry<String, Long> historyEntry :
- newArchivesImportHistory.entrySet()) {
- bw.write(String.valueOf(historyEntry.getValue()) + ","
- + historyEntry.getKey() + "\n");
- }
- bw.close();
- } catch (IOException e) {
- logger.log(Level.WARNING, "Could not write archives import "
- + "history file.");
}
}
+
logger.info("Finished importing relay descriptors from local "
+ "directory:\nParsed " + parsedFiles + ", ignored "
+ ignoredFiles + " files.");
}
- public void parse(byte[] data) {
- try {
- /* Remove any @ lines at the beginning of the file and parse the
- * first non-@ line to find out the descriptor type. */
- BufferedReader br = new BufferedReader(new StringReader(new String(
- data, "US-ASCII")));
- String line = br.readLine();
- while (line != null && line.startsWith("@")) {
- line = br.readLine();
- }
- if (line == null) {
- this.logger.fine("We were given a file that doesn't contain a "
- + "single descriptor for parsing. Ignoring.");
- return;
+ private void addRelayNetworkStatusConsensus(
+ RelayNetworkStatusConsensus consensus) {
+ for (NetworkStatusEntry statusEntry :
+ consensus.getStatusEntries().values()) {
+ this.rddi.addStatusEntry(consensus.getValidAfterMillis(),
+ statusEntry.getNickname(), statusEntry.getFingerprint(),
+ statusEntry.getDescriptor(), statusEntry.getPublishedMillis(),
+ statusEntry.getAddress(), statusEntry.getOrPort(),
+ statusEntry.getDirPort(), statusEntry.getFlags(),
+ statusEntry.getVersion(), statusEntry.getBandwidth(),
+ statusEntry.getPortList(), statusEntry.getStatusEntryBytes());
+ try {
+ this.bsfh.addHashedRelay(DigestUtils.shaHex(Hex.decodeHex(
+ statusEntry.getFingerprint().toCharArray())).toUpperCase());
+ } catch (DecoderException e) {
}
- br.close();
+ }
+ this.rddi.addConsensus(consensus.getValidAfterMillis(),
+ consensus.getRawDescriptorBytes());
+ }
- /* Split the byte[] possibly containing multiple descriptors into
- * byte[]'s with one descriptor each and parse them. */
- String startToken = null;
- if (line.equals("network-status-version 3")) {
- startToken = "network-status-version 3";
- } else if (line.startsWith("router ")) {
- startToken = "router ";
- } else if (line.startsWith("extra-info ")) {
- startToken = "extra-info ";
- } else if (line.equals("dir-key-certificate-version 3")) {
- this.logger.fine("Not parsing dir key certificate.");
- return;
- } else {
- this.logger.warning("Unknown descriptor type. First line is '"
- + line + "'. Ignoring.");
- return;
- }
- String splitToken = "\n" + startToken;
- String ascii = new String(data, "US-ASCII");
- int length = data.length, start = ascii.indexOf(startToken);
- while (start < length) {
- int end = ascii.indexOf(splitToken, start);
- if (end < 0) {
- end = length;
- } else {
- end += 1;
- }
- byte[] descBytes = new byte[end - start];
- System.arraycopy(data, start, descBytes, 0, end - start);
- parseSingleDescriptor(descBytes);
- start = end;
+ private void addRelayNetworkStatusVote(RelayNetworkStatusVote vote) {
+ this.rddi.addVote(vote.getValidAfterMillis(), vote.getIdentity(),
+ vote.getRawDescriptorBytes());
+ }
+
+ private void addServerDescriptor(ServerDescriptor descriptor) {
+ String digest = null;
+ try {
+ String ascii = new String(descriptor.getRawDescriptorBytes(),
+ "US-ASCII");
+ String startToken = "router ";
+ String sigToken = "\nrouter-signature\n";
+ int start = ascii.indexOf(startToken);
+ int sig = ascii.indexOf(sigToken) + sigToken.length();
+ if (start >= 0 || sig >= 0 || sig > start) {
+ byte[] forDigest = new byte[sig - start];
+ System.arraycopy(descriptor.getRawDescriptorBytes(), start,
+ forDigest, 0, sig - start);
+ digest = DigestUtils.shaHex(forDigest);
}
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Could not parse descriptor. "
- + "Skipping.", e);
+ } catch (UnsupportedEncodingException e) {
+ }
+ if (digest != null) {
+ this.rddi.addServerDescriptor(digest, descriptor.getNickname(),
+ descriptor.getAddress(), descriptor.getOrPort(),
+ descriptor.getDirPort(), descriptor.getFingerprint(),
+ descriptor.getBandwidthRate(), descriptor.getBandwidthBurst(),
+ descriptor.getBandwidthObserved(), descriptor.getPlatform(),
+ descriptor.getPublishedMillis(), descriptor.getUptime(),
+ descriptor.getExtraInfoDigest(),
+ descriptor.getRawDescriptorBytes());
}
}
- private void parseSingleDescriptor(byte[] data) {
+ private void addExtraInfoDescriptor(ExtraInfoDescriptor descriptor) {
+ if (descriptor.getDirreqV3Reqs() != null) {
+ int allUsers = 0;
+ Map<String, String> obs = new HashMap<String, String>();
+ for (Map.Entry<String, Integer> e :
+ descriptor.getDirreqV3Reqs().entrySet()) {
+ String country = e.getKey();
+ int users = e.getValue() - 4;
+ allUsers += users;
+ obs.put(country, "" + users);
+ }
+ obs.put("zy", "" + allUsers);
+ this.rddi.addDirReqStats(descriptor.getFingerprint(),
+ descriptor.getDirreqStatsEndMillis(),
+ descriptor.getDirreqStatsIntervalLength(), obs);
+ }
+ if (descriptor.getConnBiDirectStatsEndMillis() >= 0L) {
+ this.rddi.addConnBiDirect(descriptor.getFingerprint(),
+ descriptor.getConnBiDirectStatsEndMillis(),
+ descriptor.getConnBiDirectStatsIntervalLength(),
+ descriptor.getConnBiDirectBelow(),
+ descriptor.getConnBiDirectRead(),
+ descriptor.getConnBiDirectWrite(),
+ descriptor.getConnBiDirectBoth());
+ }
+ List<String> bandwidthHistoryLines = new ArrayList<String>();
+ if (descriptor.getWriteHistory() != null) {
+ bandwidthHistoryLines.add(descriptor.getWriteHistory().getLine());
+ }
+ if (descriptor.getReadHistory() != null) {
+ bandwidthHistoryLines.add(descriptor.getReadHistory().getLine());
+ }
+ if (descriptor.getDirreqWriteHistory() != null) {
+ bandwidthHistoryLines.add(
+ descriptor.getDirreqWriteHistory().getLine());
+ }
+ if (descriptor.getDirreqReadHistory() != null) {
+ bandwidthHistoryLines.add(
+ descriptor.getDirreqReadHistory().getLine());
+ }
+ String digest = null;
try {
- /* Convert descriptor to ASCII for parsing. This means we'll lose
- * the non-ASCII chars, but we don't care about them for parsing
- * anyway. */
- BufferedReader br = new BufferedReader(new StringReader(new String(
- data, "US-ASCII")));
- String line = br.readLine();
- SimpleDateFormat parseFormat =
- new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- parseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- if (line.equals("network-status-version 3")) {
- // TODO when parsing the current consensus, check the fresh-until
- // time to see when we switch from hourly to half-hourly
- // consensuses
- boolean isConsensus = true;
- String validAfterTime = null, nickname = null,
- relayIdentity = null, serverDesc = null, version = null,
- ports = null;
- String dirSource = null, address = null;
- long validAfter = -1L, published = -1L, bandwidth = -1L,
- orPort = 0L, dirPort = 0L;
- SortedSet<String> relayFlags = null;
- StringBuilder rawStatusEntry = null;
- SortedSet<String> hashedRelayIdentities = new TreeSet<String>();
- while ((line = br.readLine()) != null) {
- if (line.equals("vote-status vote")) {
- isConsensus = false;
- } else if (line.startsWith("valid-after ")) {
- validAfterTime = line.substring("valid-after ".length());
- validAfter = parseFormat.parse(validAfterTime).getTime();
- } else if (line.startsWith("dir-source ")) {
- dirSource = line.split(" ")[2];
- } else if (line.startsWith("r ")) {
- if (isConsensus && relayIdentity != null &&
- this.rddi != null) {
- byte[] rawDescriptor = rawStatusEntry.toString().getBytes();
- this.rddi.addStatusEntry(validAfter, nickname,
- relayIdentity, serverDesc, published, address, orPort,
- dirPort, relayFlags, version, bandwidth, ports,
- rawDescriptor);
- relayFlags = null;
- version = null;
- bandwidth = -1L;
- ports = null;
- }
- rawStatusEntry = new StringBuilder(line + "\n");
- String[] parts = line.split(" ");
- if (parts.length < 9) {
- this.logger.log(Level.WARNING, "Could not parse r line '"
- + line + "' in descriptor. Skipping.");
- break;
- }
- nickname = parts[1];
- relayIdentity = Hex.encodeHexString(
- Base64.decodeBase64(parts[2] + "=")).
- toLowerCase();
- hashedRelayIdentities.add(DigestUtils.shaHex(
- Base64.decodeBase64(parts[2] + "=")).
- toUpperCase());
- serverDesc = Hex.encodeHexString(Base64.decodeBase64(
- parts[3] + "=")).toLowerCase();
- published = parseFormat.parse(parts[4] + " " + parts[5]).
- getTime();
- address = parts[6];
- orPort = Long.parseLong(parts[7]);
- dirPort = Long.parseLong(parts[8]);
- } else if (line.startsWith("s ") || line.equals("s")) {
- rawStatusEntry.append(line + "\n");
- relayFlags = new TreeSet<String>();
- if (line.length() > 2) {
- for (String flag : line.substring(2).split(" ")) {
- relayFlags.add(flag);
- }
- }
- } else if (line.startsWith("v ")) {
- rawStatusEntry.append(line + "\n");
- version = line.substring(2);
- } else if (line.startsWith("w ")) {
- rawStatusEntry.append(line + "\n");
- String[] parts = line.split(" ");
- for (String part : parts) {
- if (part.startsWith("Bandwidth=")) {
- bandwidth = Long.parseLong(part.substring(
- "Bandwidth=".length()));
- }
- }
- } else if (line.startsWith("p ")) {
- rawStatusEntry.append(line + "\n");
- ports = line.substring(2);
- }
- }
- if (isConsensus) {
- if (this.bsfh != null) {
- for (String hashedRelayIdentity : hashedRelayIdentities) {
- this.bsfh.addHashedRelay(hashedRelayIdentity);
- }
- }
- if (this.rddi != null) {
- this.rddi.addConsensus(validAfter, data);
- if (relayIdentity != null) {
- byte[] rawDescriptor = rawStatusEntry.toString().getBytes();
- this.rddi.addStatusEntry(validAfter, nickname,
- relayIdentity, serverDesc, published, address, orPort,
- dirPort, relayFlags, version, bandwidth, ports,
- rawDescriptor);
- }
- }
- } else {
- if (this.rddi != null) {
- this.rddi.addVote(validAfter, dirSource, data);
- }
- }
- } else if (line.startsWith("router ")) {
- String platformLine = null, bandwidthLine = null,
- extraInfoDigest = null, relayIdentifier = null;
- String[] parts = line.split(" ");
- String nickname = parts[1];
- String address = parts[2];
- int orPort = Integer.parseInt(parts[3]);
- int dirPort = Integer.parseInt(parts[4]);
- long published = -1L, uptime = -1L;
- while ((line = br.readLine()) != null) {
- if (line.startsWith("platform ")) {
- platformLine = line;
- } else if (line.startsWith("published ")) {
- String publishedTime = line.substring("published ".length());
- published = parseFormat.parse(publishedTime).getTime();
- } else if (line.startsWith("opt fingerprint") ||
- line.startsWith("fingerprint")) {
- relayIdentifier = line.substring(line.startsWith("opt ") ?
- "opt fingerprint".length() : "fingerprint".length()).
- replaceAll(" ", "").toLowerCase();
- } else if (line.startsWith("bandwidth ")) {
- bandwidthLine = line;
- } else if (line.startsWith("opt extra-info-digest ") ||
- line.startsWith("extra-info-digest ")) {
- extraInfoDigest = line.startsWith("opt ") ?
- line.split(" ")[2].toLowerCase() :
- line.split(" ")[1].toLowerCase();
- } else if (line.startsWith("uptime ")) {
- uptime = Long.parseLong(line.substring("uptime ".length()));
- }
- }
- String ascii = new String(data, "US-ASCII");
- String startToken = "router ";
- String sigToken = "\nrouter-signature\n";
- int start = ascii.indexOf(startToken);
- int sig = ascii.indexOf(sigToken) + sigToken.length();
- String digest = null;
- if (start >= 0 || sig >= 0 || sig > start) {
- byte[] forDigest = new byte[sig - start];
- System.arraycopy(data, start, forDigest, 0, sig - start);
- digest = DigestUtils.shaHex(forDigest);
- }
- if (this.rddi != null && digest != null) {
- String[] bwParts = bandwidthLine.split(" ");
- long bandwidthAvg = Long.parseLong(bwParts[1]);
- long bandwidthBurst = Long.parseLong(bwParts[2]);
- long bandwidthObserved = Long.parseLong(bwParts[3]);
- String platform = platformLine.substring("platform ".length());
- this.rddi.addServerDescriptor(digest, nickname, address, orPort,
- dirPort, relayIdentifier, bandwidthAvg, bandwidthBurst,
- bandwidthObserved, platform, published, uptime,
- extraInfoDigest, data);
- }
- } else if (line.startsWith("extra-info ")) {
- String nickname = line.split(" ")[1];
- long published = -1L;
- String dir = line.split(" ")[2];
- String statsEnd = null;
- long seconds = -1L;
- List<String> bandwidthHistory = new ArrayList<String>();
- while ((line = br.readLine()) != null) {
- if (line.startsWith("published ")) {
- String publishedTime = line.substring("published ".length());
- published = parseFormat.parse(publishedTime).getTime();
- } else if (line.startsWith("read-history ") ||
- line.startsWith("write-history ") ||
- line.startsWith("dirreq-read-history ") ||
- line.startsWith("dirreq-write-history ")) {
- bandwidthHistory.add(line);
- } else if (line.startsWith("dirreq-stats-end ")) {
- String[] parts = line.split(" ");
- if (parts.length < 5) {
- this.logger.warning("Could not parse dirreq-stats-end "
- + "line '" + line + "' in descriptor. Skipping.");
- break;
- }
- statsEnd = parts[1] + " " + parts[2];
- seconds = Long.parseLong(parts[3].substring(1));
- } else if (line.startsWith("dirreq-v3-reqs ")
- && line.length() > "dirreq-v3-reqs ".length()) {
- if (this.rddi != null) {
- try {
- int allUsers = 0;
- Map<String, String> obs = new HashMap<String, String>();
- String[] parts = line.substring("dirreq-v3-reqs ".
- length()).split(",");
- for (String p : parts) {
- String country = p.substring(0, 2);
- int users = Integer.parseInt(p.substring(3)) - 4;
- allUsers += users;
- obs.put(country, "" + users);
- }
- obs.put("zy", "" + allUsers);
- this.rddi.addDirReqStats(dir, statsEnd, seconds, obs);
- } catch (NumberFormatException e) {
- this.logger.log(Level.WARNING, "Could not parse "
- + "dirreq-v3-reqs line '" + line + "' in descriptor. "
- + "Skipping.", e);
- break;
- }
- }
- } else if (line.startsWith("conn-bi-direct ")) {
- if (this.rddi != null) {
- String[] parts = line.split(" ");
- if (parts.length == 6 &&
- parts[5].split(",").length == 4) {
- try {
- String connBiDirectStatsEnd = parts[1] + " " + parts[2];
- long connBiDirectSeconds = Long.parseLong(parts[3].
- substring(1));
- String[] parts2 = parts[5].split(",");
- long below = Long.parseLong(parts2[0]);
- long read = Long.parseLong(parts2[1]);
- long write = Long.parseLong(parts2[2]);
- long both = Long.parseLong(parts2[3]);
- this.rddi.addConnBiDirect(dir, connBiDirectStatsEnd,
- connBiDirectSeconds, below, read, write, both);
- } catch (NumberFormatException e) {
- this.logger.log(Level.WARNING, "Number format "
- + "exception while parsing conn-bi-direct stats "
- + "string '" + line + "'. Skipping.", e);
- }
- } else {
- this.logger.warning("Skipping invalid conn-bi-direct "
- + "stats string '" + line + "'.");
- }
- }
- }
- }
- String ascii = new String(data, "US-ASCII");
- String startToken = "extra-info ";
- String sigToken = "\nrouter-signature\n";
- String digest = null;
- int start = ascii.indexOf(startToken);
- int sig = ascii.indexOf(sigToken) + sigToken.length();
- if (start >= 0 || sig >= 0 || sig > start) {
- byte[] forDigest = new byte[sig - start];
- System.arraycopy(data, start, forDigest, 0, sig - start);
- digest = DigestUtils.shaHex(forDigest);
- }
- if (this.rddi != null && digest != null) {
- this.rddi.addExtraInfoDescriptor(digest, nickname,
- dir.toLowerCase(), published, data, bandwidthHistory);
- }
+ String ascii = new String(descriptor.getRawDescriptorBytes(),
+ "US-ASCII");
+ String startToken = "extra-info ";
+ String sigToken = "\nrouter-signature\n";
+ int start = ascii.indexOf(startToken);
+ int sig = ascii.indexOf(sigToken) + sigToken.length();
+ if (start >= 0 || sig >= 0 || sig > start) {
+ byte[] forDigest = new byte[sig - start];
+ System.arraycopy(descriptor.getRawDescriptorBytes(), start,
+ forDigest, 0, sig - start);
+ digest = DigestUtils.shaHex(forDigest);
}
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Could not parse descriptor. "
- + "Skipping.", e);
- } catch (ParseException e) {
- this.logger.log(Level.WARNING, "Could not parse descriptor. "
- + "Skipping.", e);
+ } catch (UnsupportedEncodingException e) {
+ }
+ if (digest != null) {
+ this.rddi.addExtraInfoDescriptor(digest, descriptor.getNickname(),
+ descriptor.getFingerprint().toLowerCase(),
+ descriptor.getPublishedMillis(),
+ descriptor.getRawDescriptorBytes(), bandwidthHistoryLines);
}
}
}
diff --git a/src/org/torproject/ernie/cron/Configuration.java b/src/org/torproject/ernie/cron/Configuration.java
index 0a493cb..1f30eb9 100644
--- a/src/org/torproject/ernie/cron/Configuration.java
+++ b/src/org/torproject/ernie/cron/Configuration.java
@@ -17,7 +17,7 @@ import java.util.logging.Logger;
*/
public class Configuration {
private boolean importDirectoryArchives = false;
- private String directoryArchivesDirectory = "archives/";
+ private String directoryArchivesDirectory = "in/relay-descriptors/";
private boolean keepDirectoryArchiveImportHistory = false;
private boolean importSanitizedBridges = false;
private String sanitizedBridgesDirectory = "bridges/";
diff --git a/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java b/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java
index 35ba90e..3ee7b6b 100644
--- a/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java
+++ b/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java
@@ -1017,21 +1017,14 @@ public final class RelayDescriptorDatabaseImporter {
/**
* Insert a conn-bi-direct stats string into the database.
*/
- public void addConnBiDirect(String source, String statsEnd,
+ public void addConnBiDirect(String source, long statsEndMillis,
long seconds, long below, long read, long write, long both) {
- long statsEndTime = 0L;
- try {
- statsEndTime = this.dateTimeFormat.parse(statsEnd).getTime();
- } catch (ParseException e) {
- this.logger.log(Level.WARNING, "Could not add conn-bi-direct "
- + "stats string with interval ending '" + statsEnd + "'.", e);
- return;
- }
+ String statsEnd = this.dateTimeFormat.format(statsEndMillis);
if (this.importIntoDatabase) {
try {
- this.addDateToScheduledUpdates(statsEndTime);
+ this.addDateToScheduledUpdates(statsEndMillis);
Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
- Timestamp statsEndTimestamp = new Timestamp(statsEndTime);
+ Timestamp statsEndTimestamp = new Timestamp(statsEndMillis);
this.psBs.setString(1, source);
this.psBs.setTimestamp(2, statsEndTimestamp, cal);
ResultSet rs = psBs.executeQuery();
@@ -1085,21 +1078,14 @@ public final class RelayDescriptorDatabaseImporter {
* Adds observations on the number of directory requests by country as
* seen on a directory at a given date to the database.
*/
- public void addDirReqStats(String source, String statsEnd, long seconds,
- Map<String, String> dirReqsPerCountry) {
- long statsEndTime = 0L;
- try {
- statsEndTime = this.dateTimeFormat.parse(statsEnd).getTime();
- } catch (ParseException e) {
- this.logger.log(Level.WARNING, "Could not add dirreq stats with "
- + "interval ending '" + statsEnd + "'.", e);
- return;
- }
+ public void addDirReqStats(String source, long statsEndMillis,
+ long seconds, Map<String, String> dirReqsPerCountry) {
+ String statsEnd = this.dateTimeFormat.format(statsEndMillis);
if (this.importIntoDatabase) {
try {
- this.addDateToScheduledUpdates(statsEndTime);
+ this.addDateToScheduledUpdates(statsEndMillis);
Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
- Timestamp statsEndTimestamp = new Timestamp(statsEndTime);
+ Timestamp statsEndTimestamp = new Timestamp(statsEndMillis);
this.psQs.setString(1, source);
this.psQs.setTimestamp(2, statsEndTimestamp, cal);
ResultSet rs = psQs.executeQuery();
More information about the tor-commits
mailing list