[tor-commits] [metrics-web/master] Remove relay descriptor parser.
karsten at torproject.org
karsten at torproject.org
Thu Mar 22 11:48:14 UTC 2012
commit a2b00950c8b4991e20e3026c04f8b5b49ce530d3
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Wed Mar 21 12:20:01 2012 +0100
Remove relay descriptor parser.
The purpose of this commit is to prepare adding metrics-lib as library for
descriptor parsing.
---
src/org/torproject/ernie/cron/ArchiveReader.java | 367 ++++++++++++++++++-
src/org/torproject/ernie/cron/Main.java | 9 +-
.../ernie/cron/RelayDescriptorParser.java | 387 --------------------
3 files changed, 365 insertions(+), 398 deletions(-)
diff --git a/src/org/torproject/ernie/cron/ArchiveReader.java b/src/org/torproject/ernie/cron/ArchiveReader.java
index 20a0905..ed1e505 100644
--- a/src/org/torproject/ernie/cron/ArchiveReader.java
+++ b/src/org/torproject/ernie/cron/ArchiveReader.java
@@ -11,30 +11,67 @@ import java.io.FileInputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
+import java.io.StringReader;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.SortedMap;
+import java.util.SortedSet;
import java.util.Stack;
+import java.util.TimeZone;
import java.util.TreeMap;
+import java.util.TreeSet;
import java.util.logging.Level;
import java.util.logging.Logger;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.codec.digest.DigestUtils;
+
/**
* Read in all files in a given directory and pass buffered readers of
* them to the relay descriptor parser.
*/
public class ArchiveReader {
- public ArchiveReader(RelayDescriptorParser rdp, File archivesDirectory,
+
+ /**
+ * Stats file handler that accepts parse results for bridge statistics.
+ */
+ private BridgeStatsFileHandler bsfh;
+
+ /**
+ * Relay descriptor database importer that stores relay descriptor
+ * contents for later evaluation.
+ */
+ private RelayDescriptorDatabaseImporter rddi;
+
+ /**
+ * Logger for this class.
+ */
+ private Logger logger;
+
+ private SimpleDateFormat dateTimeFormat;
+
+ public ArchiveReader(RelayDescriptorDatabaseImporter rddi,
+ BridgeStatsFileHandler bsfh, File archivesDirectory,
File statsDirectory, boolean keepImportHistory) {
- if (rdp == null || archivesDirectory == null ||
+ if (archivesDirectory == null ||
statsDirectory == null) {
throw new IllegalArgumentException();
}
+ this.rddi = rddi;
+ this.bsfh = bsfh;
+
+ this.dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ this.dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+
int parsedFiles = 0, ignoredFiles = 0;
- Logger logger = Logger.getLogger(ArchiveReader.class.getName());
+ this.logger = Logger.getLogger(ArchiveReader.class.getName());
SortedMap<String, Long>
lastArchivesImportHistory = new TreeMap<String, Long>(),
newArchivesImportHistory = new TreeMap<String, Long>();
@@ -101,7 +138,7 @@ public class ArchiveReader {
}
bis.close();
byte[] allData = baos.toByteArray();
- rdp.parse(allData);
+ this.parse(allData);
parsedFiles++;
} catch (IOException e) {
problems.add(pop);
@@ -148,5 +185,327 @@ public class ArchiveReader {
+ "directory:\nParsed " + parsedFiles + ", ignored "
+ ignoredFiles + " files.");
}
+
+ public void parse(byte[] data) {
+ try {
+ /* Remove any @ lines at the beginning of the file and parse the
+ * first non-@ line to find out the descriptor type. */
+ BufferedReader br = new BufferedReader(new StringReader(new String(
+ data, "US-ASCII")));
+ String line = br.readLine();
+ while (line != null && line.startsWith("@")) {
+ line = br.readLine();
+ }
+ if (line == null) {
+ this.logger.fine("We were given a file that doesn't contain a "
+ + "single descriptor for parsing. Ignoring.");
+ return;
+ }
+ br.close();
+
+ /* Split the byte[] possibly containing multiple descriptors into
+ * byte[]'s with one descriptor each and parse them. */
+ String startToken = null;
+ if (line.equals("network-status-version 3")) {
+ startToken = "network-status-version 3";
+ } else if (line.startsWith("router ")) {
+ startToken = "router ";
+ } else if (line.startsWith("extra-info ")) {
+ startToken = "extra-info ";
+ } else if (line.equals("dir-key-certificate-version 3")) {
+ this.logger.fine("Not parsing dir key certificate.");
+ return;
+ } else {
+ this.logger.warning("Unknown descriptor type. First line is '"
+ + line + "'. Ignoring.");
+ return;
+ }
+ String splitToken = "\n" + startToken;
+ String ascii = new String(data, "US-ASCII");
+ int length = data.length, start = ascii.indexOf(startToken);
+ while (start < length) {
+ int end = ascii.indexOf(splitToken, start);
+ if (end < 0) {
+ end = length;
+ } else {
+ end += 1;
+ }
+ byte[] descBytes = new byte[end - start];
+ System.arraycopy(data, start, descBytes, 0, end - start);
+ parseSingleDescriptor(descBytes);
+ start = end;
+ }
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not parse descriptor. "
+ + "Skipping.", e);
+ }
+ }
+
+ private void parseSingleDescriptor(byte[] data) {
+ try {
+ /* Convert descriptor to ASCII for parsing. This means we'll lose
+ * the non-ASCII chars, but we don't care about them for parsing
+ * anyway. */
+ BufferedReader br = new BufferedReader(new StringReader(new String(
+ data, "US-ASCII")));
+ String line = br.readLine();
+ SimpleDateFormat parseFormat =
+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ parseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ if (line.equals("network-status-version 3")) {
+ // TODO when parsing the current consensus, check the fresh-until
+ // time to see when we switch from hourly to half-hourly
+ // consensuses
+ boolean isConsensus = true;
+ String validAfterTime = null, nickname = null,
+ relayIdentity = null, serverDesc = null, version = null,
+ ports = null;
+ String dirSource = null, address = null;
+ long validAfter = -1L, published = -1L, bandwidth = -1L,
+ orPort = 0L, dirPort = 0L;
+ SortedSet<String> relayFlags = null;
+ StringBuilder rawStatusEntry = null;
+ SortedSet<String> hashedRelayIdentities = new TreeSet<String>();
+ while ((line = br.readLine()) != null) {
+ if (line.equals("vote-status vote")) {
+ isConsensus = false;
+ } else if (line.startsWith("valid-after ")) {
+ validAfterTime = line.substring("valid-after ".length());
+ validAfter = parseFormat.parse(validAfterTime).getTime();
+ } else if (line.startsWith("dir-source ")) {
+ dirSource = line.split(" ")[2];
+ } else if (line.startsWith("r ")) {
+ if (isConsensus && relayIdentity != null &&
+ this.rddi != null) {
+ byte[] rawDescriptor = rawStatusEntry.toString().getBytes();
+ this.rddi.addStatusEntry(validAfter, nickname,
+ relayIdentity, serverDesc, published, address, orPort,
+ dirPort, relayFlags, version, bandwidth, ports,
+ rawDescriptor);
+ relayFlags = null;
+ version = null;
+ bandwidth = -1L;
+ ports = null;
+ }
+ rawStatusEntry = new StringBuilder(line + "\n");
+ String[] parts = line.split(" ");
+ if (parts.length < 9) {
+ this.logger.log(Level.WARNING, "Could not parse r line '"
+ + line + "' in descriptor. Skipping.");
+ break;
+ }
+ nickname = parts[1];
+ relayIdentity = Hex.encodeHexString(
+ Base64.decodeBase64(parts[2] + "=")).
+ toLowerCase();
+ hashedRelayIdentities.add(DigestUtils.shaHex(
+ Base64.decodeBase64(parts[2] + "=")).
+ toUpperCase());
+ serverDesc = Hex.encodeHexString(Base64.decodeBase64(
+ parts[3] + "=")).toLowerCase();
+ published = parseFormat.parse(parts[4] + " " + parts[5]).
+ getTime();
+ address = parts[6];
+ orPort = Long.parseLong(parts[7]);
+ dirPort = Long.parseLong(parts[8]);
+ } else if (line.startsWith("s ") || line.equals("s")) {
+ rawStatusEntry.append(line + "\n");
+ relayFlags = new TreeSet<String>();
+ if (line.length() > 2) {
+ for (String flag : line.substring(2).split(" ")) {
+ relayFlags.add(flag);
+ }
+ }
+ } else if (line.startsWith("v ")) {
+ rawStatusEntry.append(line + "\n");
+ version = line.substring(2);
+ } else if (line.startsWith("w ")) {
+ rawStatusEntry.append(line + "\n");
+ String[] parts = line.split(" ");
+ for (String part : parts) {
+ if (part.startsWith("Bandwidth=")) {
+ bandwidth = Long.parseLong(part.substring(
+ "Bandwidth=".length()));
+ }
+ }
+ } else if (line.startsWith("p ")) {
+ rawStatusEntry.append(line + "\n");
+ ports = line.substring(2);
+ }
+ }
+ if (isConsensus) {
+ if (this.bsfh != null) {
+ for (String hashedRelayIdentity : hashedRelayIdentities) {
+ this.bsfh.addHashedRelay(hashedRelayIdentity);
+ }
+ }
+ if (this.rddi != null) {
+ this.rddi.addConsensus(validAfter, data);
+ if (relayIdentity != null) {
+ byte[] rawDescriptor = rawStatusEntry.toString().getBytes();
+ this.rddi.addStatusEntry(validAfter, nickname,
+ relayIdentity, serverDesc, published, address, orPort,
+ dirPort, relayFlags, version, bandwidth, ports,
+ rawDescriptor);
+ }
+ }
+ } else {
+ if (this.rddi != null) {
+ this.rddi.addVote(validAfter, dirSource, data);
+ }
+ }
+ } else if (line.startsWith("router ")) {
+ String platformLine = null, bandwidthLine = null,
+ extraInfoDigest = null, relayIdentifier = null;
+ String[] parts = line.split(" ");
+ String nickname = parts[1];
+ String address = parts[2];
+ int orPort = Integer.parseInt(parts[3]);
+ int dirPort = Integer.parseInt(parts[4]);
+ long published = -1L, uptime = -1L;
+ while ((line = br.readLine()) != null) {
+ if (line.startsWith("platform ")) {
+ platformLine = line;
+ } else if (line.startsWith("published ")) {
+ String publishedTime = line.substring("published ".length());
+ published = parseFormat.parse(publishedTime).getTime();
+ } else if (line.startsWith("opt fingerprint") ||
+ line.startsWith("fingerprint")) {
+ relayIdentifier = line.substring(line.startsWith("opt ") ?
+ "opt fingerprint".length() : "fingerprint".length()).
+ replaceAll(" ", "").toLowerCase();
+ } else if (line.startsWith("bandwidth ")) {
+ bandwidthLine = line;
+ } else if (line.startsWith("opt extra-info-digest ") ||
+ line.startsWith("extra-info-digest ")) {
+ extraInfoDigest = line.startsWith("opt ") ?
+ line.split(" ")[2].toLowerCase() :
+ line.split(" ")[1].toLowerCase();
+ } else if (line.startsWith("uptime ")) {
+ uptime = Long.parseLong(line.substring("uptime ".length()));
+ }
+ }
+ String ascii = new String(data, "US-ASCII");
+ String startToken = "router ";
+ String sigToken = "\nrouter-signature\n";
+ int start = ascii.indexOf(startToken);
+ int sig = ascii.indexOf(sigToken) + sigToken.length();
+ String digest = null;
+ if (start >= 0 || sig >= 0 || sig > start) {
+ byte[] forDigest = new byte[sig - start];
+ System.arraycopy(data, start, forDigest, 0, sig - start);
+ digest = DigestUtils.shaHex(forDigest);
+ }
+ if (this.rddi != null && digest != null) {
+ String[] bwParts = bandwidthLine.split(" ");
+ long bandwidthAvg = Long.parseLong(bwParts[1]);
+ long bandwidthBurst = Long.parseLong(bwParts[2]);
+ long bandwidthObserved = Long.parseLong(bwParts[3]);
+ String platform = platformLine.substring("platform ".length());
+ this.rddi.addServerDescriptor(digest, nickname, address, orPort,
+ dirPort, relayIdentifier, bandwidthAvg, bandwidthBurst,
+ bandwidthObserved, platform, published, uptime,
+ extraInfoDigest, data);
+ }
+ } else if (line.startsWith("extra-info ")) {
+ String nickname = line.split(" ")[1];
+ long published = -1L;
+ String dir = line.split(" ")[2];
+ String statsEnd = null;
+ long seconds = -1L;
+ List<String> bandwidthHistory = new ArrayList<String>();
+ while ((line = br.readLine()) != null) {
+ if (line.startsWith("published ")) {
+ String publishedTime = line.substring("published ".length());
+ published = parseFormat.parse(publishedTime).getTime();
+ } else if (line.startsWith("read-history ") ||
+ line.startsWith("write-history ") ||
+ line.startsWith("dirreq-read-history ") ||
+ line.startsWith("dirreq-write-history ")) {
+ bandwidthHistory.add(line);
+ } else if (line.startsWith("dirreq-stats-end ")) {
+ String[] parts = line.split(" ");
+ if (parts.length < 5) {
+ this.logger.warning("Could not parse dirreq-stats-end "
+ + "line '" + line + "' in descriptor. Skipping.");
+ break;
+ }
+ statsEnd = parts[1] + " " + parts[2];
+ seconds = Long.parseLong(parts[3].substring(1));
+ } else if (line.startsWith("dirreq-v3-reqs ")
+ && line.length() > "dirreq-v3-reqs ".length()) {
+ if (this.rddi != null) {
+ try {
+ int allUsers = 0;
+ Map<String, String> obs = new HashMap<String, String>();
+ String[] parts = line.substring("dirreq-v3-reqs ".
+ length()).split(",");
+ for (String p : parts) {
+ String country = p.substring(0, 2);
+ int users = Integer.parseInt(p.substring(3)) - 4;
+ allUsers += users;
+ obs.put(country, "" + users);
+ }
+ obs.put("zy", "" + allUsers);
+ this.rddi.addDirReqStats(dir, statsEnd, seconds, obs);
+ } catch (NumberFormatException e) {
+ this.logger.log(Level.WARNING, "Could not parse "
+ + "dirreq-v3-reqs line '" + line + "' in descriptor. "
+ + "Skipping.", e);
+ break;
+ }
+ }
+ } else if (line.startsWith("conn-bi-direct ")) {
+ if (this.rddi != null) {
+ String[] parts = line.split(" ");
+ if (parts.length == 6 &&
+ parts[5].split(",").length == 4) {
+ try {
+ String connBiDirectStatsEnd = parts[1] + " " + parts[2];
+ long connBiDirectSeconds = Long.parseLong(parts[3].
+ substring(1));
+ String[] parts2 = parts[5].split(",");
+ long below = Long.parseLong(parts2[0]);
+ long read = Long.parseLong(parts2[1]);
+ long write = Long.parseLong(parts2[2]);
+ long both = Long.parseLong(parts2[3]);
+ this.rddi.addConnBiDirect(dir, connBiDirectStatsEnd,
+ connBiDirectSeconds, below, read, write, both);
+ } catch (NumberFormatException e) {
+ this.logger.log(Level.WARNING, "Number format "
+ + "exception while parsing conn-bi-direct stats "
+ + "string '" + line + "'. Skipping.", e);
+ }
+ } else {
+ this.logger.warning("Skipping invalid conn-bi-direct "
+ + "stats string '" + line + "'.");
+ }
+ }
+ }
+ }
+ String ascii = new String(data, "US-ASCII");
+ String startToken = "extra-info ";
+ String sigToken = "\nrouter-signature\n";
+ String digest = null;
+ int start = ascii.indexOf(startToken);
+ int sig = ascii.indexOf(sigToken) + sigToken.length();
+ if (start >= 0 || sig >= 0 || sig > start) {
+ byte[] forDigest = new byte[sig - start];
+ System.arraycopy(data, start, forDigest, 0, sig - start);
+ digest = DigestUtils.shaHex(forDigest);
+ }
+ if (this.rddi != null && digest != null) {
+ this.rddi.addExtraInfoDescriptor(digest, nickname,
+ dir.toLowerCase(), published, data, bandwidthHistory);
+ }
+ }
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not parse descriptor. "
+ + "Skipping.", e);
+ } catch (ParseException e) {
+ this.logger.log(Level.WARNING, "Could not parse descriptor. "
+ + "Skipping.", e);
+ }
+ }
}
diff --git a/src/org/torproject/ernie/cron/Main.java b/src/org/torproject/ernie/cron/Main.java
index 57ad804..d631a7b 100644
--- a/src/org/torproject/ernie/cron/Main.java
+++ b/src/org/torproject/ernie/cron/Main.java
@@ -47,15 +47,10 @@ public class Main {
config.getWriteRelayDescriptorsRawFiles() ?
config.getRelayDescriptorRawFilesDirectory() : null) : null;
- // Prepare relay descriptor parser (only if we are writing the
- // consensus-health page to disk)
- RelayDescriptorParser rdp = rddi != null ?
- new RelayDescriptorParser(rddi, bsfh) : null;
-
// Import relay descriptors
- if (rdp != null) {
+ if (rddi != null) {
if (config.getImportDirectoryArchives()) {
- new ArchiveReader(rdp,
+ new ArchiveReader(rddi, bsfh,
new File(config.getDirectoryArchivesDirectory()),
statsDirectory,
config.getKeepDirectoryArchiveImportHistory());
diff --git a/src/org/torproject/ernie/cron/RelayDescriptorParser.java b/src/org/torproject/ernie/cron/RelayDescriptorParser.java
deleted file mode 100644
index ed52d4c..0000000
--- a/src/org/torproject/ernie/cron/RelayDescriptorParser.java
+++ /dev/null
@@ -1,387 +0,0 @@
-/* Copyright 2011, 2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.cron;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.StringReader;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedSet;
-import java.util.TimeZone;
-import java.util.TreeSet;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.codec.digest.DigestUtils;
-
-/**
- * Parses relay descriptors including network status consensuses and
- * votes, server and extra-info descriptors, and passes the results to the
- * stats handlers, to the archive writer, or to the relay descriptor
- * downloader.
- */
-public class RelayDescriptorParser {
-
- /**
- * Stats file handler that accepts parse results for bridge statistics.
- */
- private BridgeStatsFileHandler bsfh;
-
- /**
- * Relay descriptor database importer that stores relay descriptor
- * contents for later evaluation.
- */
- private RelayDescriptorDatabaseImporter rddi;
-
- /**
- * Logger for this class.
- */
- private Logger logger;
-
- private SimpleDateFormat dateTimeFormat;
-
- /**
- * Initializes this class.
- */
- public RelayDescriptorParser(RelayDescriptorDatabaseImporter rddi,
- BridgeStatsFileHandler bsfh) {
- this.rddi = rddi;
- this.bsfh = bsfh;
-
- /* Initialize logger. */
- this.logger = Logger.getLogger(RelayDescriptorParser.class.getName());
-
- this.dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- this.dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- }
-
- public void parse(byte[] data) {
- try {
- /* Remove any @ lines at the beginning of the file and parse the
- * first non-@ line to find out the descriptor type. */
- BufferedReader br = new BufferedReader(new StringReader(new String(
- data, "US-ASCII")));
- String line = br.readLine();
- while (line != null && line.startsWith("@")) {
- line = br.readLine();
- }
- if (line == null) {
- this.logger.fine("We were given a file that doesn't contain a "
- + "single descriptor for parsing. Ignoring.");
- return;
- }
- br.close();
-
- /* Split the byte[] possibly containing multiple descriptors into
- * byte[]'s with one descriptor each and parse them. */
- String startToken = null;
- if (line.equals("network-status-version 3")) {
- startToken = "network-status-version 3";
- } else if (line.startsWith("router ")) {
- startToken = "router ";
- } else if (line.startsWith("extra-info ")) {
- startToken = "extra-info ";
- } else if (line.equals("dir-key-certificate-version 3")) {
- this.logger.fine("Not parsing dir key certificate.");
- return;
- } else {
- this.logger.warning("Unknown descriptor type. First line is '"
- + line + "'. Ignoring.");
- return;
- }
- String splitToken = "\n" + startToken;
- String ascii = new String(data, "US-ASCII");
- int length = data.length, start = ascii.indexOf(startToken);
- while (start < length) {
- int end = ascii.indexOf(splitToken, start);
- if (end < 0) {
- end = length;
- } else {
- end += 1;
- }
- byte[] descBytes = new byte[end - start];
- System.arraycopy(data, start, descBytes, 0, end - start);
- parseSingleDescriptor(descBytes);
- start = end;
- }
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Could not parse descriptor. "
- + "Skipping.", e);
- }
- }
-
- private void parseSingleDescriptor(byte[] data) {
- try {
- /* Convert descriptor to ASCII for parsing. This means we'll lose
- * the non-ASCII chars, but we don't care about them for parsing
- * anyway. */
- BufferedReader br = new BufferedReader(new StringReader(new String(
- data, "US-ASCII")));
- String line = br.readLine();
- SimpleDateFormat parseFormat =
- new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- parseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- if (line.equals("network-status-version 3")) {
- // TODO when parsing the current consensus, check the fresh-until
- // time to see when we switch from hourly to half-hourly
- // consensuses
- boolean isConsensus = true;
- String validAfterTime = null, nickname = null,
- relayIdentity = null, serverDesc = null, version = null,
- ports = null;
- String dirSource = null, address = null;
- long validAfter = -1L, published = -1L, bandwidth = -1L,
- orPort = 0L, dirPort = 0L;
- SortedSet<String> relayFlags = null;
- StringBuilder rawStatusEntry = null;
- SortedSet<String> hashedRelayIdentities = new TreeSet<String>();
- while ((line = br.readLine()) != null) {
- if (line.equals("vote-status vote")) {
- isConsensus = false;
- } else if (line.startsWith("valid-after ")) {
- validAfterTime = line.substring("valid-after ".length());
- validAfter = parseFormat.parse(validAfterTime).getTime();
- } else if (line.startsWith("dir-source ")) {
- dirSource = line.split(" ")[2];
- } else if (line.startsWith("r ")) {
- if (isConsensus && relayIdentity != null &&
- this.rddi != null) {
- byte[] rawDescriptor = rawStatusEntry.toString().getBytes();
- this.rddi.addStatusEntry(validAfter, nickname,
- relayIdentity, serverDesc, published, address, orPort,
- dirPort, relayFlags, version, bandwidth, ports,
- rawDescriptor);
- relayFlags = null;
- version = null;
- bandwidth = -1L;
- ports = null;
- }
- rawStatusEntry = new StringBuilder(line + "\n");
- String[] parts = line.split(" ");
- if (parts.length < 9) {
- this.logger.log(Level.WARNING, "Could not parse r line '"
- + line + "' in descriptor. Skipping.");
- break;
- }
- nickname = parts[1];
- relayIdentity = Hex.encodeHexString(
- Base64.decodeBase64(parts[2] + "=")).
- toLowerCase();
- hashedRelayIdentities.add(DigestUtils.shaHex(
- Base64.decodeBase64(parts[2] + "=")).
- toUpperCase());
- serverDesc = Hex.encodeHexString(Base64.decodeBase64(
- parts[3] + "=")).toLowerCase();
- published = parseFormat.parse(parts[4] + " " + parts[5]).
- getTime();
- address = parts[6];
- orPort = Long.parseLong(parts[7]);
- dirPort = Long.parseLong(parts[8]);
- } else if (line.startsWith("s ") || line.equals("s")) {
- rawStatusEntry.append(line + "\n");
- relayFlags = new TreeSet<String>();
- if (line.length() > 2) {
- for (String flag : line.substring(2).split(" ")) {
- relayFlags.add(flag);
- }
- }
- } else if (line.startsWith("v ")) {
- rawStatusEntry.append(line + "\n");
- version = line.substring(2);
- } else if (line.startsWith("w ")) {
- rawStatusEntry.append(line + "\n");
- String[] parts = line.split(" ");
- for (String part : parts) {
- if (part.startsWith("Bandwidth=")) {
- bandwidth = Long.parseLong(part.substring(
- "Bandwidth=".length()));
- }
- }
- } else if (line.startsWith("p ")) {
- rawStatusEntry.append(line + "\n");
- ports = line.substring(2);
- }
- }
- if (isConsensus) {
- if (this.bsfh != null) {
- for (String hashedRelayIdentity : hashedRelayIdentities) {
- this.bsfh.addHashedRelay(hashedRelayIdentity);
- }
- }
- if (this.rddi != null) {
- this.rddi.addConsensus(validAfter, data);
- if (relayIdentity != null) {
- byte[] rawDescriptor = rawStatusEntry.toString().getBytes();
- this.rddi.addStatusEntry(validAfter, nickname,
- relayIdentity, serverDesc, published, address, orPort,
- dirPort, relayFlags, version, bandwidth, ports,
- rawDescriptor);
- }
- }
- } else {
- if (this.rddi != null) {
- this.rddi.addVote(validAfter, dirSource, data);
- }
- }
- } else if (line.startsWith("router ")) {
- String platformLine = null, bandwidthLine = null,
- extraInfoDigest = null, relayIdentifier = null;
- String[] parts = line.split(" ");
- String nickname = parts[1];
- String address = parts[2];
- int orPort = Integer.parseInt(parts[3]);
- int dirPort = Integer.parseInt(parts[4]);
- long published = -1L, uptime = -1L;
- while ((line = br.readLine()) != null) {
- if (line.startsWith("platform ")) {
- platformLine = line;
- } else if (line.startsWith("published ")) {
- String publishedTime = line.substring("published ".length());
- published = parseFormat.parse(publishedTime).getTime();
- } else if (line.startsWith("opt fingerprint") ||
- line.startsWith("fingerprint")) {
- relayIdentifier = line.substring(line.startsWith("opt ") ?
- "opt fingerprint".length() : "fingerprint".length()).
- replaceAll(" ", "").toLowerCase();
- } else if (line.startsWith("bandwidth ")) {
- bandwidthLine = line;
- } else if (line.startsWith("opt extra-info-digest ") ||
- line.startsWith("extra-info-digest ")) {
- extraInfoDigest = line.startsWith("opt ") ?
- line.split(" ")[2].toLowerCase() :
- line.split(" ")[1].toLowerCase();
- } else if (line.startsWith("uptime ")) {
- uptime = Long.parseLong(line.substring("uptime ".length()));
- }
- }
- String ascii = new String(data, "US-ASCII");
- String startToken = "router ";
- String sigToken = "\nrouter-signature\n";
- int start = ascii.indexOf(startToken);
- int sig = ascii.indexOf(sigToken) + sigToken.length();
- String digest = null;
- if (start >= 0 || sig >= 0 || sig > start) {
- byte[] forDigest = new byte[sig - start];
- System.arraycopy(data, start, forDigest, 0, sig - start);
- digest = DigestUtils.shaHex(forDigest);
- }
- if (this.rddi != null && digest != null) {
- String[] bwParts = bandwidthLine.split(" ");
- long bandwidthAvg = Long.parseLong(bwParts[1]);
- long bandwidthBurst = Long.parseLong(bwParts[2]);
- long bandwidthObserved = Long.parseLong(bwParts[3]);
- String platform = platformLine.substring("platform ".length());
- this.rddi.addServerDescriptor(digest, nickname, address, orPort,
- dirPort, relayIdentifier, bandwidthAvg, bandwidthBurst,
- bandwidthObserved, platform, published, uptime,
- extraInfoDigest, data);
- }
- } else if (line.startsWith("extra-info ")) {
- String nickname = line.split(" ")[1];
- long published = -1L;
- String dir = line.split(" ")[2];
- String statsEnd = null;
- long seconds = -1L;
- List<String> bandwidthHistory = new ArrayList<String>();
- while ((line = br.readLine()) != null) {
- if (line.startsWith("published ")) {
- String publishedTime = line.substring("published ".length());
- published = parseFormat.parse(publishedTime).getTime();
- } else if (line.startsWith("read-history ") ||
- line.startsWith("write-history ") ||
- line.startsWith("dirreq-read-history ") ||
- line.startsWith("dirreq-write-history ")) {
- bandwidthHistory.add(line);
- } else if (line.startsWith("dirreq-stats-end ")) {
- String[] parts = line.split(" ");
- if (parts.length < 5) {
- this.logger.warning("Could not parse dirreq-stats-end "
- + "line '" + line + "' in descriptor. Skipping.");
- break;
- }
- statsEnd = parts[1] + " " + parts[2];
- seconds = Long.parseLong(parts[3].substring(1));
- } else if (line.startsWith("dirreq-v3-reqs ")
- && line.length() > "dirreq-v3-reqs ".length()) {
- if (this.rddi != null) {
- try {
- int allUsers = 0;
- Map<String, String> obs = new HashMap<String, String>();
- String[] parts = line.substring("dirreq-v3-reqs ".
- length()).split(",");
- for (String p : parts) {
- String country = p.substring(0, 2);
- int users = Integer.parseInt(p.substring(3)) - 4;
- allUsers += users;
- obs.put(country, "" + users);
- }
- obs.put("zy", "" + allUsers);
- this.rddi.addDirReqStats(dir, statsEnd, seconds, obs);
- } catch (NumberFormatException e) {
- this.logger.log(Level.WARNING, "Could not parse "
- + "dirreq-v3-reqs line '" + line + "' in descriptor. "
- + "Skipping.", e);
- break;
- }
- }
- } else if (line.startsWith("conn-bi-direct ")) {
- if (this.rddi != null) {
- String[] parts = line.split(" ");
- if (parts.length == 6 &&
- parts[5].split(",").length == 4) {
- try {
- String connBiDirectStatsEnd = parts[1] + " " + parts[2];
- long connBiDirectSeconds = Long.parseLong(parts[3].
- substring(1));
- String[] parts2 = parts[5].split(",");
- long below = Long.parseLong(parts2[0]);
- long read = Long.parseLong(parts2[1]);
- long write = Long.parseLong(parts2[2]);
- long both = Long.parseLong(parts2[3]);
- this.rddi.addConnBiDirect(dir, connBiDirectStatsEnd,
- connBiDirectSeconds, below, read, write, both);
- } catch (NumberFormatException e) {
- this.logger.log(Level.WARNING, "Number format "
- + "exception while parsing conn-bi-direct stats "
- + "string '" + line + "'. Skipping.", e);
- }
- } else {
- this.logger.warning("Skipping invalid conn-bi-direct "
- + "stats string '" + line + "'.");
- }
- }
- }
- }
- String ascii = new String(data, "US-ASCII");
- String startToken = "extra-info ";
- String sigToken = "\nrouter-signature\n";
- String digest = null;
- int start = ascii.indexOf(startToken);
- int sig = ascii.indexOf(sigToken) + sigToken.length();
- if (start >= 0 || sig >= 0 || sig > start) {
- byte[] forDigest = new byte[sig - start];
- System.arraycopy(data, start, forDigest, 0, sig - start);
- digest = DigestUtils.shaHex(forDigest);
- }
- if (this.rddi != null && digest != null) {
- this.rddi.addExtraInfoDescriptor(digest, nickname,
- dir.toLowerCase(), published, data, bandwidthHistory);
- }
- }
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Could not parse descriptor. "
- + "Skipping.", e);
- } catch (ParseException e) {
- this.logger.log(Level.WARNING, "Could not parse descriptor. "
- + "Skipping.", e);
- }
- }
-}
-
More information about the tor-commits
mailing list