[tor-commits] [metrics-web/master] Stop importing raw consensuses and server descriptors.
karsten at torproject.org
karsten at torproject.org
Thu Dec 13 15:56:28 UTC 2012
commit 1f2e10e5c1d0636335b521a33ef876778c13ded7
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Thu Dec 13 16:11:14 2012 +0100
Stop importing raw consensuses and server descriptors.
---
db/tordir.sql | 2 -
.../cron/RelayDescriptorDatabaseImporter.java | 48 +++++--------------
2 files changed, 13 insertions(+), 37 deletions(-)
diff --git a/db/tordir.sql b/db/tordir.sql
index 5f533c4..3749774 100644
--- a/db/tordir.sql
+++ b/db/tordir.sql
@@ -19,7 +19,6 @@ CREATE TABLE descriptor (
published TIMESTAMP WITHOUT TIME ZONE NOT NULL,
uptime BIGINT,
extrainfo CHARACTER(40),
- rawdesc BYTEA NOT NULL,
CONSTRAINT descriptor_pkey PRIMARY KEY (descriptor)
);
@@ -131,7 +130,6 @@ CREATE TRIGGER insert_statusentry_trigger
-- Contains all of the consensuses published by the directories.
CREATE TABLE consensus (
validafter TIMESTAMP WITHOUT TIME ZONE NOT NULL,
- rawdesc BYTEA NOT NULL,
CONSTRAINT consensus_pkey PRIMARY KEY (validafter)
);
diff --git a/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java b/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java
index 1bbd8d9..5d2724a 100644
--- a/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java
+++ b/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java
@@ -256,12 +256,12 @@ public final class RelayDescriptorDatabaseImporter {
+ "(descriptor, nickname, address, orport, dirport, "
+ "fingerprint, bandwidthavg, bandwidthburst, "
+ "bandwidthobserved, platform, published, uptime, "
- + "extrainfo, rawdesc) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, "
- + "?, ?, ?, ?)");
+ + "extrainfo) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, "
+ + "?)");
this.csH = conn.prepareCall("{call insert_bwhist(?, ?, ?, ?, ?, "
+ "?)}");
this.psC = conn.prepareStatement("INSERT INTO consensus "
- + "(validafter, rawdesc) VALUES (?, ?)");
+ + "(validafter) VALUES (?)");
this.psQ = conn.prepareStatement("INSERT INTO dirreq_stats "
+ "(source, statsend, seconds, country, requests) VALUES "
+ "(?, ?, ?, ?, ?)");
@@ -454,7 +454,7 @@ public final class RelayDescriptorDatabaseImporter {
String address, int orPort, int dirPort, String relayIdentifier,
long bandwidthAvg, long bandwidthBurst, long bandwidthObserved,
String platform, long published, long uptime,
- String extraInfoDigest, byte[] rawDescriptor) {
+ String extraInfoDigest) {
if (this.importIntoDatabase) {
try {
this.addDateToScheduledUpdates(published);
@@ -484,7 +484,6 @@ public final class RelayDescriptorDatabaseImporter {
this.psD.setTimestamp(11, new Timestamp(published), cal);
this.psD.setLong(12, uptime);
this.psD.setString(13, extraInfoDigest);
- this.psD.setBytes(14, rawDescriptor);
this.psD.executeUpdate();
rdsCount++;
if (rdsCount % autoCommitCount == 0) {
@@ -509,8 +508,7 @@ public final class RelayDescriptorDatabaseImporter {
this.descriptorOut.write(" COPY descriptor (descriptor, "
+ "nickname, address, orport, dirport, fingerprint, "
+ "bandwidthavg, bandwidthburst, bandwidthobserved, "
- + "platform, published, uptime, extrainfo, rawdesc) FROM "
- + "stdin;\n");
+ + "platform, published, uptime, extrainfo) FROM stdin;\n");
}
this.descriptorOut.write(descriptor.toLowerCase() + "\t"
+ nickname + "\t" + address + "\t" + orPort + "\t" + dirPort
@@ -521,17 +519,9 @@ public final class RelayDescriptorDatabaseImporter {
+ "\t" + this.dateTimeFormat.format(published) + "\t"
+ (uptime >= 0 ? uptime : "\\N") + "\t"
+ (extraInfoDigest != null ? extraInfoDigest : "\\N")
- + "\t");
- this.descriptorOut.write(PGbytea.toPGString(rawDescriptor).
- replaceAll("\\\\", "\\\\\\\\") + "\n");
+ + "\n");
} catch (UnsupportedEncodingException e) {
// US-ASCII is supported for sure
- } catch (SQLException e) {
- this.logger.log(Level.WARNING, "Could not write server "
- + "descriptor to raw database import file. We won't make "
- + "any further attempts to write raw import files in this "
- + "execution.", e);
- this.writeRawImportFiles = false;
} catch (IOException e) {
this.logger.log(Level.WARNING, "Could not write server "
+ "descriptor to raw database import file. We won't make "
@@ -547,7 +537,7 @@ public final class RelayDescriptorDatabaseImporter {
*/
public void addExtraInfoDescriptor(String extraInfoDigest,
String nickname, String fingerprint, long published,
- byte[] rawDescriptor, List<String> bandwidthHistoryLines) {
+ List<String> bandwidthHistoryLines) {
if (!bandwidthHistoryLines.isEmpty()) {
this.addBandwidthHistory(fingerprint.toLowerCase(), published,
bandwidthHistoryLines);
@@ -803,7 +793,7 @@ public final class RelayDescriptorDatabaseImporter {
/**
* Insert network status consensus into database.
*/
- public void addConsensus(long validAfter, byte[] rawDescriptor) {
+ public void addConsensus(long validAfter) {
if (this.importIntoDatabase) {
try {
this.addDateToScheduledUpdates(validAfter);
@@ -815,7 +805,6 @@ public final class RelayDescriptorDatabaseImporter {
if (rs.getInt(1) == 0) {
this.psC.clearParameters();
this.psC.setTimestamp(1, validAfterTimestamp, cal);
- this.psC.setBytes(2, rawDescriptor);
this.psC.executeUpdate();
rcsCount++;
if (rcsCount % autoCommitCount == 0) {
@@ -835,19 +824,11 @@ public final class RelayDescriptorDatabaseImporter {
new File(rawFilesDirectory).mkdirs();
this.consensusOut = new BufferedWriter(new FileWriter(
rawFilesDirectory + "/consensus.sql"));
- this.consensusOut.write(" COPY consensus (validafter, rawdesc) "
+ this.consensusOut.write(" COPY consensus (validafter) "
+ "FROM stdin;\n");
}
String validAfterString = this.dateTimeFormat.format(validAfter);
- this.consensusOut.write(validAfterString + "\t");
- this.consensusOut.write(PGbytea.toPGString(rawDescriptor).
- replaceAll("\\\\", "\\\\\\\\") + "\n");
- } catch (SQLException e) {
- this.logger.log(Level.WARNING, "Could not write network status "
- + "consensus to raw database import file. We won't make "
- + "any further attempts to write raw import files in this "
- + "execution.", e);
- this.writeRawImportFiles = false;
+ this.consensusOut.write(validAfterString + "\n");
} catch (IOException e) {
this.logger.log(Level.WARNING, "Could not write network status "
+ "consensus to raw database import file. We won't make "
@@ -967,8 +948,7 @@ public final class RelayDescriptorDatabaseImporter {
statusEntry.getBandwidth(), statusEntry.getPortList(),
statusEntry.getStatusEntryBytes());
}
- this.addConsensus(consensus.getValidAfterMillis(),
- consensus.getRawDescriptorBytes());
+ this.addConsensus(consensus.getValidAfterMillis());
}
private void addServerDescriptor(ServerDescriptor descriptor) {
@@ -978,8 +958,7 @@ public final class RelayDescriptorDatabaseImporter {
descriptor.getFingerprint(), descriptor.getBandwidthRate(),
descriptor.getBandwidthBurst(), descriptor.getBandwidthObserved(),
descriptor.getPlatform(), descriptor.getPublishedMillis(),
- descriptor.getUptime(), descriptor.getExtraInfoDigest(),
- descriptor.getRawDescriptorBytes());
+ descriptor.getUptime(), descriptor.getExtraInfoDigest());
}
private void addExtraInfoDescriptor(ExtraInfoDescriptor descriptor) {
@@ -1016,8 +995,7 @@ public final class RelayDescriptorDatabaseImporter {
this.addExtraInfoDescriptor(descriptor.getExtraInfoDigest(),
descriptor.getNickname(),
descriptor.getFingerprint().toLowerCase(),
- descriptor.getPublishedMillis(),
- descriptor.getRawDescriptorBytes(), bandwidthHistoryLines);
+ descriptor.getPublishedMillis(), bandwidthHistoryLines);
}
/**
More information about the tor-commits
mailing list