commit 85dc1ad844373bb88fb67bc94e32269990438fcb Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Wed Mar 2 20:32:09 2011 +0100 Remove bridge statistics from metrics-db. --- config.template | 22 - .../ernie/db/BridgeDescriptorParser.java | 158 ++----- .../ernie/db/BridgeStatsFileHandler.java | 519 -------------------- src/org/torproject/ernie/db/Configuration.java | 59 +-- .../ernie/db/ConsensusStatsFileHandler.java | 282 ----------- src/org/torproject/ernie/db/Main.java | 39 +-- .../torproject/ernie/db/RelayDescriptorParser.java | 14 +- .../ernie/db/SanitizedBridgesReader.java | 114 ----- .../ernie/test/SanitizedBridgesReaderTest.java | 33 -- 9 files changed, 42 insertions(+), 1198 deletions(-) diff --git a/config.template b/config.template index 7385f7b..3477801 100644 --- a/config.template +++ b/config.template @@ -26,18 +26,6 @@ ## download missing relay descriptors from #DownloadFromDirectoryAuthorities 86.59.21.38,194.109.206.212,212.112.245.170 # -## Import sanitized bridges from disk, if available -#ImportSanitizedBridges 0 -# -## Relative path to directory to import sanitized bridges from -#SanitizedBridgesDirectory bridges/ -# -## Keep a history of imported sanitized bridge descriptors. This history -## can be useful when importing from a changing data source to avoid -## importing descriptors more than once, but it can be confusing to users -## who don't know about it. -#KeepSanitizedBridgesImportHistory 0 -# ## Import bridge snapshots from disk, if available #ImportBridgeSnapshots 0 # @@ -75,10 +63,6 @@ ## JDBC string for relay descriptor database #RelayDescriptorDatabaseJDBC jdbc:postgresql://localhost/tordir?user=ernie&password=password # -## Write statistics about the current consensus and votes to the -## website -#WriteConsensusHealth 0 -# ## Write sanitized bridges to disk #WriteSanitizedBridges 0 # @@ -93,10 +77,4 @@ # ## Relative path to directory to write sanitized bridges to #SanitizedBridgesWriteDirectory sanitized-bridges/ -# -## Write consensus stats to disk -#WriteConsensusStats 0 -# -## Write bridge stats to disk -#WriteBridgeStats 0 diff --git a/src/org/torproject/ernie/db/BridgeDescriptorParser.java b/src/org/torproject/ernie/db/BridgeDescriptorParser.java index e2d8bd0..9453088 100644 --- a/src/org/torproject/ernie/db/BridgeDescriptorParser.java +++ b/src/org/torproject/ernie/db/BridgeDescriptorParser.java @@ -9,14 +9,9 @@ import java.util.logging.*; import org.apache.commons.codec.digest.*; public class BridgeDescriptorParser { - private ConsensusStatsFileHandler csfh; - private BridgeStatsFileHandler bsfh; private SanitizedBridgesWriter sbw; private Logger logger; - public BridgeDescriptorParser(ConsensusStatsFileHandler csfh, - BridgeStatsFileHandler bsfh, SanitizedBridgesWriter sbw) { - this.csfh = csfh; - this.bsfh = bsfh; + public BridgeDescriptorParser(SanitizedBridgesWriter sbw) { this.sbw = sbw; this.logger = Logger.getLogger(BridgeDescriptorParser.class.getName()); @@ -25,139 +20,46 @@ public class BridgeDescriptorParser { try { BufferedReader br = new BufferedReader(new StringReader( new String(allData, "US-ASCII"))); - SimpleDateFormat timeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - timeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - String hashedIdentity = null, platformLine = null, - publishedLine = null, geoipStartTimeLine = null, - bridgeStatsEndLine = null; boolean skip = false; - String line = null; - while ((line = br.readLine()) != null) { - if (line.startsWith("r ")) { - if (this.sbw != null) { - if (sanitized) { - this.sbw.storeSanitizedNetworkStatus(allData, dateTime); - } else { - this.sbw.sanitizeAndStoreNetworkStatus(allData, dateTime); - } + String line = br.readLine(); + if (line != null) { + return; + } else if (line.startsWith("r ")) { + if (this.sbw != null) { + if (sanitized) { + this.sbw.storeSanitizedNetworkStatus(allData, dateTime); + } else { + this.sbw.sanitizeAndStoreNetworkStatus(allData, dateTime); } - int runningBridges = 0; - while ((line = br.readLine()) != null) { - if (line.startsWith("s ") && line.contains(" Running")) { - runningBridges++; - } - } - if (this.csfh != null) { - this.csfh.addBridgeConsensusResults(dateTime, runningBridges); - } - } else if (line.startsWith("router ")) { - if (this.sbw != null) { - if (sanitized) { - this.sbw.storeSanitizedServerDescriptor(allData); - } else { - this.sbw.sanitizeAndStoreServerDescriptor(allData); - } - } - } else if (line.startsWith("extra-info ")) { - if (this.sbw != null) { - if (sanitized) { - this.sbw.storeSanitizedExtraInfoDescriptor(allData); - } else { - this.sbw.sanitizeAndStoreExtraInfoDescriptor(allData); - } - } - hashedIdentity = sanitized ? line.split(" ")[2] - : DigestUtils.shaHex(line.split(" ")[2]).toUpperCase(); - if (this.bsfh != null) { - skip = this.bsfh.isKnownRelay(hashedIdentity); - } - } else if (!skip && line.startsWith("platform ")) { - platformLine = line; - } else if (!skip && line.startsWith("published ")) { - publishedLine = line; - } else if (line.startsWith("opt fingerprint") || - line.startsWith("fingerprint")) { - String identity = line.substring(line.startsWith("opt ") ? - "opt fingerprint".length() : "fingerprint".length()). - replaceAll(" ", "").toLowerCase(); - hashedIdentity = sanitized ? identity - : DigestUtils.shaHex(identity).toUpperCase(); - } else if (!skip && line.startsWith("geoip-start-time ")) { - geoipStartTimeLine = line; - } else if (!skip && line.startsWith("geoip-client-origins") - && line.split(" ").length > 1) { - if (publishedLine == null || - geoipStartTimeLine == null) { - this.logger.warning("Either published line or " - + "geoip-start-time line is not present in " - + (sanitized ? "sanitized" : "non-sanitized") - + " bridge descriptors from " + dateTime + "."); - break; - } - long published = timeFormat.parse(publishedLine. - substring("published ".length())).getTime(); - long started = timeFormat.parse(geoipStartTimeLine. - substring("geoip-start-time ".length())).getTime(); - long seconds = (published - started) / 1000L; - double allUsers = 0.0D; - Map<String, String> obs = new HashMap<String, String>(); - String[] parts = line.split(" ")[1].split(","); - for (String p : parts) { - String country = p.substring(0, 2); - double users = ((double) Long.parseLong(p.substring(3)) - 4L) - * 86400.0D / ((double) seconds); - allUsers += users; - obs.put(country, String.format("%.2f", users)); - } - obs.put("zy", String.format("%.2f", allUsers)); - String date = publishedLine.split(" ")[1]; - String time = publishedLine.split(" ")[2]; - if (this.bsfh != null) { - this.bsfh.addObs(hashedIdentity, date, time, obs); - } - } else if (!skip && line.startsWith("bridge-stats-end ")) { - bridgeStatsEndLine = line; - } else if (!skip && line.startsWith("bridge-ips") - && line.split(" ").length > 1) { - if (bridgeStatsEndLine == null) { - this.logger.warning("bridge-ips line without preceding " - + "bridge-stats-end line in " - + (sanitized ? "sanitized" : "non-sanitized") - + " bridge descriptor."); - break; + } + int runningBridges = 0; + while ((line = br.readLine()) != null) { + if (line.startsWith("s ") && line.contains(" Running")) { + runningBridges++; } - double allUsers = 0.0D; - Map<String, String> obs = new HashMap<String, String>(); - String[] parts = line.split(" ")[1].split(","); - for (String p : parts) { - String country = p.substring(0, 2); - double users = (double) Long.parseLong(p.substring(3)) - 4L; - allUsers += users; - obs.put(country, String.format("%.2f", users)); + } + } else if (line.startsWith("router ")) { + if (this.sbw != null) { + if (sanitized) { + this.sbw.storeSanitizedServerDescriptor(allData); + } else { + this.sbw.sanitizeAndStoreServerDescriptor(allData); } - obs.put("zy", String.format("%.2f", allUsers)); - String date = bridgeStatsEndLine.split(" ")[1]; - String time = bridgeStatsEndLine.split(" ")[2]; - if (this.bsfh != null) { - this.bsfh.addObs(hashedIdentity, date, time, obs); + } + } else if (line.startsWith("extra-info ")) { + if (this.sbw != null) { + if (sanitized) { + this.sbw.storeSanitizedExtraInfoDescriptor(allData); + } else { + this.sbw.sanitizeAndStoreExtraInfoDescriptor(allData); } } } - if (this.bsfh != null && platformLine != null && - platformLine.startsWith("platform Tor 0.2.2")) { - String date = publishedLine.split(" ")[1]; - String time = publishedLine.split(" ")[2]; - this.bsfh.addZeroTwoTwoDescriptor(hashedIdentity, date, time); - } } catch (IOException e) { this.logger.log(Level.WARNING, "Could not parse bridge descriptor.", e); return; - } catch (ParseException e) { - this.logger.log(Level.WARNING, "Could not parse bridge descriptor.", - e); - return; } } } + diff --git a/src/org/torproject/ernie/db/BridgeStatsFileHandler.java b/src/org/torproject/ernie/db/BridgeStatsFileHandler.java deleted file mode 100644 index 8a417b5..0000000 --- a/src/org/torproject/ernie/db/BridgeStatsFileHandler.java +++ /dev/null @@ -1,519 +0,0 @@ -/* Copyright 2010 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.ernie.db; - -import java.io.*; -import java.sql.*; -import java.text.*; -import java.util.*; -import java.util.logging.*; - -/** - * Determines estimates of bridge users per country and day from the - * extra-info descriptors that bridges publish. In a first step, the - * number of unique IP addresses that bridges see are normalized to a - * 24-hour period. In the next step, all bridges are excluded that have - * been running as a relay. Finally, observations are summed up and - * written to <code>stats/bridge-stats</code>. - */ -public class BridgeStatsFileHandler { - - /** - * Two-letter country codes of known countries. - */ - private SortedSet<String> countries; - - /** - * Intermediate results file containing bridge user numbers by country - * as seen by single bridges, normalized to 24-hour periods. - */ - private File bridgeStatsRawFile; - - /** - * Bridge user numbers by country as seen by single bridges on a given - * day. Map keys are bridge and date written as "bridge,date", map - * values are lines as read from <code>stats/bridge-stats-raw</code>. - */ - private SortedMap<String, Map<String, String>> bridgeUsersRaw; - - /** - * Helper file containing the hashed relay identities of all known - * relays. These hashes are compared to the bridge identity hashes to - * exclude bridges that have been known as relays from the statistics. - */ - private File hashedRelayIdentitiesFile; - - /** - * Known hashed relay identities used to exclude bridges that have been - * running as relays. - */ - private SortedSet<String> hashedRelays; - - /** - * Helper file containing extra-info descriptors published by 0.2.2.x - * bridges. If these descriptors contain geoip-stats, they are not - * included in the results, because stats are very likely broken. - */ - private File zeroTwoTwoDescriptorsFile; - - /** - * Extra-info descriptors published by 0.2.2.x bridges. If these - * descriptors contain geoip-stats, they are not included in the - * results, because stats are very likely broken. - */ - private SortedSet<String> zeroTwoTwoDescriptors; - - /** - * Final results file containing the number of bridge users per country - * and day. This file is not read in during initialization, but - * overwritten at the end of the execution. - */ - private File bridgeStatsFile; - - /** - * Logger for this class. - */ - private Logger logger; - - /* Database connection string. */ - private String connectionURL = null; - - /** - * Initializes this class, including reading in intermediate results - * files <code>stats/bridge-stats-raw</code> and - * <code>stats/hashed-relay-identities</code>. - */ - public BridgeStatsFileHandler(String connectionURL) { - - /* Initialize set of known countries. */ - this.countries = new TreeSet<String>(); - this.countries.add("zy"); - - /* Initialize local data structures to hold results. */ - this.bridgeUsersRaw = new TreeMap<String, Map<String, String>>(); - this.hashedRelays = new TreeSet<String>(); - this.zeroTwoTwoDescriptors = new TreeSet<String>(); - - /* Initialize file names for intermediate and final results. */ - this.bridgeStatsRawFile = new File("stats/bridge-stats-raw"); - this.bridgeStatsFile = new File("stats/bridge-stats"); - this.hashedRelayIdentitiesFile = new File( - "stats/hashed-relay-identities"); - this.zeroTwoTwoDescriptorsFile = new File( - "stats/v022-bridge-descriptors"); - - /* Initialize database connection string. */ - this.connectionURL = connectionURL; - - /* Initialize logger. */ - this.logger = Logger.getLogger( - BridgeStatsFileHandler.class.getName()); - - /* Read in bridge user numbers by country as seen by single bridges, - * normalized to 24-hour periods. */ - if (this.bridgeStatsRawFile.exists()) { - try { - this.logger.fine("Reading file " - + this.bridgeStatsRawFile.getAbsolutePath() + "..."); - BufferedReader br = new BufferedReader(new FileReader( - this.bridgeStatsRawFile)); - String line = br.readLine(); - if (line != null) { - /* The first line should contain headers that we need to parse - * in order to learn what countries we were interested in when - * writing this file. */ - if (!line.startsWith("bridge,date,time,")) { - this.logger.warning("Incorrect first line '" + line + "' in " - + this.bridgeStatsRawFile.getAbsolutePath() + "! This line " - + "should contain headers! Aborting to read in this " - + "file!"); - } else { - String[] headers = line.split(","); - for (int i = 3; i < headers.length; i++) { - if (!headers[i].equals("all")) { - this.countries.add(headers[i]); - } - } - /* Read in the rest of the file. */ - while ((line = br.readLine()) != null) { - String[] parts = line.split(","); - if (parts.length != headers.length) { - this.logger.warning("Corrupt line '" + line + "' in file " - + this.bridgeStatsRawFile.getAbsolutePath() - + "! Aborting to read this file!"); - break; - } - String hashedBridgeIdentity = parts[0]; - String date = parts[1]; - String time = parts[2]; - SortedMap<String, String> obs = - new TreeMap<String, String>(); - for (int i = 3; i < parts.length; i++) { - if (parts[i].equals("NA")) { - continue; - } - if (headers[i].equals("all")) { - obs.put("zy", parts[i]); - } else { - obs.put(headers[i], parts[i]); - } - } - this.addObs(hashedBridgeIdentity, date, time, obs); - } - } - } - br.close(); - this.logger.fine("Finished reading file " - + this.bridgeStatsRawFile.getAbsolutePath() + "."); - } catch (IOException e) { - this.logger.log(Level.WARNING, "Failed to read file " - + this.bridgeStatsRawFile.getAbsolutePath() + "!", e); - } - } - - /* Read in known hashed relay identities used to exclude bridges that - * have been running as relays. */ - if (this.hashedRelayIdentitiesFile.exists()) { - try { - this.logger.fine("Reading file " - + this.hashedRelayIdentitiesFile.getAbsolutePath() + "..."); - BufferedReader br = new BufferedReader(new FileReader( - this.hashedRelayIdentitiesFile)); - String line = null; - /* Read in all lines from the file and memorize them. */ - while ((line = br.readLine()) != null) { - this.hashedRelays.add(line); - } - br.close(); - this.logger.fine("Finished reading file " - + this.hashedRelayIdentitiesFile.getAbsolutePath() + "."); - } catch (IOException e) { - this.logger.log(Level.WARNING, "Failed to read file " - + this.hashedRelayIdentitiesFile.getAbsolutePath() + "!", e); - } - } - - /* Read in known extra-info descriptors published by 0.2.2.x - * bridges. */ - if (this.zeroTwoTwoDescriptorsFile.exists()) { - try { - this.logger.fine("Reading file " - + this.zeroTwoTwoDescriptorsFile.getAbsolutePath() + "..."); - BufferedReader br = new BufferedReader(new FileReader( - this.zeroTwoTwoDescriptorsFile)); - String line = null; - /* Read in all lines from the file and memorize them. */ - while ((line = br.readLine()) != null) { - this.zeroTwoTwoDescriptors.add(line); - } - br.close(); - this.logger.fine("Finished reading file " - + this.zeroTwoTwoDescriptorsFile.getAbsolutePath() + "."); - } catch (IOException e) { - this.logger.log(Level.WARNING, "Failed to read file " - + this.zeroTwoTwoDescriptorsFile.getAbsolutePath() + "!", e); - } - } - } - - /** - * Adds a hashed relay identity string to the list of bridges that we - * are going to ignore in the future. If we counted user numbers from - * bridges that have been running as relays, our numbers would be far - * higher than what we think is correct. - */ - public void addHashedRelay(String hashedRelayIdentity) { - if (!this.hashedRelays.contains(hashedRelayIdentity)) { - this.logger.finer("Adding new hashed relay identity: " - + hashedRelayIdentity); - this.hashedRelays.add(hashedRelayIdentity); - } - } - - /** - * Adds an extra-info descriptor identifier published by an 0.2.2.x - * bridges. If this extra-info descriptor contains geoip-stats, they are - * not included in the results, because stats are very likely broken. - */ - public void addZeroTwoTwoDescriptor(String hashedBridgeIdentity, - String date, String time) { - String value = hashedBridgeIdentity.toUpperCase() + "," + date + "," - + time; - if (!this.zeroTwoTwoDescriptors.contains(value)) { - this.logger.finer("Adding new bridge 0.2.2.x extra-info " - + "descriptor: " + value); - this.zeroTwoTwoDescriptors.add(value); - } - } - - /** - * Returns whether the given fingerprint is a known hashed relay - * identity. <code>BridgeDescriptorParser</code> uses this information - * to decide whether to continue parsing a bridge extra-descriptor - * descriptor or not. - */ - public boolean isKnownRelay(String hashedBridgeIdentity) { - return this.hashedRelays.contains(hashedBridgeIdentity); - } - - /** - * Adds bridge user numbers by country as seen by a single bridge on a - * given date and time. Bridges can publish statistics on unique IP - * addresses multiple times a day, but we only want to include one - * observation per day. If we already have an observation from the given - * bridge and day, we keep the one with the later publication time and - * discard the other one. - */ - public void addObs(String hashedIdentity, String date, String time, - Map<String, String> obs) { - for (String country : obs.keySet()) { - this.countries.add(country); - } - String shortKey = hashedIdentity + "," + date; - String longKey = shortKey + "," + time; - SortedMap<String, Map<String, String>> tailMap = - this.bridgeUsersRaw.tailMap(shortKey); - String nextKey = tailMap.isEmpty() ? null : tailMap.firstKey(); - if (nextKey == null || !nextKey.startsWith(shortKey)) { - this.logger.finer("Adding new bridge user numbers for key " - + longKey); - this.bridgeUsersRaw.put(longKey, obs); - } else if (longKey.compareTo(nextKey) > 0) { - this.logger.finer("Replacing existing bridge user numbers (" + - nextKey + " with new numbers: " + longKey); - this.bridgeUsersRaw.put(longKey, obs); - } else { - this.logger.finer("Not replacing existing bridge user numbers (" + - nextKey + " with new numbers (" + longKey + ")."); - } - } - - /** - * Writes the list of hashed relay identities and bridge user numbers as - * observed by single bridges to disk, aggregates per-day statistics for - * all bridges, and writes those to disk, too. - */ - public void writeFiles() { - - /* Write hashed relay identities to disk. */ - try { - this.logger.fine("Writing file " - + this.hashedRelayIdentitiesFile.getAbsolutePath() + "..."); - this.hashedRelayIdentitiesFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter( - this.hashedRelayIdentitiesFile)); - for (String hashedRelay : this.hashedRelays) { - bw.append(hashedRelay + "\n"); - } - bw.close(); - this.logger.fine("Finished writing file " - + this.hashedRelayIdentitiesFile.getAbsolutePath() + "."); - } catch (IOException e) { - this.logger.log(Level.WARNING, "Failed to write " - + this.hashedRelayIdentitiesFile.getAbsolutePath() + "!", e); - } - - /* Write bridge extra-info descriptor identifiers to disk. */ - try { - this.logger.fine("Writing file " - + this.zeroTwoTwoDescriptorsFile.getAbsolutePath() + "..."); - this.zeroTwoTwoDescriptorsFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter( - this.zeroTwoTwoDescriptorsFile)); - for (String descriptorIdentifier : this.zeroTwoTwoDescriptors) { - bw.append(descriptorIdentifier + "\n"); - } - bw.close(); - this.logger.fine("Finished writing file " - + this.zeroTwoTwoDescriptorsFile.getAbsolutePath() + "."); - } catch (IOException e) { - this.logger.log(Level.WARNING, "Failed to write " - + this.zeroTwoTwoDescriptorsFile.getAbsolutePath() + "!", e); - } - - /* Write observations made by single bridges to disk. */ - try { - this.logger.fine("Writing file " - + this.bridgeStatsRawFile.getAbsolutePath() + "..."); - this.bridgeStatsRawFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter( - this.bridgeStatsRawFile)); - bw.append("bridge,date,time"); - for (String c : this.countries) { - if (c.equals("zy")) { - bw.append(",all"); - } else { - bw.append("," + c); - } - } - bw.append("\n"); - for (Map.Entry<String, Map<String, String>> e : - this.bridgeUsersRaw.entrySet()) { - String longKey = e.getKey(); - String[] parts = longKey.split(","); - String hashedBridgeIdentity = parts[0]; - String date = parts[1]; - String time = parts[2]; - if (!this.hashedRelays.contains(hashedBridgeIdentity) && - !this.zeroTwoTwoDescriptors.contains(longKey)) { - Map<String, String> obs = e.getValue(); - StringBuilder sb = new StringBuilder(longKey); - for (String c : this.countries) { - sb.append("," + (obs.containsKey(c) && - !obs.get(c).startsWith("-") ? obs.get(c) : "NA")); - } - String line = sb.toString(); - bw.append(line + "\n"); - } - } - bw.close(); - this.logger.fine("Finished writing file " - + this.bridgeStatsRawFile.getAbsolutePath() + "."); - } catch (IOException e) { - this.logger.log(Level.WARNING, "Failed to write " - + this.bridgeStatsRawFile.getAbsolutePath() + "!", e); - } - - /* Aggregate per-day statistics. */ - SortedMap<String, double[]> bridgeUsersPerDay = - new TreeMap<String, double[]>(); - for (Map.Entry<String, Map<String, String>> e : - this.bridgeUsersRaw.entrySet()) { - String longKey = e.getKey(); - String[] parts = longKey.split(","); - String hashedBridgeIdentity = parts[0]; - String date = parts[1]; - String time = parts[2]; - if (!this.hashedRelays.contains(hashedBridgeIdentity) && - !this.zeroTwoTwoDescriptors.contains(longKey)) { - double[] users = bridgeUsersPerDay.get(date); - Map<String, String> obs = e.getValue(); - if (users == null) { - users = new double[this.countries.size()]; - bridgeUsersPerDay.put(date, users); - } - int i = 0; - for (String c : this.countries) { - if (obs.containsKey(c) && !obs.get(c).startsWith("-")) { - users[i] += Double.parseDouble(obs.get(c)); - } - i++; - } - } - } - - /* Write final results of bridge users per day and country to - * <code>stats/bridge-stats</code>. */ - try { - this.logger.fine("Writing file " - + this.bridgeStatsRawFile.getAbsolutePath() + "..."); - this.bridgeStatsFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter( - this.bridgeStatsFile)); - bw.append("date"); - for (String c : this.countries) { - if (c.equals("zy")) { - bw.append(",all"); - } else { - bw.append("," + c); - } - } - bw.append("\n"); - - /* Write current observation. */ - for (Map.Entry<String, double[]> e : bridgeUsersPerDay.entrySet()) { - String date = e.getKey(); - bw.append(date); - double[] users = e.getValue(); - for (int i = 0; i < users.length; i++) { - bw.append("," + String.format("%.2f", users[i])); - } - bw.append("\n"); - } - bw.close(); - this.logger.fine("Finished writing file " - + this.bridgeStatsFile.getAbsolutePath() + "."); - } catch (IOException e) { - this.logger.log(Level.WARNING, "Failed to write " - + this.bridgeStatsFile.getAbsolutePath() + "!", e); - } - - /* Add daily bridge users to database. */ - if (connectionURL != null) { - try { - List<String> countryList = new ArrayList<String>(); - for (String c : this.countries) { - countryList.add(c); - } - Map<String, Integer> insertRows = new HashMap<String, Integer>(), - updateRows = new HashMap<String, Integer>(); - for (Map.Entry<String, double[]> e : - bridgeUsersPerDay.entrySet()) { - String date = e.getKey(); - double[] users = e.getValue(); - for (int i = 0; i < users.length; i++) { - int usersInt = (int) users[i]; - if (usersInt < 1) { - continue; - } - String country = countryList.get(i); - String key = date + "," + country; - insertRows.put(key, usersInt); - } - } - Connection conn = DriverManager.getConnection(connectionURL); - conn.setAutoCommit(false); - Statement statement = conn.createStatement(); - ResultSet rs = statement.executeQuery( - "SELECT date, country, users FROM bridge_stats"); - while (rs.next()) { - String date = rs.getDate(1).toString(); - String country = rs.getString(2); - String key = date + "," + country; - if (insertRows.containsKey(key)) { - int insertRow = insertRows.remove(key); - int oldUsers = rs.getInt(3); - if (oldUsers != insertRow) { - updateRows.put(key, insertRow); - } - } - } - rs.close(); - PreparedStatement psU = conn.prepareStatement( - "UPDATE bridge_stats SET users = ? " - + "WHERE date = ? AND country = ?"); - for (Map.Entry<String, Integer> e : updateRows.entrySet()) { - String[] keyParts = e.getKey().split(","); - java.sql.Date date = java.sql.Date.valueOf(keyParts[0]); - String country = keyParts[1]; - int users = e.getValue(); - psU.clearParameters(); - psU.setInt(1, users); - psU.setDate(2, date); - psU.setString(3, country); - psU.executeUpdate(); - } - PreparedStatement psI = conn.prepareStatement( - "INSERT INTO bridge_stats (users, date, country) " - + "VALUES (?, ?, ?)"); - for (Map.Entry<String, Integer> e : insertRows.entrySet()) { - String[] keyParts = e.getKey().split(","); - java.sql.Date date = java.sql.Date.valueOf(keyParts[0]); - String country = keyParts[1]; - int users = e.getValue(); - psI.clearParameters(); - psI.setInt(1, users); - psI.setDate(2, date); - psI.setString(3, country); - psI.executeUpdate(); - } - conn.commit(); - conn.close(); - } catch (SQLException e) { - logger.log(Level.WARNING, "Failed to add daily bridge users to " - + "database.", e); - } - } - } -} - diff --git a/src/org/torproject/ernie/db/Configuration.java b/src/org/torproject/ernie/db/Configuration.java index b8f8f39..3554e23 100644 --- a/src/org/torproject/ernie/db/Configuration.java +++ b/src/org/torproject/ernie/db/Configuration.java @@ -13,8 +13,6 @@ import java.util.logging.*; * configuration. */ public class Configuration { - private boolean writeConsensusStats = false; - private boolean writeBridgeStats = false; private boolean writeDirectoryArchives = false; private String directoryArchivesOutputDirectory = "directory-archive/"; private boolean importCachedRelayDescriptors = false; @@ -30,9 +28,6 @@ public class Configuration { private boolean replaceIPAddressesWithHashes = false; private long limitBridgeDescriptorMappings = -1L; private String sanitizedBridgesWriteDirectory = "sanitized-bridges/"; - private boolean importSanitizedBridges = false; - private String sanitizedBridgesDirectory = "bridges/"; - private boolean keepSanitizedBridgesImportHistory = false; private boolean importBridgeSnapshots = false; private String bridgeSnapshotsDirectory = "bridge-directories/"; private boolean importWriteTorperfStats = false; @@ -67,12 +62,6 @@ public class Configuration { while ((line = br.readLine()) != null) { if (line.startsWith("#") || line.length() < 1) { continue; - } else if (line.startsWith("WriteConsensusStats")) { - this.writeConsensusStats = Integer.parseInt( - line.split(" ")[1]) != 0; - } else if (line.startsWith("WriteBridgeStats")) { - this.writeBridgeStats = Integer.parseInt( - line.split(" ")[1]) != 0; } else if (line.startsWith("WriteDirectoryArchives")) { this.writeDirectoryArchives = Integer.parseInt( line.split(" ")[1]) != 0; @@ -111,14 +100,6 @@ public class Configuration { line.split(" ")[1]); } else if (line.startsWith("SanitizedBridgesWriteDirectory")) { this.sanitizedBridgesWriteDirectory = line.split(" ")[1]; - } else if (line.startsWith("ImportSanitizedBridges")) { - this.importSanitizedBridges = Integer.parseInt( - line.split(" ")[1]) != 0; - } else if (line.startsWith("SanitizedBridgesDirectory")) { - this.sanitizedBridgesDirectory = line.split(" ")[1]; - } else if (line.startsWith("KeepSanitizedBridgesImportHistory")) { - this.keepSanitizedBridgesImportHistory = Integer.parseInt( - line.split(" ")[1]) != 0; } else if (line.startsWith("ImportBridgeSnapshots")) { this.importBridgeSnapshots = Integer.parseInt( line.split(" ")[1]) != 0; @@ -184,13 +165,11 @@ public class Configuration { /** Make some checks if configuration is valid. */ if (!this.importCachedRelayDescriptors && !this.importDirectoryArchives && !this.downloadRelayDescriptors && - !this.importSanitizedBridges && !this.importBridgeSnapshots && - !this.importWriteTorperfStats && + !this.importBridgeSnapshots && !this.importWriteTorperfStats && !this.downloadProcessGetTorStats && !this.downloadExitList && !this.writeDirectoryArchives && !this.writeAggregateStatsDatabase && - !this.writeSanitizedBridges && !this.writeConsensusStats && - !this.writeBridgeStats) { + !this.writeSanitizedBridges) { logger.warning("We have not been configured to read data from any " + "data source or write data to any data sink. You need to " + "edit your config file (" + configFile.getAbsolutePath() @@ -199,8 +178,7 @@ public class Configuration { } if ((this.importCachedRelayDescriptors || this.importDirectoryArchives || this.downloadRelayDescriptors) && - !(this.writeDirectoryArchives || this.writeConsensusStats || - this.writeBridgeStats)) { + !this.writeDirectoryArchives) { logger.warning("We are configured to import/download relay " + "descriptors, but we don't have a single data sink to write " + "relay descriptors to."); @@ -212,23 +190,13 @@ public class Configuration { + "least one data sink, but we don't have a single data source " + "containing relay descriptors."); } - if (!(this.importCachedRelayDescriptors || - this.importDirectoryArchives || this.downloadRelayDescriptors || - this.importSanitizedBridges || this.importBridgeSnapshots) && - (this.writeBridgeStats || this.writeConsensusStats)) { - logger.warning("We are configured to write relay or bridge " - + "descriptors to at least one data sink, but we have neither " - + "data sources containing relay nor bridge descriptors."); - } - if ((this.importSanitizedBridges || this.importBridgeSnapshots) && - !(this.writeSanitizedBridges || this.writeConsensusStats || - this.writeBridgeStats || this.writeAggregateStatsDatabase)) { + if (this.importBridgeSnapshots && !(this.writeSanitizedBridges || + this.writeAggregateStatsDatabase)) { logger.warning("We are configured to import/download bridge " + "descriptors, but we don't have a single data sink to write " + "bridge descriptors to."); } - if (!(this.importSanitizedBridges || this.importBridgeSnapshots) && - (this.writeSanitizedBridges)) { + if (!this.importBridgeSnapshots && this.writeSanitizedBridges) { logger.warning("We are configured to write bridge descriptor to at " + "least one data sink, but we don't have a single data source " + "containing bridge descriptors."); @@ -239,12 +207,6 @@ public class Configuration { + "but not to import them into the database."); } } - public boolean getWriteConsensusStats() { - return this.writeConsensusStats; - } - public boolean getWriteBridgeStats() { - return this.writeBridgeStats; - } public boolean getWriteDirectoryArchives() { return this.writeDirectoryArchives; } @@ -284,15 +246,6 @@ public class Configuration { public String getSanitizedBridgesWriteDirectory() { return this.sanitizedBridgesWriteDirectory; } - public boolean getImportSanitizedBridges() { - return this.importSanitizedBridges; - } - public String getSanitizedBridgesDirectory() { - return this.sanitizedBridgesDirectory; - } - public boolean getKeepSanitizedBridgesImportHistory() { - return this.keepSanitizedBridgesImportHistory; - } public boolean getImportBridgeSnapshots() { return this.importBridgeSnapshots; } diff --git a/src/org/torproject/ernie/db/ConsensusStatsFileHandler.java b/src/org/torproject/ernie/db/ConsensusStatsFileHandler.java deleted file mode 100644 index 81e4e3a..0000000 --- a/src/org/torproject/ernie/db/ConsensusStatsFileHandler.java +++ /dev/null @@ -1,282 +0,0 @@ -/* Copyright 2010 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.ernie.db; - -import java.io.*; -import java.sql.*; -import java.text.*; -import java.util.*; -import java.util.logging.*; - -/** - * Generates statistics on the average number of relays and bridges per - * day. Accepts parse results from <code>RelayDescriptorParser</code> and - * <code>BridgeDescriptorParser</code> and stores them in intermediate - * result files <code>stats/consensus-stats-raw</code> and - * <code>stats/bridge-consensus-stats-raw</code>. Writes final results to - * <code>stats/consensus-stats</code> for all days for which at least half - * of the expected consensuses or statuses are known. - */ -public class ConsensusStatsFileHandler { - - /** - * Intermediate results file holding the number of running bridges per - * bridge status. - */ - private File bridgeConsensusStatsRawFile; - - /** - * Number of running bridges in a given bridge status. Map keys are - * bridge status times formatted as "yyyy-MM-dd HH:mm:ss", map values - * are lines as read from <code>stats/bridge-consensus-stats-raw</code>. - */ - private SortedMap<String, String> bridgesRaw; - - /** - * Average number of running bridges per day. Map keys are dates - * formatted as "yyyy-MM-dd", map values are the last column as written - * to <code>stats/consensus-stats</code>. - */ - private SortedMap<String, String> bridgesPerDay; - - /** - * Logger for this class. - */ - private Logger logger; - - private int bridgeResultsAdded = 0; - - /* Database connection string. */ - private String connectionURL = null; - - /** - * Initializes this class, including reading in intermediate results - * files <code>stats/consensus-stats-raw</code> and - * <code>stats/bridge-consensus-stats-raw</code> and final results file - * <code>stats/consensus-stats</code>. - */ - public ConsensusStatsFileHandler(String connectionURL) { - - /* Initialize local data structures to hold intermediate and final - * results. */ - this.bridgesPerDay = new TreeMap<String, String>(); - this.bridgesRaw = new TreeMap<String, String>(); - - /* Initialize file names for intermediate and final results files. */ - this.bridgeConsensusStatsRawFile = new File( - "stats/bridge-consensus-stats-raw"); - - /* Initialize database connection string. */ - this.connectionURL = connectionURL; - - /* Initialize logger. */ - this.logger = Logger.getLogger( - ConsensusStatsFileHandler.class.getName()); - - /* Read in number of running bridges per bridge status. */ - if (this.bridgeConsensusStatsRawFile.exists()) { - try { - this.logger.fine("Reading file " - + this.bridgeConsensusStatsRawFile.getAbsolutePath() + "..."); - BufferedReader br = new BufferedReader(new FileReader( - this.bridgeConsensusStatsRawFile)); - String line = null; - while ((line = br.readLine()) != null) { - if (line.startsWith("date")) { - /* Skip headers. */ - continue; - } - String[] parts = line.split(","); - if (parts.length != 2) { - this.logger.warning("Corrupt line '" + line + "' in file " - + this.bridgeConsensusStatsRawFile.getAbsolutePath() - + "! Aborting to read this file!"); - break; - } - String dateTime = parts[0]; - this.bridgesRaw.put(dateTime, line); - } - br.close(); - this.logger.fine("Finished reading file " - + this.bridgeConsensusStatsRawFile.getAbsolutePath() + "."); - } catch (IOException e) { - this.logger.log(Level.WARNING, "Failed to read file " - + this.bridgeConsensusStatsRawFile.getAbsolutePath() + "!", - e); - } - } - } - - /** - * Adds the intermediate results of the number of running bridges in a - * given bridge status to the existing observations. - */ - public void addBridgeConsensusResults(String published, int running) { - String line = published + "," + running; - if (!this.bridgesRaw.containsKey(published)) { - this.logger.finer("Adding new bridge numbers: " + line); - this.bridgesRaw.put(published, line); - this.bridgeResultsAdded++; - } else if (!line.equals(this.bridgesRaw.get(published))) { - this.logger.warning("The numbers of running bridges we were just " - + "given (" + line + ") are different from what we learned " - + "before (" + this.bridgesRaw.get(published) + ")! " - + "Overwriting!"); - this.bridgesRaw.put(published, line); - } - } - - /** - * Aggregates the raw observations on relay and bridge numbers and - * writes both raw and aggregate observations to disk. - */ - public void writeFiles() { - - /* Did we learn anything new about average relay or bridge numbers in - * this run? */ - boolean writeConsensusStats = false; - - /* Go through raw observations of numbers of running bridges in bridge - * statuses, calculate averages per day, and add these averages to - * final results. */ - if (!this.bridgesRaw.isEmpty()) { - String tempDate = null; - int brunning = 0, statuses = 0; - Iterator<String> it = this.bridgesRaw.values().iterator(); - boolean haveWrittenFinalLine = false; - while (it.hasNext() || !haveWrittenFinalLine) { - String next = it.hasNext() ? it.next() : null; - /* Finished reading a day or even all lines? */ - if (tempDate != null && (next == null - || !next.substring(0, 10).equals(tempDate))) { - /* Only write results if we have seen at least half of all - * statuses. */ - if (statuses >= 24) { - String line = "," + (brunning / statuses); - /* Are our results new? */ - if (!this.bridgesPerDay.containsKey(tempDate)) { - this.logger.finer("Adding new average bridge numbers: " - + tempDate + line); - this.bridgesPerDay.put(tempDate, line); - writeConsensusStats = true; - } else if (!line.equals(this.bridgesPerDay.get(tempDate))) { - this.logger.finer("Replacing existing average bridge " - + "numbers (" + this.bridgesPerDay.get(tempDate) - + " with new numbers: " + line); - this.bridgesPerDay.put(tempDate, line); - writeConsensusStats = true; - } - } - brunning = statuses = 0; - haveWrittenFinalLine = (next == null); - } - /* Sum up number of running bridges. */ - if (next != null) { - tempDate = next.substring(0, 10); - statuses++; - brunning += Integer.parseInt(next.split(",")[1]); - } - } - } - - /* Write raw numbers of running bridges to disk. */ - try { - this.logger.fine("Writing file " - + this.bridgeConsensusStatsRawFile.getAbsolutePath() + "..."); - this.bridgeConsensusStatsRawFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter( - new FileWriter(this.bridgeConsensusStatsRawFile)); - bw.append("datetime,brunning\n"); - for (String line : this.bridgesRaw.values()) { - bw.append(line + "\n"); - } - bw.close(); - this.logger.fine("Finished writing file " - + this.bridgeConsensusStatsRawFile.getAbsolutePath() + "."); - } catch (IOException e) { - this.logger.log(Level.WARNING, "Failed to write file " - + this.bridgeConsensusStatsRawFile.getAbsolutePath() + "!", - e); - } - - /* Add average number of bridges per day to the database. */ - if (connectionURL != null) { - try { - Map<String, String> insertRows = new HashMap<String, String>(), - updateRows = new HashMap<String, String>(); - insertRows.putAll(this.bridgesPerDay); - Connection conn = DriverManager.getConnection(connectionURL); - conn.setAutoCommit(false); - Statement statement = conn.createStatement(); - ResultSet rs = statement.executeQuery( - "SELECT date, avg_running FROM bridge_network_size"); - while (rs.next()) { - String date = rs.getDate(1).toString(); - if (insertRows.containsKey(date)) { - String insertRow = insertRows.remove(date); - long newAvgRunning = Long.parseLong(insertRow.substring(1)); - long oldAvgRunning = rs.getLong(2); - if (newAvgRunning != oldAvgRunning) { - updateRows.put(date, insertRow); - } - } - } - rs.close(); - PreparedStatement psU = conn.prepareStatement( - "UPDATE bridge_network_size SET avg_running = ? " - + "WHERE date = ?"); - for (Map.Entry<String, String> e : updateRows.entrySet()) { - java.sql.Date date = java.sql.Date.valueOf(e.getKey()); - long avgRunning = Long.parseLong(e.getValue().substring(1)); - psU.clearParameters(); - psU.setLong(1, avgRunning); - psU.setDate(2, date); - psU.executeUpdate(); - } - PreparedStatement psI = conn.prepareStatement( - "INSERT INTO bridge_network_size (avg_running, date) " - + "VALUES (?, ?)"); - for (Map.Entry<String, String> e : insertRows.entrySet()) { - java.sql.Date date = java.sql.Date.valueOf(e.getKey()); - long avgRunning = Long.parseLong(e.getValue().substring(1)); - psI.clearParameters(); - psI.setLong(1, avgRunning); - psI.setDate(2, date); - psI.executeUpdate(); - } - conn.commit(); - conn.close(); - } catch (SQLException e) { - logger.log(Level.WARNING, "Failed to add average bridge numbers " - + "to database.", e); - } - } - - /* Write stats. */ - StringBuilder dumpStats = new StringBuilder("Finished writing " - + "statistics on bridge network statuses to disk.\nAdded " - + this.bridgeResultsAdded + " bridge network status(es) in this " - + "execution."); - long now = System.currentTimeMillis(); - SimpleDateFormat dateTimeFormat = - new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - if (this.bridgesRaw.isEmpty()) { - dumpStats.append("\nNo bridge status known yet."); - } else { - dumpStats.append("\nLast known bridge status was published " - + this.bridgesRaw.lastKey() + "."); - try { - if (now - 6L * 60L * 60L * 1000L > dateTimeFormat.parse( - this.bridgesRaw.lastKey()).getTime()) { - logger.warning("Last known bridge status is more than 6 hours " - + "old: " + this.bridgesRaw.lastKey()); - } - } catch (ParseException e) { - /* Can't parse the timestamp? Whatever. */ - } - } - logger.info(dumpStats.toString()); - } -} - diff --git a/src/org/torproject/ernie/db/Main.java b/src/org/torproject/ernie/db/Main.java index 4dced2c..2dc0c86 100644 --- a/src/org/torproject/ernie/db/Main.java +++ b/src/org/torproject/ernie/db/Main.java @@ -33,12 +33,6 @@ public class Main { // Define stats directory for temporary files File statsDirectory = new File("stats"); - // Prepare bridge stats file handler - BridgeStatsFileHandler bsfh = config.getWriteBridgeStats() ? - new BridgeStatsFileHandler( - config.getWriteAggregateStatsDatabase() ? - config.getRelayDescriptorDatabaseJDBC() : null) : null; - // Prepare writing relay descriptor archive to disk ArchiveWriter aw = config.getWriteDirectoryArchives() ? new ArchiveWriter( @@ -46,9 +40,8 @@ public class Main { // Prepare relay descriptor parser (only if we are writing stats or // directory archives to disk) - RelayDescriptorParser rdp = config.getWriteBridgeStats() || - config.getWriteDirectoryArchives() ? - new RelayDescriptorParser(bsfh, aw) : null; + RelayDescriptorParser rdp = aw != null ? + new RelayDescriptorParser(aw) : null; // Import/download relay descriptors from the various sources if (rdp != null) { @@ -56,7 +49,7 @@ public class Main { if (config.getDownloadRelayDescriptors()) { List<String> dirSources = config.getDownloadFromDirectoryAuthorities(); - boolean downloadCurrentConsensus = aw != null || bsfh != null; + boolean downloadCurrentConsensus = aw != null; boolean downloadCurrentVotes = aw != null; boolean downloadAllServerDescriptors = aw != null; boolean downloadAllExtraInfos = aw != null; @@ -100,12 +93,6 @@ public class Main { aw = null; } - // Prepare consensus stats file handler - ConsensusStatsFileHandler csfh = config.getWriteConsensusStats() ? - new ConsensusStatsFileHandler( - config.getWriteAggregateStatsDatabase() ? - config.getRelayDescriptorDatabaseJDBC() : null) : null; - // Prepare sanitized bridge descriptor writer SanitizedBridgesWriter sbw = config.getWriteSanitizedBridges() ? new SanitizedBridgesWriter( @@ -114,16 +101,10 @@ public class Main { config.getLimitBridgeDescriptorMappings()) : null; // Prepare bridge descriptor parser - BridgeDescriptorParser bdp = config.getWriteConsensusStats() || - config.getWriteBridgeStats() || config.getWriteSanitizedBridges() - ? new BridgeDescriptorParser(csfh, bsfh, sbw) : null; + BridgeDescriptorParser bdp = config.getWriteSanitizedBridges() + ? new BridgeDescriptorParser(sbw) : null; // Import bridge descriptors - if (bdp != null && config.getImportSanitizedBridges()) { - new SanitizedBridgesReader(bdp, - new File(config.getSanitizedBridgesDirectory()), - statsDirectory, config.getKeepSanitizedBridgesImportHistory()); - } if (bdp != null && config.getImportBridgeSnapshots()) { new BridgeSnapshotReader(bdp, new File(config.getBridgeSnapshotsDirectory()), @@ -136,16 +117,6 @@ public class Main { sbw = null; } - // Write updated stats files to disk - if (bsfh != null) { - bsfh.writeFiles(); - bsfh = null; - } - if (csfh != null) { - csfh.writeFiles(); - csfh = null; - } - // Import and process torperf stats if (config.getImportWriteTorperfStats()) { new TorperfProcessor(new File(config.getTorperfDirectory()), diff --git a/src/org/torproject/ernie/db/RelayDescriptorParser.java b/src/org/torproject/ernie/db/RelayDescriptorParser.java index fcfc0eb..bfd0033 100644 --- a/src/org/torproject/ernie/db/RelayDescriptorParser.java +++ b/src/org/torproject/ernie/db/RelayDescriptorParser.java @@ -18,11 +18,6 @@ import org.apache.commons.codec.binary.*; public class RelayDescriptorParser { /** - * Stats file handler that accepts parse results for bridge statistics. - */ - private BridgeStatsFileHandler bsfh; - - /** * File writer that writes descriptor contents to files in a * directory-archive directory structure. */ @@ -44,9 +39,7 @@ public class RelayDescriptorParser { /** * Initializes this class. */ - public RelayDescriptorParser(BridgeStatsFileHandler bsfh, - ArchiveWriter aw) { - this.bsfh = bsfh; + public RelayDescriptorParser(ArchiveWriter aw) { this.aw = aw; /* Initialize logger. */ @@ -121,11 +114,6 @@ public class RelayDescriptorParser { } } if (isConsensus) { - if (this.bsfh != null) { - for (String hashedRelayIdentity : hashedRelayIdentities) { - this.bsfh.addHashedRelay(hashedRelayIdentity); - } - } if (this.rdd != null) { this.rdd.haveParsedConsensus(validAfterTime, dirSources, serverDescriptors); diff --git a/src/org/torproject/ernie/db/SanitizedBridgesReader.java b/src/org/torproject/ernie/db/SanitizedBridgesReader.java deleted file mode 100644 index 2ddf1dd..0000000 --- a/src/org/torproject/ernie/db/SanitizedBridgesReader.java +++ /dev/null @@ -1,114 +0,0 @@ -/* Copyright 2010 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.ernie.db; - -import java.io.*; -import java.util.*; -import java.util.logging.*; - -public class SanitizedBridgesReader { - public SanitizedBridgesReader(BridgeDescriptorParser bdp, - File bridgesDir, File statsDirectory, boolean keepImportHistory) { - - if (bdp == null || bridgesDir == null || statsDirectory == null) { - throw new IllegalArgumentException(); - } - - Logger logger = - Logger.getLogger(SanitizedBridgesReader.class.getName()); - SortedSet<String> bridgesImportHistory = new TreeSet<String>(); - File bridgesImportHistoryFile = - new File(statsDirectory, "bridges-import-history"); - if (keepImportHistory && bridgesImportHistoryFile.exists()) { - try { - BufferedReader br = new BufferedReader(new FileReader( - bridgesImportHistoryFile)); - String line = null; - while ((line = br.readLine()) != null) { - bridgesImportHistory.add(line); - } - br.close(); - } catch (IOException e) { - logger.log(Level.WARNING, "Could not read in bridge descriptor " - + "import history file. Skipping."); - } - } - if (bridgesDir.exists()) { - logger.fine("Importing files in directory " + bridgesDir + "/..."); - Stack<File> filesInInputDir = new Stack<File>(); - filesInInputDir.add(bridgesDir); - List<File> problems = new ArrayList<File>(); - while (!filesInInputDir.isEmpty()) { - File pop = filesInInputDir.pop(); - if (pop.isDirectory()) { - for (File f : pop.listFiles()) { - filesInInputDir.add(f); - } - continue; - } else if (keepImportHistory && bridgesImportHistory.contains( - pop.getName())) { - continue; - } else { - try { - BufferedInputStream bis = new BufferedInputStream( - new FileInputStream(pop)); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - int len; - byte[] data = new byte[1024]; - while ((len = bis.read(data, 0, 1024)) >= 0) { - baos.write(data, 0, len); - } - bis.close(); - byte[] allData = baos.toByteArray(); - String fn = pop.getName(); - // TODO dateTime extraction doesn't work for sanitized network - // statuses! - String dateTime = fn.substring(0, 4) + "-" + fn.substring(4, 6) - + "-" + fn.substring(6, 8) + " " + fn.substring(9, 11) - + ":" + fn.substring(11, 13) + ":" + fn.substring(13, 15); - bdp.parse(allData, dateTime, true); - if (keepImportHistory) { - bridgesImportHistory.add(pop.getName()); - } - } catch (IOException e) { - problems.add(pop); - if (problems.size() > 3) { - break; - } - } - } - } - if (problems.isEmpty()) { - logger.fine("Finished importing files in directory " + bridgesDir - + "/."); - } else { - StringBuilder sb = new StringBuilder("Failed importing files in " - + "directory " + bridgesDir + "/:"); - int printed = 0; - for (File f : problems) { - sb.append("\n " + f.getAbsolutePath()); - if (++printed >= 3) { - sb.append("\n ... more"); - break; - } - } - logger.warning(sb.toString()); - } - if (keepImportHistory) { - try { - bridgesImportHistoryFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter( - bridgesImportHistoryFile)); - for (String line : bridgesImportHistory) { - bw.write(line + "\n"); - } - bw.close(); - } catch (IOException e) { - logger.log(Level.WARNING, "Could not write bridge descriptor " - + "import history file."); - } - } - } - } -} - diff --git a/src/org/torproject/ernie/test/SanitizedBridgesReaderTest.java b/src/org/torproject/ernie/test/SanitizedBridgesReaderTest.java deleted file mode 100644 index dd5f31e..0000000 --- a/src/org/torproject/ernie/test/SanitizedBridgesReaderTest.java +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright 2011 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.ernie.test; - -import org.torproject.ernie.db.*; - -import java.io.*; - -import org.junit.*; -import org.junit.rules.*; -import static org.junit.Assert.*; - -public class SanitizedBridgesReaderTest { - - private File tempSanitizedBridgesDirectory; - private File tempStatsDirectory; - - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - @Before - public void createTempDirectories() { - this.tempSanitizedBridgesDirectory = folder.newFolder("bridges"); - this.tempStatsDirectory = folder.newFolder("stats"); - } - - @Test(expected = IllegalArgumentException.class) - public void testBridgeDescriptorParserNull() { - new SanitizedBridgesReader(null, this.tempSanitizedBridgesDirectory, - this.tempStatsDirectory, false); - } -} -