- tor-commits - lists.torproject.org

[metrics-web/master] Prepare bridge statistics as part of metrics-web.
by karsten＠torproject.org 03 Mar '11

03 Mar '11

commit d547c8ecbadca64b046a041d94e80dddbeecc8c3 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Mar 2 20:31:42 2011 +0100 Prepare bridge statistics as part of metrics-web. --- build.xml | 1 + config.template | 15 + lib/junit-4.8.2.jar | Bin 0 -> 237344 bytes .../ernie/cron/BridgeDescriptorParser.java | 141 ++++++ .../ernie/cron/BridgeStatsFileHandler.java | 519 ++++++++++++++++++++ src/org/torproject/ernie/cron/Configuration.java | 27 + .../ernie/cron/ConsensusStatsFileHandler.java | 282 +++++++++++ src/org/torproject/ernie/cron/Main.java | 34 ++- .../ernie/cron/RelayDescriptorParser.java | 17 +- .../ernie/cron/SanitizedBridgesReader.java | 114 +++++ .../ernie/test/SanitizedBridgesReaderTest.java | 33 ++ 11 files changed, 1181 insertions(+), 2 deletions(-) diff --git a/build.xml b/build.xml index 446572b..cdae4e1 100644 --- a/build.xml +++ b/build.xml @@ -14,6 +14,7 @@ <pathelement path="${classes}"/> <pathelement location="lib/commons-codec-1.4.jar"/> <pathelement location="lib/postgresql-8.4-702.jdbc3.jar"/> + <pathelement location="lib/junit-4.8.2.jar"/> </path> <target name="init"> diff --git a/config.template b/config.template index 1d0701c..f2dc9cb 100644 --- a/config.template +++ b/config.template @@ -10,6 +10,18 @@ ## again, but it can be confusing to users who don't know about it. #KeepDirectoryArchiveImportHistory 0 # +## Import sanitized bridges from disk, if available +#ImportSanitizedBridges 0 +# +## Relative path to directory to import sanitized bridges from +#SanitizedBridgesDirectory bridges/ +# +## Keep a history of imported sanitized bridge descriptors. This history +## can be useful when importing from a changing data source to avoid +## importing descriptors more than once, but it can be confusing to users +## who don't know about it. +#KeepSanitizedBridgesImportHistory 0 +# ## Write relay descriptors to the database #WriteRelayDescriptorDatabase 0 # @@ -27,4 +39,7 @@ ## Write statistics about the current consensus and votes to the ## website #WriteConsensusHealth 0 +# +## Write bridge stats to disk +#WriteBridgeStats 0 diff --git a/lib/junit-4.8.2.jar b/lib/junit-4.8.2.jar new file mode 100644 index 0000000..5b4bb84 Binary files /dev/null and b/lib/junit-4.8.2.jar differ diff --git a/src/org/torproject/ernie/cron/BridgeDescriptorParser.java b/src/org/torproject/ernie/cron/BridgeDescriptorParser.java new file mode 100644 index 0000000..4777f58 --- /dev/null +++ b/src/org/torproject/ernie/cron/BridgeDescriptorParser.java @@ -0,0 +1,141 @@ +/* Copyright 2011 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.ernie.cron; + +import java.io.*; +import java.text.*; +import java.util.*; +import java.util.logging.*; +import org.apache.commons.codec.digest.*; + +public class BridgeDescriptorParser { + private ConsensusStatsFileHandler csfh; + private BridgeStatsFileHandler bsfh; + private Logger logger; + public BridgeDescriptorParser(ConsensusStatsFileHandler csfh, + BridgeStatsFileHandler bsfh) { + this.csfh = csfh; + this.bsfh = bsfh; + this.logger = + Logger.getLogger(BridgeDescriptorParser.class.getName()); + } + public void parse(byte[] allData, String dateTime, boolean sanitized) { + try { + BufferedReader br = new BufferedReader(new StringReader( + new String(allData, "US-ASCII"))); + SimpleDateFormat timeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + timeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + String hashedIdentity = null, platformLine = null, + publishedLine = null, geoipStartTimeLine = null, + bridgeStatsEndLine = null; + boolean skip = false; + String line = null; + while ((line = br.readLine()) != null) { + if (line.startsWith("r ")) { + int runningBridges = 0; + while ((line = br.readLine()) != null) { + if (line.startsWith("s ") && line.contains(" Running")) { + runningBridges++; + } + } + if (this.csfh != null) { + this.csfh.addBridgeConsensusResults(dateTime, runningBridges); + } + } else if (line.startsWith("router ")) { + } else if (line.startsWith("extra-info ")) { + hashedIdentity = sanitized ? line.split(" ")[2] + : DigestUtils.shaHex(line.split(" ")[2]).toUpperCase(); + if (this.bsfh != null) { + skip = this.bsfh.isKnownRelay(hashedIdentity); + } + } else if (!skip && line.startsWith("platform ")) { + platformLine = line; + } else if (!skip && line.startsWith("published ")) { + publishedLine = line; + } else if (line.startsWith("opt fingerprint") || + line.startsWith("fingerprint")) { + String identity = line.substring(line.startsWith("opt ") ? + "opt fingerprint".length() : "fingerprint".length()). + replaceAll(" ", "").toLowerCase(); + hashedIdentity = sanitized ? identity + : DigestUtils.shaHex(identity).toUpperCase(); + } else if (!skip && line.startsWith("geoip-start-time ")) { + geoipStartTimeLine = line; + } else if (!skip && line.startsWith("geoip-client-origins") + && line.split(" ").length > 1) { + if (publishedLine == null || + geoipStartTimeLine == null) { + this.logger.warning("Either published line or " + + "geoip-start-time line is not present in " + + (sanitized ? "sanitized" : "non-sanitized") + + " bridge descriptors from " + dateTime + "."); + break; + } + long published = timeFormat.parse(publishedLine. + substring("published ".length())).getTime(); + long started = timeFormat.parse(geoipStartTimeLine. + substring("geoip-start-time ".length())).getTime(); + long seconds = (published - started) / 1000L; + double allUsers = 0.0D; + Map<String, String> obs = new HashMap<String, String>(); + String[] parts = line.split(" ")[1].split(","); + for (String p : parts) { + String country = p.substring(0, 2); + double users = ((double) Long.parseLong(p.substring(3)) - 4L) + * 86400.0D / ((double) seconds); + allUsers += users; + obs.put(country, String.format("%.2f", users)); + } + obs.put("zy", String.format("%.2f", allUsers)); + String date = publishedLine.split(" ")[1]; + String time = publishedLine.split(" ")[2]; + if (this.bsfh != null) { + this.bsfh.addObs(hashedIdentity, date, time, obs); + } + } else if (!skip && line.startsWith("bridge-stats-end ")) { + bridgeStatsEndLine = line; + } else if (!skip && line.startsWith("bridge-ips") + && line.split(" ").length > 1) { + if (bridgeStatsEndLine == null) { + this.logger.warning("bridge-ips line without preceding " + + "bridge-stats-end line in " + + (sanitized ? "sanitized" : "non-sanitized") + + " bridge descriptor."); + break; + } + double allUsers = 0.0D; + Map<String, String> obs = new HashMap<String, String>(); + String[] parts = line.split(" ")[1].split(","); + for (String p : parts) { + String country = p.substring(0, 2); + double users = (double) Long.parseLong(p.substring(3)) - 4L; + allUsers += users; + obs.put(country, String.format("%.2f", users)); + } + obs.put("zy", String.format("%.2f", allUsers)); + String date = bridgeStatsEndLine.split(" ")[1]; + String time = bridgeStatsEndLine.split(" ")[2]; + if (this.bsfh != null) { + this.bsfh.addObs(hashedIdentity, date, time, obs); + } + } + } + if (this.bsfh != null && platformLine != null && + platformLine.startsWith("platform Tor 0.2.2")) { + String date = publishedLine.split(" ")[1]; + String time = publishedLine.split(" ")[2]; + this.bsfh.addZeroTwoTwoDescriptor(hashedIdentity, date, time); + } + } catch (IOException e) { + this.logger.log(Level.WARNING, "Could not parse bridge descriptor.", + e); + return; + } catch (ParseException e) { + this.logger.log(Level.WARNING, "Could not parse bridge descriptor.", + e); + return; + } + } +} + diff --git a/src/org/torproject/ernie/cron/BridgeStatsFileHandler.java b/src/org/torproject/ernie/cron/BridgeStatsFileHandler.java new file mode 100644 index 0000000..58ee3a2 --- /dev/null +++ b/src/org/torproject/ernie/cron/BridgeStatsFileHandler.java @@ -0,0 +1,519 @@ +/* Copyright 2011 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.ernie.cron; + +import java.io.*; +import java.sql.*; +import java.text.*; +import java.util.*; +import java.util.logging.*; + +/** + * Determines estimates of bridge users per country and day from the + * extra-info descriptors that bridges publish. In a first step, the + * number of unique IP addresses that bridges see are normalized to a + * 24-hour period. In the next step, all bridges are excluded that have + * been running as a relay. Finally, observations are summed up and + * written to <code>stats/bridge-stats</code>. + */ +public class BridgeStatsFileHandler { + + /** + * Two-letter country codes of known countries. + */ + private SortedSet<String> countries; + + /** + * Intermediate results file containing bridge user numbers by country + * as seen by single bridges, normalized to 24-hour periods. + */ + private File bridgeStatsRawFile; + + /** + * Bridge user numbers by country as seen by single bridges on a given + * day. Map keys are bridge and date written as "bridge,date", map + * values are lines as read from <code>stats/bridge-stats-raw</code>. + */ + private SortedMap<String, Map<String, String>> bridgeUsersRaw; + + /** + * Helper file containing the hashed relay identities of all known + * relays. These hashes are compared to the bridge identity hashes to + * exclude bridges that have been known as relays from the statistics. + */ + private File hashedRelayIdentitiesFile; + + /** + * Known hashed relay identities used to exclude bridges that have been + * running as relays. + */ + private SortedSet<String> hashedRelays; + + /** + * Helper file containing extra-info descriptors published by 0.2.2.x + * bridges. If these descriptors contain geoip-stats, they are not + * included in the results, because stats are very likely broken. + */ + private File zeroTwoTwoDescriptorsFile; + + /** + * Extra-info descriptors published by 0.2.2.x bridges. If these + * descriptors contain geoip-stats, they are not included in the + * results, because stats are very likely broken. + */ + private SortedSet<String> zeroTwoTwoDescriptors; + + /** + * Final results file containing the number of bridge users per country + * and day. This file is not read in during initialization, but + * overwritten at the end of the execution. + */ + private File bridgeStatsFile; + + /** + * Logger for this class. + */ + private Logger logger; + + /* Database connection string. */ + private String connectionURL = null; + + /** + * Initializes this class, including reading in intermediate results + * files <code>stats/bridge-stats-raw</code> and + * <code>stats/hashed-relay-identities</code>. + */ + public BridgeStatsFileHandler(String connectionURL) { + + /* Initialize set of known countries. */ + this.countries = new TreeSet<String>(); + this.countries.add("zy"); + + /* Initialize local data structures to hold results. */ + this.bridgeUsersRaw = new TreeMap<String, Map<String, String>>(); + this.hashedRelays = new TreeSet<String>(); + this.zeroTwoTwoDescriptors = new TreeSet<String>(); + + /* Initialize file names for intermediate and final results. */ + this.bridgeStatsRawFile = new File("stats/bridge-stats-raw"); + this.bridgeStatsFile = new File("stats/bridge-stats"); + this.hashedRelayIdentitiesFile = new File( + "stats/hashed-relay-identities"); + this.zeroTwoTwoDescriptorsFile = new File( + "stats/v022-bridge-descriptors"); + + /* Initialize database connection string. */ + this.connectionURL = connectionURL; + + /* Initialize logger. */ + this.logger = Logger.getLogger( + BridgeStatsFileHandler.class.getName()); + + /* Read in bridge user numbers by country as seen by single bridges, + * normalized to 24-hour periods. */ + if (this.bridgeStatsRawFile.exists()) { + try { + this.logger.fine("Reading file " + + this.bridgeStatsRawFile.getAbsolutePath() + "..."); + BufferedReader br = new BufferedReader(new FileReader( + this.bridgeStatsRawFile)); + String line = br.readLine(); + if (line != null) { + /* The first line should contain headers that we need to parse + * in order to learn what countries we were interested in when + * writing this file. */ + if (!line.startsWith("bridge,date,time,")) { + this.logger.warning("Incorrect first line '" + line + "' in " + + this.bridgeStatsRawFile.getAbsolutePath() + "! This line " + + "should contain headers! Aborting to read in this " + + "file!"); + } else { + String[] headers = line.split(","); + for (int i = 3; i < headers.length; i++) { + if (!headers[i].equals("all")) { + this.countries.add(headers[i]); + } + } + /* Read in the rest of the file. */ + while ((line = br.readLine()) != null) { + String[] parts = line.split(","); + if (parts.length != headers.length) { + this.logger.warning("Corrupt line '" + line + "' in file " + + this.bridgeStatsRawFile.getAbsolutePath() + + "! Aborting to read this file!"); + break; + } + String hashedBridgeIdentity = parts[0]; + String date = parts[1]; + String time = parts[2]; + SortedMap<String, String> obs = + new TreeMap<String, String>(); + for (int i = 3; i < parts.length; i++) { + if (parts[i].equals("NA")) { + continue; + } + if (headers[i].equals("all")) { + obs.put("zy", parts[i]); + } else { + obs.put(headers[i], parts[i]); + } + } + this.addObs(hashedBridgeIdentity, date, time, obs); + } + } + } + br.close(); + this.logger.fine("Finished reading file " + + this.bridgeStatsRawFile.getAbsolutePath() + "."); + } catch (IOException e) { + this.logger.log(Level.WARNING, "Failed to read file " + + this.bridgeStatsRawFile.getAbsolutePath() + "!", e); + } + } + + /* Read in known hashed relay identities used to exclude bridges that + * have been running as relays. */ + if (this.hashedRelayIdentitiesFile.exists()) { + try { + this.logger.fine("Reading file " + + this.hashedRelayIdentitiesFile.getAbsolutePath() + "..."); + BufferedReader br = new BufferedReader(new FileReader( + this.hashedRelayIdentitiesFile)); + String line = null; + /* Read in all lines from the file and memorize them. */ + while ((line = br.readLine()) != null) { + this.hashedRelays.add(line); + } + br.close(); + this.logger.fine("Finished reading file " + + this.hashedRelayIdentitiesFile.getAbsolutePath() + "."); + } catch (IOException e) { + this.logger.log(Level.WARNING, "Failed to read file " + + this.hashedRelayIdentitiesFile.getAbsolutePath() + "!", e); + } + } + + /* Read in known extra-info descriptors published by 0.2.2.x + * bridges. */ + if (this.zeroTwoTwoDescriptorsFile.exists()) { + try { + this.logger.fine("Reading file " + + this.zeroTwoTwoDescriptorsFile.getAbsolutePath() + "..."); + BufferedReader br = new BufferedReader(new FileReader( + this.zeroTwoTwoDescriptorsFile)); + String line = null; + /* Read in all lines from the file and memorize them. */ + while ((line = br.readLine()) != null) { + this.zeroTwoTwoDescriptors.add(line); + } + br.close(); + this.logger.fine("Finished reading file " + + this.zeroTwoTwoDescriptorsFile.getAbsolutePath() + "."); + } catch (IOException e) { + this.logger.log(Level.WARNING, "Failed to read file " + + this.zeroTwoTwoDescriptorsFile.getAbsolutePath() + "!", e); + } + } + } + + /** + * Adds a hashed relay identity string to the list of bridges that we + * are going to ignore in the future. If we counted user numbers from + * bridges that have been running as relays, our numbers would be far + * higher than what we think is correct. + */ + public void addHashedRelay(String hashedRelayIdentity) { + if (!this.hashedRelays.contains(hashedRelayIdentity)) { + this.logger.finer("Adding new hashed relay identity: " + + hashedRelayIdentity); + this.hashedRelays.add(hashedRelayIdentity); + } + } + + /** + * Adds an extra-info descriptor identifier published by an 0.2.2.x + * bridges. If this extra-info descriptor contains geoip-stats, they are + * not included in the results, because stats are very likely broken. + */ + public void addZeroTwoTwoDescriptor(String hashedBridgeIdentity, + String date, String time) { + String value = hashedBridgeIdentity.toUpperCase() + "," + date + "," + + time; + if (!this.zeroTwoTwoDescriptors.contains(value)) { + this.logger.finer("Adding new bridge 0.2.2.x extra-info " + + "descriptor: " + value); + this.zeroTwoTwoDescriptors.add(value); + } + } + + /** + * Returns whether the given fingerprint is a known hashed relay + * identity. <code>BridgeDescriptorParser</code> uses this information + * to decide whether to continue parsing a bridge extra-descriptor + * descriptor or not. + */ + public boolean isKnownRelay(String hashedBridgeIdentity) { + return this.hashedRelays.contains(hashedBridgeIdentity); + } + + /** + * Adds bridge user numbers by country as seen by a single bridge on a + * given date and time. Bridges can publish statistics on unique IP + * addresses multiple times a day, but we only want to include one + * observation per day. If we already have an observation from the given + * bridge and day, we keep the one with the later publication time and + * discard the other one. + */ + public void addObs(String hashedIdentity, String date, String time, + Map<String, String> obs) { + for (String country : obs.keySet()) { + this.countries.add(country); + } + String shortKey = hashedIdentity + "," + date; + String longKey = shortKey + "," + time; + SortedMap<String, Map<String, String>> tailMap = + this.bridgeUsersRaw.tailMap(shortKey); + String nextKey = tailMap.isEmpty() ? null : tailMap.firstKey(); + if (nextKey == null || !nextKey.startsWith(shortKey)) { + this.logger.finer("Adding new bridge user numbers for key " + + longKey); + this.bridgeUsersRaw.put(longKey, obs); + } else if (longKey.compareTo(nextKey) > 0) { + this.logger.finer("Replacing existing bridge user numbers (" + + nextKey + " with new numbers: " + longKey); + this.bridgeUsersRaw.put(longKey, obs); + } else { + this.logger.finer("Not replacing existing bridge user numbers (" + + nextKey + " with new numbers (" + longKey + ")."); + } + } + + /** + * Writes the list of hashed relay identities and bridge user numbers as + * observed by single bridges to disk, aggregates per-day statistics for + * all bridges, and writes those to disk, too. + */ + public void writeFiles() { + + /* Write hashed relay identities to disk. */ + try { + this.logger.fine("Writing file " + + this.hashedRelayIdentitiesFile.getAbsolutePath() + "..."); + this.hashedRelayIdentitiesFile.getParentFile().mkdirs(); + BufferedWriter bw = new BufferedWriter(new FileWriter( + this.hashedRelayIdentitiesFile)); + for (String hashedRelay : this.hashedRelays) { + bw.append(hashedRelay + "\n"); + } + bw.close(); + this.logger.fine("Finished writing file " + + this.hashedRelayIdentitiesFile.getAbsolutePath() + "."); + } catch (IOException e) { + this.logger.log(Level.WARNING, "Failed to write " + + this.hashedRelayIdentitiesFile.getAbsolutePath() + "!", e); + } + + /* Write bridge extra-info descriptor identifiers to disk. */ + try { + this.logger.fine("Writing file " + + this.zeroTwoTwoDescriptorsFile.getAbsolutePath() + "..."); + this.zeroTwoTwoDescriptorsFile.getParentFile().mkdirs(); + BufferedWriter bw = new BufferedWriter(new FileWriter( + this.zeroTwoTwoDescriptorsFile)); + for (String descriptorIdentifier : this.zeroTwoTwoDescriptors) { + bw.append(descriptorIdentifier + "\n"); + } + bw.close(); + this.logger.fine("Finished writing file " + + this.zeroTwoTwoDescriptorsFile.getAbsolutePath() + "."); + } catch (IOException e) { + this.logger.log(Level.WARNING, "Failed to write " + + this.zeroTwoTwoDescriptorsFile.getAbsolutePath() + "!", e); + } + + /* Write observations made by single bridges to disk. */ + try { + this.logger.fine("Writing file " + + this.bridgeStatsRawFile.getAbsolutePath() + "..."); + this.bridgeStatsRawFile.getParentFile().mkdirs(); + BufferedWriter bw = new BufferedWriter(new FileWriter( + this.bridgeStatsRawFile)); + bw.append("bridge,date,time"); + for (String c : this.countries) { + if (c.equals("zy")) { + bw.append(",all"); + } else { + bw.append("," + c); + } + } + bw.append("\n"); + for (Map.Entry<String, Map<String, String>> e : + this.bridgeUsersRaw.entrySet()) { + String longKey = e.getKey(); + String[] parts = longKey.split(","); + String hashedBridgeIdentity = parts[0]; + String date = parts[1]; + String time = parts[2]; + if (!this.hashedRelays.contains(hashedBridgeIdentity) && + !this.zeroTwoTwoDescriptors.contains(longKey)) { + Map<String, String> obs = e.getValue(); + StringBuilder sb = new StringBuilder(longKey); + for (String c : this.countries) { + sb.append("," + (obs.containsKey(c) && + !obs.get(c).startsWith("-") ? obs.get(c) : "NA")); + } + String line = sb.toString(); + bw.append(line + "\n"); + } + } + bw.close(); + this.logger.fine("Finished writing file " + + this.bridgeStatsRawFile.getAbsolutePath() + "."); + } catch (IOException e) { + this.logger.log(Level.WARNING, "Failed to write " + + this.bridgeStatsRawFile.getAbsolutePath() + "!", e); + } + + /* Aggregate per-day statistics. */ + SortedMap<String, double[]> bridgeUsersPerDay = + new TreeMap<String, double[]>(); + for (Map.Entry<String, Map<String, String>> e : + this.bridgeUsersRaw.entrySet()) { + String longKey = e.getKey(); + String[] parts = longKey.split(","); + String hashedBridgeIdentity = parts[0]; + String date = parts[1]; + String time = parts[2]; + if (!this.hashedRelays.contains(hashedBridgeIdentity) && + !this.zeroTwoTwoDescriptors.contains(longKey)) { + double[] users = bridgeUsersPerDay.get(date); + Map<String, String> obs = e.getValue(); + if (users == null) { + users = new double[this.countries.size()]; + bridgeUsersPerDay.put(date, users); + } + int i = 0; + for (String c : this.countries) { + if (obs.containsKey(c) && !obs.get(c).startsWith("-")) { + users[i] += Double.parseDouble(obs.get(c)); + } + i++; + } + } + } + + /* Write final results of bridge users per day and country to + * <code>stats/bridge-stats</code>. */ + try { + this.logger.fine("Writing file " + + this.bridgeStatsRawFile.getAbsolutePath() + "..."); + this.bridgeStatsFile.getParentFile().mkdirs(); + BufferedWriter bw = new BufferedWriter(new FileWriter( + this.bridgeStatsFile)); + bw.append("date"); + for (String c : this.countries) { + if (c.equals("zy")) { + bw.append(",all"); + } else { + bw.append("," + c); + } + } + bw.append("\n"); + + /* Write current observation. */ + for (Map.Entry<String, double[]> e : bridgeUsersPerDay.entrySet()) { + String date = e.getKey(); + bw.append(date); + double[] users = e.getValue(); + for (int i = 0; i < users.length; i++) { + bw.append("," + String.format("%.2f", users[i])); + } + bw.append("\n"); + } + bw.close(); + this.logger.fine("Finished writing file " + + this.bridgeStatsFile.getAbsolutePath() + "."); + } catch (IOException e) { + this.logger.log(Level.WARNING, "Failed to write " + + this.bridgeStatsFile.getAbsolutePath() + "!", e); + } + + /* Add daily bridge users to database. */ + if (connectionURL != null) { + try { + List<String> countryList = new ArrayList<String>(); + for (String c : this.countries) { + countryList.add(c); + } + Map<String, Integer> insertRows = new HashMap<String, Integer>(), + updateRows = new HashMap<String, Integer>(); + for (Map.Entry<String, double[]> e : + bridgeUsersPerDay.entrySet()) { + String date = e.getKey(); + double[] users = e.getValue(); + for (int i = 0; i < users.length; i++) { + int usersInt = (int) users[i]; + if (usersInt < 1) { + continue; + } + String country = countryList.get(i); + String key = date + "," + country; + insertRows.put(key, usersInt); + } + } + Connection conn = DriverManager.getConnection(connectionURL); + conn.setAutoCommit(false); + Statement statement = conn.createStatement(); + ResultSet rs = statement.executeQuery( + "SELECT date, country, users FROM bridge_stats"); + while (rs.next()) { + String date = rs.getDate(1).toString(); + String country = rs.getString(2); + String key = date + "," + country; + if (insertRows.containsKey(key)) { + int insertRow = insertRows.remove(key); + int oldUsers = rs.getInt(3); + if (oldUsers != insertRow) { + updateRows.put(key, insertRow); + } + } + } + rs.close(); + PreparedStatement psU = conn.prepareStatement( + "UPDATE bridge_stats SET users = ? " + + "WHERE date = ? AND country = ?"); + for (Map.Entry<String, Integer> e : updateRows.entrySet()) { + String[] keyParts = e.getKey().split(","); + java.sql.Date date = java.sql.Date.valueOf(keyParts[0]); + String country = keyParts[1]; + int users = e.getValue(); + psU.clearParameters(); + psU.setInt(1, users); + psU.setDate(2, date); + psU.setString(3, country); + psU.executeUpdate(); + } + PreparedStatement psI = conn.prepareStatement( + "INSERT INTO bridge_stats (users, date, country) " + + "VALUES (?, ?, ?)"); + for (Map.Entry<String, Integer> e : insertRows.entrySet()) { + String[] keyParts = e.getKey().split(","); + java.sql.Date date = java.sql.Date.valueOf(keyParts[0]); + String country = keyParts[1]; + int users = e.getValue(); + psI.clearParameters(); + psI.setInt(1, users); + psI.setDate(2, date); + psI.setString(3, country); + psI.executeUpdate(); + } + conn.commit(); + conn.close(); + } catch (SQLException e) { + logger.log(Level.WARNING, "Failed to add daily bridge users to " + + "database.", e); + } + } + } +} + diff --git a/src/org/torproject/ernie/cron/Configuration.java b/src/org/torproject/ernie/cron/Configuration.java index 66ad778..818f9e9 100644 --- a/src/org/torproject/ernie/cron/Configuration.java +++ b/src/org/torproject/ernie/cron/Configuration.java @@ -16,12 +16,16 @@ public class Configuration { private boolean importDirectoryArchives = false; private String directoryArchivesDirectory = "archives/"; private boolean keepDirectoryArchiveImportHistory = false; + private boolean importSanitizedBridges = false; + private String sanitizedBridgesDirectory = "bridges/"; + private boolean keepSanitizedBridgesImportHistory = false; private boolean writeRelayDescriptorDatabase = false; private String relayDescriptorDatabaseJdbc = "jdbc:postgresql://localhost/tordir?user=metrics&password=password"; private boolean writeRelayDescriptorsRawFiles = false; private String relayDescriptorRawFilesDirectory = "pg-import/"; private boolean writeConsensusHealth = false; + private boolean writeBridgeStats = false; public Configuration() { /* Initialize logger. */ @@ -47,6 +51,14 @@ public class Configuration { } else if (line.startsWith("KeepDirectoryArchiveImportHistory")) { this.keepDirectoryArchiveImportHistory = Integer.parseInt( line.split(" ")[1]) != 0; + } else if (line.startsWith("ImportSanitizedBridges")) { + this.importSanitizedBridges = Integer.parseInt( + line.split(" ")[1]) != 0; + } else if (line.startsWith("SanitizedBridgesDirectory")) { + this.sanitizedBridgesDirectory = line.split(" ")[1]; + } else if (line.startsWith("KeepSanitizedBridgesImportHistory")) { + this.keepSanitizedBridgesImportHistory = Integer.parseInt( + line.split(" ")[1]) != 0; } else if (line.startsWith("WriteRelayDescriptorDatabase")) { this.writeRelayDescriptorDatabase = Integer.parseInt( line.split(" ")[1]) != 0; @@ -60,6 +72,9 @@ public class Configuration { } else if (line.startsWith("WriteConsensusHealth")) { this.writeConsensusHealth = Integer.parseInt( line.split(" ")[1]) != 0; + } else if (line.startsWith("WriteBridgeStats")) { + this.writeBridgeStats = Integer.parseInt( + line.split(" ")[1]) != 0; } else { logger.severe("Configuration file contains unrecognized " + "configuration key in line '" + line + "'! Exiting!"); @@ -97,6 +112,15 @@ public class Configuration { public boolean getWriteRelayDescriptorDatabase() { return this.writeRelayDescriptorDatabase; } + public boolean getImportSanitizedBridges() { + return this.importSanitizedBridges; + } + public String getSanitizedBridgesDirectory() { + return this.sanitizedBridgesDirectory; + } + public boolean getKeepSanitizedBridgesImportHistory() { + return this.keepSanitizedBridgesImportHistory; + } public String getRelayDescriptorDatabaseJDBC() { return this.relayDescriptorDatabaseJdbc; } @@ -109,5 +133,8 @@ public class Configuration { public boolean getWriteConsensusHealth() { return this.writeConsensusHealth; } + public boolean getWriteBridgeStats() { + return this.writeBridgeStats; + } } diff --git a/src/org/torproject/ernie/cron/ConsensusStatsFileHandler.java b/src/org/torproject/ernie/cron/ConsensusStatsFileHandler.java new file mode 100644 index 0000000..4ad5300 --- /dev/null +++ b/src/org/torproject/ernie/cron/ConsensusStatsFileHandler.java @@ -0,0 +1,282 @@ +/* Copyright 2011 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.ernie.cron; + +import java.io.*; +import java.sql.*; +import java.text.*; +import java.util.*; +import java.util.logging.*; + +/** + * Generates statistics on the average number of relays and bridges per + * day. Accepts parse results from <code>RelayDescriptorParser</code> and + * <code>BridgeDescriptorParser</code> and stores them in intermediate + * result files <code>stats/consensus-stats-raw</code> and + * <code>stats/bridge-consensus-stats-raw</code>. Writes final results to + * <code>stats/consensus-stats</code> for all days for which at least half + * of the expected consensuses or statuses are known. + */ +public class ConsensusStatsFileHandler { + + /** + * Intermediate results file holding the number of running bridges per + * bridge status. + */ + private File bridgeConsensusStatsRawFile; + + /** + * Number of running bridges in a given bridge status. Map keys are + * bridge status times formatted as "yyyy-MM-dd HH:mm:ss", map values + * are lines as read from <code>stats/bridge-consensus-stats-raw</code>. + */ + private SortedMap<String, String> bridgesRaw; + + /** + * Average number of running bridges per day. Map keys are dates + * formatted as "yyyy-MM-dd", map values are the last column as written + * to <code>stats/consensus-stats</code>. + */ + private SortedMap<String, String> bridgesPerDay; + + /** + * Logger for this class. + */ + private Logger logger; + + private int bridgeResultsAdded = 0; + + /* Database connection string. */ + private String connectionURL = null; + + /** + * Initializes this class, including reading in intermediate results + * files <code>stats/consensus-stats-raw</code> and + * <code>stats/bridge-consensus-stats-raw</code> and final results file + * <code>stats/consensus-stats</code>. + */ + public ConsensusStatsFileHandler(String connectionURL) { + + /* Initialize local data structures to hold intermediate and final + * results. */ + this.bridgesPerDay = new TreeMap<String, String>(); + this.bridgesRaw = new TreeMap<String, String>(); + + /* Initialize file names for intermediate and final results files. */ + this.bridgeConsensusStatsRawFile = new File( + "stats/bridge-consensus-stats-raw"); + + /* Initialize database connection string. */ + this.connectionURL = connectionURL; + + /* Initialize logger. */ + this.logger = Logger.getLogger( + ConsensusStatsFileHandler.class.getName()); + + /* Read in number of running bridges per bridge status. */ + if (this.bridgeConsensusStatsRawFile.exists()) { + try { + this.logger.fine("Reading file " + + this.bridgeConsensusStatsRawFile.getAbsolutePath() + "..."); + BufferedReader br = new BufferedReader(new FileReader( + this.bridgeConsensusStatsRawFile)); + String line = null; + while ((line = br.readLine()) != null) { + if (line.startsWith("date")) { + /* Skip headers. */ + continue; + } + String[] parts = line.split(","); + if (parts.length != 2) { + this.logger.warning("Corrupt line '" + line + "' in file " + + this.bridgeConsensusStatsRawFile.getAbsolutePath() + + "! Aborting to read this file!"); + break; + } + String dateTime = parts[0]; + this.bridgesRaw.put(dateTime, line); + } + br.close(); + this.logger.fine("Finished reading file " + + this.bridgeConsensusStatsRawFile.getAbsolutePath() + "."); + } catch (IOException e) { + this.logger.log(Level.WARNING, "Failed to read file " + + this.bridgeConsensusStatsRawFile.getAbsolutePath() + "!", + e); + } + } + } + + /** + * Adds the intermediate results of the number of running bridges in a + * given bridge status to the existing observations. + */ + public void addBridgeConsensusResults(String published, int running) { + String line = published + "," + running; + if (!this.bridgesRaw.containsKey(published)) { + this.logger.finer("Adding new bridge numbers: " + line); + this.bridgesRaw.put(published, line); + this.bridgeResultsAdded++; + } else if (!line.equals(this.bridgesRaw.get(published))) { + this.logger.warning("The numbers of running bridges we were just " + + "given (" + line + ") are different from what we learned " + + "before (" + this.bridgesRaw.get(published) + ")! " + + "Overwriting!"); + this.bridgesRaw.put(published, line); + } + } + + /** + * Aggregates the raw observations on relay and bridge numbers and + * writes both raw and aggregate observations to disk. + */ + public void writeFiles() { + + /* Did we learn anything new about average relay or bridge numbers in + * this run? */ + boolean writeConsensusStats = false; + + /* Go through raw observations of numbers of running bridges in bridge + * statuses, calculate averages per day, and add these averages to + * final results. */ + if (!this.bridgesRaw.isEmpty()) { + String tempDate = null; + int brunning = 0, statuses = 0; + Iterator<String> it = this.bridgesRaw.values().iterator(); + boolean haveWrittenFinalLine = false; + while (it.hasNext() || !haveWrittenFinalLine) { + String next = it.hasNext() ? it.next() : null; + /* Finished reading a day or even all lines? */ + if (tempDate != null && (next == null + || !next.substring(0, 10).equals(tempDate))) { + /* Only write results if we have seen at least half of all + * statuses. */ + if (statuses >= 24) { + String line = "," + (brunning / statuses); + /* Are our results new? */ + if (!this.bridgesPerDay.containsKey(tempDate)) { + this.logger.finer("Adding new average bridge numbers: " + + tempDate + line); + this.bridgesPerDay.put(tempDate, line); + writeConsensusStats = true; + } else if (!line.equals(this.bridgesPerDay.get(tempDate))) { + this.logger.finer("Replacing existing average bridge " + + "numbers (" + this.bridgesPerDay.get(tempDate) + + " with new numbers: " + line); + this.bridgesPerDay.put(tempDate, line); + writeConsensusStats = true; + } + } + brunning = statuses = 0; + haveWrittenFinalLine = (next == null); + } + /* Sum up number of running bridges. */ + if (next != null) { + tempDate = next.substring(0, 10); + statuses++; + brunning += Integer.parseInt(next.split(",")[1]); + } + } + } + + /* Write raw numbers of running bridges to disk. */ + try { + this.logger.fine("Writing file " + + this.bridgeConsensusStatsRawFile.getAbsolutePath() + "..."); + this.bridgeConsensusStatsRawFile.getParentFile().mkdirs(); + BufferedWriter bw = new BufferedWriter( + new FileWriter(this.bridgeConsensusStatsRawFile)); + bw.append("datetime,brunning\n"); + for (String line : this.bridgesRaw.values()) { + bw.append(line + "\n"); + } + bw.close(); + this.logger.fine("Finished writing file " + + this.bridgeConsensusStatsRawFile.getAbsolutePath() + "."); + } catch (IOException e) { + this.logger.log(Level.WARNING, "Failed to write file " + + this.bridgeConsensusStatsRawFile.getAbsolutePath() + "!", + e); + } + + /* Add average number of bridges per day to the database. */ + if (connectionURL != null) { + try { + Map<String, String> insertRows = new HashMap<String, String>(), + updateRows = new HashMap<String, String>(); + insertRows.putAll(this.bridgesPerDay); + Connection conn = DriverManager.getConnection(connectionURL); + conn.setAutoCommit(false); + Statement statement = conn.createStatement(); + ResultSet rs = statement.executeQuery( + "SELECT date, avg_running FROM bridge_network_size"); + while (rs.next()) { + String date = rs.getDate(1).toString(); + if (insertRows.containsKey(date)) { + String insertRow = insertRows.remove(date); + long newAvgRunning = Long.parseLong(insertRow.substring(1)); + long oldAvgRunning = rs.getLong(2); + if (newAvgRunning != oldAvgRunning) { + updateRows.put(date, insertRow); + } + } + } + rs.close(); + PreparedStatement psU = conn.prepareStatement( + "UPDATE bridge_network_size SET avg_running = ? " + + "WHERE date = ?"); + for (Map.Entry<String, String> e : updateRows.entrySet()) { + java.sql.Date date = java.sql.Date.valueOf(e.getKey()); + long avgRunning = Long.parseLong(e.getValue().substring(1)); + psU.clearParameters(); + psU.setLong(1, avgRunning); + psU.setDate(2, date); + psU.executeUpdate(); + } + PreparedStatement psI = conn.prepareStatement( + "INSERT INTO bridge_network_size (avg_running, date) " + + "VALUES (?, ?)"); + for (Map.Entry<String, String> e : insertRows.entrySet()) { + java.sql.Date date = java.sql.Date.valueOf(e.getKey()); + long avgRunning = Long.parseLong(e.getValue().substring(1)); + psI.clearParameters(); + psI.setLong(1, avgRunning); + psI.setDate(2, date); + psI.executeUpdate(); + } + conn.commit(); + conn.close(); + } catch (SQLException e) { + logger.log(Level.WARNING, "Failed to add average bridge numbers " + + "to database.", e); + } + } + + /* Write stats. */ + StringBuilder dumpStats = new StringBuilder("Finished writing " + + "statistics on bridge network statuses to disk.\nAdded " + + this.bridgeResultsAdded + " bridge network status(es) in this " + + "execution."); + long now = System.currentTimeMillis(); + SimpleDateFormat dateTimeFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + if (this.bridgesRaw.isEmpty()) { + dumpStats.append("\nNo bridge status known yet."); + } else { + dumpStats.append("\nLast known bridge status was published " + + this.bridgesRaw.lastKey() + "."); + try { + if (now - 6L * 60L * 60L * 1000L > dateTimeFormat.parse( + this.bridgesRaw.lastKey()).getTime()) { + logger.warning("Last known bridge status is more than 6 hours " + + "old: " + this.bridgesRaw.lastKey()); + } + } catch (ParseException e) { + /* Can't parse the timestamp? Whatever. */ + } + } + logger.info(dumpStats.toString()); + } +} + diff --git a/src/org/torproject/ernie/cron/Main.java b/src/org/torproject/ernie/cron/Main.java index 1a125bc..0551586 100644 --- a/src/org/torproject/ernie/cron/Main.java +++ b/src/org/torproject/ernie/cron/Main.java @@ -33,6 +33,11 @@ public class Main { // Define stats directory for temporary files File statsDirectory = new File("stats"); + // Prepare bridge stats file handler + BridgeStatsFileHandler bsfh = config.getWriteBridgeStats() ? + new BridgeStatsFileHandler( + config.getRelayDescriptorDatabaseJDBC()) : null; + // Prepare consensus health checker ConsensusHealthChecker chc = config.getWriteConsensusHealth() ? new ConsensusHealthChecker() : null; @@ -50,7 +55,7 @@ public class Main { // Prepare relay descriptor parser (only if we are writing the // consensus-health page to disk) RelayDescriptorParser rdp = chc != null || rddi != null ? - new RelayDescriptorParser(chc, rddi) : null; + new RelayDescriptorParser(chc, rddi, bsfh) : null; // Import relay descriptors if (rdp != null) { @@ -73,6 +78,33 @@ public class Main { chc = null; } + // Prepare consensus stats file handler (used for stats on running + // bridges only) + ConsensusStatsFileHandler csfh = config.getWriteBridgeStats() ? + new ConsensusStatsFileHandler( + config.getRelayDescriptorDatabaseJDBC()) : null; + + // Prepare bridge descriptor parser + BridgeDescriptorParser bdp = config.getWriteBridgeStats() ? + new BridgeDescriptorParser(csfh, bsfh) : null; + + // Import bridge descriptors + if (bdp != null && config.getImportSanitizedBridges()) { + new SanitizedBridgesReader(bdp, + new File(config.getSanitizedBridgesDirectory()), + statsDirectory, config.getKeepSanitizedBridgesImportHistory()); + } + + // Write updated stats files to disk + if (bsfh != null) { + bsfh.writeFiles(); + bsfh = null; + } + if (csfh != null) { + csfh.writeFiles(); + csfh = null; + } + // Remove lock file lf.releaseLock(); diff --git a/src/org/torproject/ernie/cron/RelayDescriptorParser.java b/src/org/torproject/ernie/cron/RelayDescriptorParser.java index d02022a..a9a0d3d 100644 --- a/src/org/torproject/ernie/cron/RelayDescriptorParser.java +++ b/src/org/torproject/ernie/cron/RelayDescriptorParser.java @@ -18,6 +18,11 @@ import org.apache.commons.codec.binary.*; public class RelayDescriptorParser { /** + * Stats file handler that accepts parse results for bridge statistics. + */ + private BridgeStatsFileHandler bsfh; + + /** * Relay descriptor database importer that stores relay descriptor * contents for later evaluation. */ @@ -36,9 +41,10 @@ public class RelayDescriptorParser { * Initializes this class. */ public RelayDescriptorParser(ConsensusHealthChecker chc, - RelayDescriptorDatabaseImporter rddi) { + RelayDescriptorDatabaseImporter rddi, BridgeStatsFileHandler bsfh) { this.chc = chc; this.rddi = rddi; + this.bsfh = bsfh; /* Initialize logger. */ this.logger = Logger.getLogger(RelayDescriptorParser.class.getName()); @@ -76,6 +82,7 @@ public class RelayDescriptorParser { orPort = 0L, dirPort = 0L; SortedSet<String> relayFlags = null; StringBuilder rawStatusEntry = null; + SortedSet<String> hashedRelayIdentities = new TreeSet<String>(); while ((line = br.readLine()) != null) { if (line.equals("vote-status vote")) { isConsensus = false; @@ -111,6 +118,9 @@ public class RelayDescriptorParser { relayIdentity = Hex.encodeHexString( Base64.decodeBase64(parts[2] + "=")). toLowerCase(); + hashedRelayIdentities.add(DigestUtils.shaHex( + Base64.decodeBase64(parts[2] + "=")). + toUpperCase()); serverDesc = Hex.encodeHexString(Base64.decodeBase64( parts[3] + "=")).toLowerCase(); published = parseFormat.parse(parts[4] + " " + parts[5]). @@ -144,6 +154,11 @@ public class RelayDescriptorParser { } } if (isConsensus) { + if (this.bsfh != null) { + for (String hashedRelayIdentity : hashedRelayIdentities) { + this.bsfh.addHashedRelay(hashedRelayIdentity); + } + } if (this.chc != null) { this.chc.processConsensus(validAfterTime, data); } diff --git a/src/org/torproject/ernie/cron/SanitizedBridgesReader.java b/src/org/torproject/ernie/cron/SanitizedBridgesReader.java new file mode 100644 index 0000000..b7f1d44 --- /dev/null +++ b/src/org/torproject/ernie/cron/SanitizedBridgesReader.java @@ -0,0 +1,114 @@ +/* Copyright 2011 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.ernie.cron; + +import java.io.*; +import java.util.*; +import java.util.logging.*; + +public class SanitizedBridgesReader { + public SanitizedBridgesReader(BridgeDescriptorParser bdp, + File bridgesDir, File statsDirectory, boolean keepImportHistory) { + + if (bdp == null || bridgesDir == null || statsDirectory == null) { + throw new IllegalArgumentException(); + } + + Logger logger = + Logger.getLogger(SanitizedBridgesReader.class.getName()); + SortedSet<String> bridgesImportHistory = new TreeSet<String>(); + File bridgesImportHistoryFile = + new File(statsDirectory, "bridges-import-history"); + if (keepImportHistory && bridgesImportHistoryFile.exists()) { + try { + BufferedReader br = new BufferedReader(new FileReader( + bridgesImportHistoryFile)); + String line = null; + while ((line = br.readLine()) != null) { + bridgesImportHistory.add(line); + } + br.close(); + } catch (IOException e) { + logger.log(Level.WARNING, "Could not read in bridge descriptor " + + "import history file. Skipping."); + } + } + if (bridgesDir.exists()) { + logger.fine("Importing files in directory " + bridgesDir + "/..."); + Stack<File> filesInInputDir = new Stack<File>(); + filesInInputDir.add(bridgesDir); + List<File> problems = new ArrayList<File>(); + while (!filesInInputDir.isEmpty()) { + File pop = filesInInputDir.pop(); + if (pop.isDirectory()) { + for (File f : pop.listFiles()) { + filesInInputDir.add(f); + } + continue; + } else if (keepImportHistory && bridgesImportHistory.contains( + pop.getName())) { + continue; + } else { + try { + BufferedInputStream bis = new BufferedInputStream( + new FileInputStream(pop)); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + int len; + byte[] data = new byte[1024]; + while ((len = bis.read(data, 0, 1024)) >= 0) { + baos.write(data, 0, len); + } + bis.close(); + byte[] allData = baos.toByteArray(); + String fn = pop.getName(); + // TODO dateTime extraction doesn't work for sanitized network + // statuses! + String dateTime = fn.substring(0, 4) + "-" + fn.substring(4, 6) + + "-" + fn.substring(6, 8) + " " + fn.substring(9, 11) + + ":" + fn.substring(11, 13) + ":" + fn.substring(13, 15); + bdp.parse(allData, dateTime, true); + if (keepImportHistory) { + bridgesImportHistory.add(pop.getName()); + } + } catch (IOException e) { + problems.add(pop); + if (problems.size() > 3) { + break; + } + } + } + } + if (problems.isEmpty()) { + logger.fine("Finished importing files in directory " + bridgesDir + + "/."); + } else { + StringBuilder sb = new StringBuilder("Failed importing files in " + + "directory " + bridgesDir + "/:"); + int printed = 0; + for (File f : problems) { + sb.append("\n " + f.getAbsolutePath()); + if (++printed >= 3) { + sb.append("\n ... more"); + break; + } + } + logger.warning(sb.toString()); + } + if (keepImportHistory) { + try { + bridgesImportHistoryFile.getParentFile().mkdirs(); + BufferedWriter bw = new BufferedWriter(new FileWriter( + bridgesImportHistoryFile)); + for (String line : bridgesImportHistory) { + bw.write(line + "\n"); + } + bw.close(); + } catch (IOException e) { + logger.log(Level.WARNING, "Could not write bridge descriptor " + + "import history file."); + } + } + } + } +} + diff --git a/src/org/torproject/ernie/test/SanitizedBridgesReaderTest.java b/src/org/torproject/ernie/test/SanitizedBridgesReaderTest.java new file mode 100644 index 0000000..6dd9132 --- /dev/null +++ b/src/org/torproject/ernie/test/SanitizedBridgesReaderTest.java @@ -0,0 +1,33 @@ +/* Copyright 2011 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.ernie.test; + +import org.torproject.ernie.cron.*; + +import java.io.*; + +import org.junit.*; +import org.junit.rules.*; +import static org.junit.Assert.*; + +public class SanitizedBridgesReaderTest { + + private File tempSanitizedBridgesDirectory; + private File tempStatsDirectory; + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Before + public void createTempDirectories() { + this.tempSanitizedBridgesDirectory = folder.newFolder("bridges"); + this.tempStatsDirectory = folder.newFolder("stats"); + } + + @Test(expected = IllegalArgumentException.class) + public void testBridgeDescriptorParserNull() { + new SanitizedBridgesReader(null, this.tempSanitizedBridgesDirectory, + this.tempStatsDirectory, false); + } +} +

1 0

[torflow/master] Update submodule....
by mikeperry＠torproject.org 03 Mar '11

03 Mar '11

commit ab33cdf934692ed00fc38ea4b9e69ac33a437fcc Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Thu Mar 3 01:56:17 2011 -0800 Update submodule.... --- TorCtl | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/TorCtl b/TorCtl index 62e4705..6eeffeb 160000 --- a/TorCtl +++ b/TorCtl @@ -1 +1 @@ -Subproject commit 62e470554c12e3fa0edd948d8f2c7c7a02217635 +Subproject commit 6eeffeb4f20aefb2f859a9f5754c34342abc307e

1 0

[pytorctl/master] Try to catch an AttributeError early..
by mikeperry＠torproject.org 03 Mar '11

03 Mar '11

commit 6eeffeb4f20aefb2f859a9f5754c34342abc307e Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Thu Mar 3 01:38:15 2011 -0800 Try to catch an AttributeError early.. It may be silently killing the bwauths on some platforms.. --- PathSupport.py | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-) diff --git a/PathSupport.py b/PathSupport.py index 3fefb24..e3ddfd1 100644 --- a/PathSupport.py +++ b/PathSupport.py @@ -1342,7 +1342,11 @@ class SmartSocket(_SocketWrapper): def __del__(self): SmartSocket._table_lock.acquire() - SmartSocket.port_table.remove(self.__local_addr) + try: + SmartSocket.port_table.remove(self.__local_addr) + except AttributeError,e: + traceback.print_exc() + plog("WARN", "Hrm. Socket instance without local_addr attribute?") SmartSocket._table_lock.release() plog("DEBUG", "Removed "+self.__local_addr+" from our local port list")

1 0

[torspec/master] Clarify that circwindow overrides the default of 1000 in tor-spec too
by nickm＠torproject.org 03 Mar '11

03 Mar '11

commit 92fe1a5cbbd8477b381bbfc40d1fe2a41ee19d5f Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Mar 3 00:00:15 2011 -0500 Clarify that circwindow overrides the default of 1000 in tor-spec too --- tor-spec.txt | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diff --git a/tor-spec.txt b/tor-spec.txt index 91ad561..e889431 100644 --- a/tor-spec.txt +++ b/tor-spec.txt @@ -965,7 +965,9 @@ see tor-design.pdf. do not apply to cells that the OR receives from one host and relays to another. - Each 'window' value is initially set to 1000 data cells + Each 'window' value is initially set based on the consensus parameter + 'circwindow' in the directory (see dir-spec.txt), or to 1000 data cells + if no 'circwindow' value is given, in each direction (cells that are not data cells do not affect the window). When an OR is willing to deliver more cells, it sends a RELAY_SENDME cell towards the OP, with Stream ID zero. When an OR

1 0

r24295: {arm} Adding config option for the connection panel refresh rate. (in arm/trunk: . src/interface/connections)
by Damian Johnson 03 Mar '11

03 Mar '11

Author: atagar Date: 2011-03-03 04:32:17 +0000 (Thu, 03 Mar 2011) New Revision: 24295 Modified: arm/trunk/armrc.sample arm/trunk/src/interface/connections/connPanel.py Log: Adding config option for the connection panel refresh rate. Modified: arm/trunk/armrc.sample =================================================================== --- arm/trunk/armrc.sample 2011-03-03 04:16:21 UTC (rev 24294) +++ arm/trunk/armrc.sample 2011-03-03 04:32:17 UTC (rev 24295) @@ -155,6 +155,11 @@ # --------------------------------- # oldPanel # includes the old connection panel in the interface +# refreshRate +# rate at which the connection panel contents is redrawn (if higher than the +# connection resolution rate then reducing this won't casue new data to +# appear more frequently - just increase the rate at which the uptime field +# is updated) # newPanel # includes the new connection panel in the interface # showColumn.* @@ -162,6 +167,7 @@ features.connection.oldPanel true features.connection.newPanel false +features.connection.refreshRate 10 features.connection.showColumn.fingerprint true features.connection.showColumn.nickname true features.connection.showColumn.destination true Modified: arm/trunk/src/interface/connections/connPanel.py =================================================================== --- arm/trunk/src/interface/connections/connPanel.py 2011-03-03 04:16:21 UTC (rev 24294) +++ arm/trunk/src/interface/connections/connPanel.py 2011-03-03 04:32:17 UTC (rev 24295) @@ -9,10 +9,8 @@ from interface.connections import listings from util import connections, enum, log, panel, torTools, uiTools -REDRAW_RATE = 10 # TODO: make a config option +DEFAULT_CONFIG = {"features.connection.refreshRate": 10} -DEFAULT_CONFIG = {} - # height of the detail panel content, not counting top and bottom border DETAILS_HEIGHT = 7 @@ -33,7 +31,8 @@ #self.sortOrdering = DEFAULT_SORT_ORDER self._config = dict(DEFAULT_CONFIG) if config: - config.update(self._config) + config.update(self._config, { + "features.connection.refreshRate": 1}) # TODO: test and add to the sample armrc #self.sortOrdering = config.getIntCSV("features.connections.order", self.sortOrdering, 3, 0, 6) @@ -99,7 +98,7 @@ while not self._halt: currentTime = time.time() - if self._isPaused or currentTime - lastDraw < REDRAW_RATE: + if self._isPaused or currentTime - lastDraw < self._config["features.connection.refreshRate"]: self._cond.acquire() if not self._halt: self._cond.wait(0.2) self._cond.release() @@ -107,7 +106,7 @@ # updates content if their's new results, otherwise just redraws self._update() self.redraw(True) - lastDraw += REDRAW_RATE + lastDraw += self._config["features.connection.refreshRate"] def draw(self, width, height): self.valsLock.acquire()

1 0

r24294: {arm} Adding config options for displaying the new and/or old conf (in arm/trunk: . src/interface)
by Damian Johnson 03 Mar '11

03 Mar '11

Author: atagar Date: 2011-03-03 04:16:21 +0000 (Thu, 03 Mar 2011) New Revision: 24294 Modified: arm/trunk/armrc.sample arm/trunk/src/interface/controller.py Log: Adding config options for displaying the new and/or old config panel. Modified: arm/trunk/armrc.sample =================================================================== --- arm/trunk/armrc.sample 2011-03-03 03:37:40 UTC (rev 24293) +++ arm/trunk/armrc.sample 2011-03-03 04:16:21 UTC (rev 24294) @@ -153,9 +153,15 @@ # Parameters for connection display # --------------------------------- -# features.connection.showColumn.* +# oldPanel +# includes the old connection panel in the interface +# newPanel +# includes the new connection panel in the interface +# showColumn.* # toggles the visability of the connection table columns +features.connection.oldPanel true +features.connection.newPanel false features.connection.showColumn.fingerprint true features.connection.showColumn.nickname true features.connection.showColumn.destination true Modified: arm/trunk/src/interface/controller.py =================================================================== --- arm/trunk/src/interface/controller.py 2011-03-03 03:37:40 UTC (rev 24293) +++ arm/trunk/src/interface/controller.py 2011-03-03 04:16:21 UTC (rev 24294) @@ -31,8 +31,6 @@ import graphing.connStats import graphing.resourceStats -INCLUDE_CONNPANEL_2 = False - CONFIRM_QUIT = True REFRESH_RATE = 5 # seconds between redrawing screen MAX_REGEX_FILTERS = 5 # maximum number of previous regex filters that'll be remembered @@ -45,17 +43,17 @@ PAGES = [ ["graph", "log"], ["conn"], + ["conn2"], ["config"], ["torrc"]] -if INCLUDE_CONNPANEL_2: - PAGES.append(["conn2"]) - PAUSEABLE = ["header", "graph", "log", "conn", "conn2"] CONFIG = {"log.torrc.readFailed": log.WARN, "features.graph.type": 1, "features.config.prepopulateEditValues": True, + "features.connection.oldPanel": True, + "features.connection.newPanel": False, "queries.refreshRate.rate": 5, "log.torEventTypeUnrecognized": log.NOTICE, "features.graph.bw.prepopulate": True, @@ -560,8 +558,18 @@ # before being positioned - the following is a quick hack til rewritten panels["log"].setPaused(True) - panels["conn"] = connPanel.ConnPanel(stdscr, conn, isBlindMode) - panels["conn2"] = interface.connections.connPanel.ConnectionPanel(stdscr, config) + if CONFIG["features.connection.oldPanel"]: + panels["conn"] = connPanel.ConnPanel(stdscr, conn, isBlindMode) + else: + panels["conn"] = panel.Panel(stdscr, "blank", 0, 0, 0) + PAUSEABLE.remove("conn") + + if CONFIG["features.connection.newPanel"]: + panels["conn2"] = interface.connections.connPanel.ConnectionPanel(stdscr, config) + else: + panels["conn2"] = panel.Panel(stdscr, "blank", 0, 0, 0) + PAUSEABLE.remove("conn2") + panels["control"] = ControlPanel(stdscr, isBlindMode) panels["config"] = configPanel.ConfigPanel(stdscr, configPanel.State.TOR, config) panels["torrc"] = torrcPanel.TorrcPanel(stdscr, torrcPanel.Config.TORRC, config) @@ -587,7 +595,8 @@ conn.add_event_listener(panels["graph"].stats["bandwidth"]) conn.add_event_listener(panels["graph"].stats["system resources"]) if not isBlindMode: conn.add_event_listener(panels["graph"].stats["connections"]) - conn.add_event_listener(panels["conn"]) + if CONFIG["features.connection.oldPanel"]: + conn.add_event_listener(panels["conn"]) conn.add_event_listener(sighupTracker) # prepopulates bandwidth values from state file @@ -614,7 +623,8 @@ # tells revised panels to run as daemons panels["header"].start() panels["log"].start() - panels["conn2"].start() + if CONFIG["features.connection.newPanel"]: + panels["conn2"].start() # warns if tor isn't updating descriptors #try: @@ -673,7 +683,8 @@ #panels["header"]._updateParams(True) # other panels that use torrc data - panels["conn"].resetOptions() + if CONFIG["features.connection.oldPanel"]: + panels["conn"].resetOptions() #if not isBlindMode: panels["graph"].stats["connections"].resetOptions(conn) #panels["graph"].stats["bandwidth"].resetOptions() @@ -734,7 +745,8 @@ isUnresponsive = False log.log(log.NOTICE, "Relay resumed") - panels["conn"].reset() + if CONFIG["features.connection.oldPanel"]: + panels["conn"].reset() # TODO: part two of hack to prevent premature drawing by log panel if page == 0 and not isPaused: panels["log"].setPaused(False) @@ -828,11 +840,11 @@ # stops panel daemons panels["header"].stop() - panels["conn2"].stop() + if CONFIG["features.connection.newPanel"]: panels["conn2"].stop() panels["log"].stop() panels["header"].join() - panels["conn2"].join() + if CONFIG["features.connection.newPanel"]: panels["conn2"].join() panels["log"].join() # joins on utility daemon threads - this might take a moment since @@ -855,10 +867,15 @@ if key == curses.KEY_LEFT: page = (page - 1) % len(PAGES) else: page = (page + 1) % len(PAGES) - # skip connections listing if it's disabled - if page == 1 and isBlindMode: - if key == curses.KEY_LEFT: page = (page - 1) % len(PAGES) - else: page = (page + 1) % len(PAGES) + # skip connections listings if it's disabled + while True: + if page == 1 and (isBlindMode or not CONFIG["features.connection.oldPanel"]): + if key == curses.KEY_LEFT: page = (page - 1) % len(PAGES) + else: page = (page + 1) % len(PAGES) + elif page == 2 and (isBlindMode or not CONFIG["features.connection.newPanel"]): + if key == curses.KEY_LEFT: page = (page - 1) % len(PAGES) + else: page = (page + 1) % len(PAGES) + else: break # pauses panels that aren't visible to prevent events from accumilating # (otherwise they'll wait on the curses lock which might get demanding) @@ -967,7 +984,9 @@ if resolverUtil == None: resolverUtil = "auto" popup.addfstr(4, 41, "u: resolving utility (%s)" % resolverUtil) - allowDnsLabel = "allow" if panels["conn"].allowDNS else "disallow" + if CONFIG["features.connection.oldPanel"]: + allowDnsLabel = "allow" if panels["conn"].allowDNS else "disallow" + else: allowDnsLabel = "disallow" popup.addfstr(5, 2, "r: permit DNS resolution (%s)" % allowDnsLabel) popup.addfstr(5, 41, "s: sort ordering") @@ -1275,7 +1294,7 @@ setPauseState(panels, isPaused, page) finally: panel.CURSES_LOCK.release() - elif key == 27 and panels["conn"].listingType == connPanel.LIST_HOSTNAME and panels["control"].resolvingCounter != -1: + elif CONFIG["features.connection.oldPanel"] and key == 27 and panels["conn"].listingType == connPanel.LIST_HOSTNAME and panels["control"].resolvingCounter != -1: # canceling hostname resolution (esc on any page) panels["conn"].listingType = connPanel.LIST_IP panels["control"].resolvingCounter = -1 @@ -1577,7 +1596,7 @@ setPauseState(panels, isPaused, page) finally: panel.CURSES_LOCK.release() - elif page == 2 and (key == ord('c') or key == ord('C')) and False: + elif page == 3 and (key == ord('c') or key == ord('C')) and False: # TODO: disabled for now (probably gonna be going with separate pages # rather than popup menu) # provides menu to pick config being displayed @@ -1600,7 +1619,7 @@ if selection != -1: panels["torrc"].setConfigType(selection) selectiveRefresh(panels, page) - elif page == 2 and (key == ord('w') or key == ord('W')): + elif page == 3 and (key == ord('w') or key == ord('W')): # display a popup for saving the current configuration panel.CURSES_LOCK.acquire() try: @@ -1730,7 +1749,7 @@ panel.CURSES_LOCK.release() panels["config"].redraw(True) - elif page == 2 and (key == ord('s') or key == ord('S')): + elif page == 3 and (key == ord('s') or key == ord('S')): # set ordering for config options titleLabel = "Config Option Ordering:" options = [configPanel.FIELD_ATTR[field][0] for field in configPanel.Field.values()] @@ -1751,7 +1770,7 @@ panels["config"].setSortOrder(resultEnums) panels["config"].redraw(True) - elif page == 2 and uiTools.isSelectionKey(key): + elif page == 3 and uiTools.isSelectionKey(key): # let the user edit the configuration value, unchanged if left blank panel.CURSES_LOCK.acquire() try: @@ -1807,7 +1826,7 @@ setPauseState(panels, isPaused, page) finally: panel.CURSES_LOCK.release() - elif page == 3 and key == ord('r') or key == ord('R'): + elif page == 4 and key == ord('r') or key == ord('R'): # reloads torrc, providing a notice if successful or not loadedTorrc = torConfig.getTorrc() loadedTorrc.getLock().acquire() @@ -1837,11 +1856,11 @@ elif page == 1: panels["conn"].handleKey(key) elif page == 2: + panels["conn2"].handleKey(key) + elif page == 3: panels["config"].handleKey(key) - elif page == 3: + elif page == 4: panels["torrc"].handleKey(key) - elif page == 4: - panels["conn2"].handleKey(key) def startTorMonitor(startTime, loggedEvents, isBlindMode): try:

1 0

r24293: {arm} Adding a config option for connection panel column visabilit (in arm/trunk: . src/interface src/interface/connections)
by Damian Johnson 03 Mar '11

03 Mar '11

Author: atagar Date: 2011-03-03 03:37:40 +0000 (Thu, 03 Mar 2011) New Revision: 24293 Modified: arm/trunk/armrc.sample arm/trunk/src/interface/connections/listings.py arm/trunk/src/interface/controller.py Log: Adding a config option for connection panel column visability. Modified: arm/trunk/armrc.sample =================================================================== --- arm/trunk/armrc.sample 2011-03-02 21:16:07 UTC (rev 24292) +++ arm/trunk/armrc.sample 2011-03-03 03:37:40 UTC (rev 24293) @@ -151,6 +151,16 @@ features.graph.bw.accounting.rate 10 features.graph.bw.accounting.isTimeLong false +# Parameters for connection display +# --------------------------------- +# features.connection.showColumn.* +# toggles the visability of the connection table columns + +features.connection.showColumn.fingerprint true +features.connection.showColumn.nickname true +features.connection.showColumn.destination true +features.connection.showColumn.expanedIp true + # Thread pool size for hostname resolutions # Determines the maximum number of concurrent requests. Upping this to around # thirty or so seems to be problematic, causing intermittently seizing. Modified: arm/trunk/src/interface/connections/listings.py =================================================================== --- arm/trunk/src/interface/connections/listings.py 2011-03-02 21:16:07 UTC (rev 24292) +++ arm/trunk/src/interface/connections/listings.py 2011-03-03 03:37:40 UTC (rev 24293) @@ -7,15 +7,14 @@ from util import connections, enum, hostnames, torTools, uiTools # Connection Categories: -# Inbound Relay connection, coming to us. -# Outbound Relay connection, leaving us. -# Exit Outbound relay connection leaving the Tor network. -# Socks Application client connection. -# Client Circuits for our client traffic. -# Directory Fetching tor consensus information. -# Control Tor controller (arm, vidalia, etc). +# Inbound Relay connection, coming to us. +# Outbound Relay connection, leaving us. +# Exit Outbound relay connection leaving the Tor network. +# Client Circuits for our client traffic. +# Application Socks connections using Tor. +# Directory Fetching tor consensus information. +# Control Tor controller (arm, vidalia, etc). -# TODO: add recognizing of CLIENT connection type DestAttr = enum.Enum("NONE", "LOCALE", "HOSTNAME") Category = enum.Enum("INBOUND", "OUTBOUND", "EXIT", "CLIENT", "APPLICATION", "DIRECTORY", "CONTROL") CATEGORY_COLOR = {Category.INBOUND: "green", Category.OUTBOUND: "blue", @@ -28,6 +27,14 @@ LABEL_FORMAT = "%s --> %s %s%s" LABEL_MIN_PADDING = 2 # min space between listing label and following data +CONFIG = {"features.connection.showColumn.fingerprint": True, + "features.connection.showColumn.nickname": True, + "features.connection.showColumn.destination": True, + "features.connection.showColumn.expanedIp": True} + +def loadConfig(config): + config.update(CONFIG) + class Endpoint: """ Collection of attributes associated with a connection endpoint. This is a @@ -348,18 +355,18 @@ usedSpace += len(src) + len(dst) # base data requires 47 characters - if width > usedSpace + 42: + if width > usedSpace + 42 and CONFIG["features.connection.showColumn.fingerprint"]: # show fingerprint (column width: 42 characters) etc += "%-40s " % self.foreign.getFingerprint() usedSpace += 42 - if addrDiffer and width > usedSpace + 28: + if addrDiffer and width > usedSpace + 28 and CONFIG["features.connection.showColumn.expanedIp"]: # include the internal address in the src (extra 28 characters) internalAddress = "%s:%s" % (self.local.getIpAddr(), self.local.getPort()) src = "%-21s --> %s" % (internalAddress, src) usedSpace += 28 - if width > usedSpace + 10: + if width > usedSpace + 10 and CONFIG["features.connection.showColumn.nickname"]: # show nickname (column width: remainder) nicknameSpace = width - usedSpace nicknameLabel = uiTools.cropStr(self.foreign.getNickname(), nicknameSpace, 0) @@ -371,17 +378,17 @@ usedSpace += len(stc) minHostnameSpace = 40 - if width > usedSpace + minHostnameSpace + 28: + if width > usedSpace + minHostnameSpace + 28 and CONFIG["features.connection.showColumn.destination"]: # show destination ip/port/locale (column width: 28 characters) etc += "%-26s " % dstAddress usedSpace += 28 - if width > usedSpace + minHostnameSpace + 42: + if width > usedSpace + minHostnameSpace + 42 and CONFIG["features.connection.showColumn.fingerprint"]: # show fingerprint (column width: 42 characters) etc += "%-40s " % self.foreign.getFingerprint() usedSpace += 42 - if width > usedSpace + minHostnameSpace + 17: + if width > usedSpace + minHostnameSpace + 17 and CONFIG["features.connection.showColumn.nickname"]: # show nickname (column width: min 17 characters, uses half of the remainder) nicknameSpace = 15 + (width - (usedSpace + minHostnameSpace + 17)) / 2 nicknameLabel = uiTools.cropStr(self.foreign.getNickname(), nicknameSpace, 0) @@ -417,12 +424,14 @@ # if there's room then also show a column with the destination # ip/port/locale (column width: 28 characters) isIpLocaleIncluded = width > usedSpace + 45 + isIpLocaleIncluded &= CONFIG["features.connection.showColumn.destination"] if isIpLocaleIncluded: nicknameSpace -= 28 - nicknameSpace = width - usedSpace - 28 if isIpLocaleVisible else width - usedSpace - nicknameLabel = uiTools.cropStr(self.foreign.getNickname(), nicknameSpace, 0) - etc += ("%%-%is " % nicknameSpace) % nicknameLabel - usedSpace += nicknameSpace + 2 + if CONFIG["features.connection.showColumn.nickname"]: + nicknameSpace = width - usedSpace - 28 if isIpLocaleVisible else width - usedSpace + nicknameLabel = uiTools.cropStr(self.foreign.getNickname(), nicknameSpace, 0) + etc += ("%%-%is " % nicknameSpace) % nicknameLabel + usedSpace += nicknameSpace + 2 if isIpLocaleIncluded: etc += "%-26s " % dstAddress @@ -434,12 +443,12 @@ else: dst = self.foreign.getNickname() minBaseSpace = 50 - if width > usedSpace + minBaseSpace + 42: + if width > usedSpace + minBaseSpace + 42 and CONFIG["features.connection.showColumn.fingerprint"]: # show fingerprint (column width: 42 characters) etc += "%-40s " % self.foreign.getFingerprint() usedSpace += 42 - if width > usedSpace + minBaseSpace + 28: + if width > usedSpace + minBaseSpace + 28 and CONFIG["features.connection.showColumn.destination"]: # show destination ip/port/locale (column width: 28 characters) etc += "%-26s " % dstAddress usedSpace += 28 Modified: arm/trunk/src/interface/controller.py =================================================================== --- arm/trunk/src/interface/controller.py 2011-03-02 21:16:07 UTC (rev 24292) +++ arm/trunk/src/interface/controller.py 2011-03-03 03:37:40 UTC (rev 24293) @@ -25,6 +25,7 @@ import fileDescriptorPopup import interface.connections.connPanel +import interface.connections.listings from util import conf, log, connections, hostnames, panel, sysTools, torConfig, torTools, uiTools import graphing.bandwidthStats import graphing.connStats @@ -426,6 +427,7 @@ config = conf.getConfig("arm") config.update(CONFIG) graphing.graphPanel.loadConfig(config) + interface.connections.listings.loadConfig(config) # adds events needed for arm functionality to the torTools REQ_EVENTS mapping # (they're then included with any setControllerEvents call, and log a more

1 0

[torspec/master] rename xxx-draft-spec-for-TLS-normalization.txt to xxx-TLS-cert-and-parameter-normalization.txt
by ioerror＠torproject.org 03 Mar '11

03 Mar '11

commit 46f7a8f427a3b36e902b59c789c21c48c7dca70e Author: Jacob Appelbaum <jacob(a)appelbaum.net> Date: Wed Mar 2 15:37:35 2011 -0800 rename xxx-draft-spec-for-TLS-normalization.txt to xxx-TLS-cert-and-parameter-normalization.txt --- .../xxx-TLS-cert-and-parameter-normalization.txt | 360 ++++++++++++++++++++ .../ideas/xxx-draft-spec-for-TLS-normalization.txt | 360 -------------------- 2 files changed, 360 insertions(+), 360 deletions(-) diff --git a/proposals/ideas/xxx-TLS-cert-and-parameter-normalization.txt b/proposals/ideas/xxx-TLS-cert-and-parameter-normalization.txt new file mode 100644 index 0000000..74704b1 --- /dev/null +++ b/proposals/ideas/xxx-TLS-cert-and-parameter-normalization.txt @@ -0,0 +1,360 @@ +Filename: xxx-TLS-cert-and-parameter-normalization.txt +Title: TLS certificate and parameter normalization +Author: Jacob Appelbaum, Gladys Shufflebottom +Created: 16-Feb-2011 +Status: Draft + + + Draft spec for TLS certificate and handshake normalization + + + Overview + +Scope + +This is a document that proposes improvements to problems with Tor's +current TLS (Transport Layer Security) certificates and handshake that will +reduce the distinguishability of Tor traffic from other encrypted traffic that +uses TLS. It also addresses some of the possible fingerprinting attacks +possible against the current Tor TLS protocol setup process. + +Motivation and history + +Censorship is an arms race and this is a step forward in the defense +of Tor. This proposal outlines ideas to make it more difficult to +fingerprint and block Tor traffic. + +Goals + +This proposal intends to normalize or remove easy-to-predict or static +values in the Tor TLS certificates and with the Tor TLS setup process. +These values can be used as criteria for the automated classification of +encrypted traffic as Tor traffic. Network observers should not be able +to trivially detect Tor merely by receiving or observing the certificate +used or advertised by a Tor relay. I also propose the creation of +a hard-to-detect covert channel through which a server can signal that it +supports the third version ("V3") of the Tor handshake protocol. + +Non-Goals + +This document is not intended to solve all of the possible active or passive +Tor fingerprinting problems. This document focuses on removing distinctive +and predictable features of TLS protocol negotiation; we do not attempt to +make guarantees about resisting other kinds of fingerprinting of Tor +traffic, such as fingerprinting techniques related to timing or volume of +transmitted data. + + Implementation details + + +Certificate Issues + +The CN or commonName ASN1 field + +Tor generates certificates with a predictable commonName field; the +field is within a given range of values that is specific to Tor. +Additionally, the generated host names have other undesirable properties. +The host names typically do not resolve in the DNS because the domain +names referred to are generated at random. Although they are syntatically +valid, they usually refer to domains that have never been registered by +any domain name registrar. + +An example of the current commonName field: CN=www.s4ku5skci.net + +An example of OpenSSL’s asn1parse over a typical Tor certificate: + + 0:d=0 hl=4 l= 438 cons: SEQUENCE + 4:d=1 hl=4 l= 287 cons: SEQUENCE + 8:d=2 hl=2 l= 3 cons: cont [ 0 ] + 10:d=3 hl=2 l= 1 prim: INTEGER :02 + 13:d=2 hl=2 l= 4 prim: INTEGER :4D3C763A + 19:d=2 hl=2 l= 13 cons: SEQUENCE + 21:d=3 hl=2 l= 9 prim: OBJECT :sha1WithRSAEncryption + 32:d=3 hl=2 l= 0 prim: NULL + 34:d=2 hl=2 l= 35 cons: SEQUENCE + 36:d=3 hl=2 l= 33 cons: SET + 38:d=4 hl=2 l= 31 cons: SEQUENCE + 40:d=5 hl=2 l= 3 prim: OBJECT :commonName + 45:d=5 hl=2 l= 24 prim: PRINTABLESTRING :www.vsbsvwu5b4soh4wg.net + 71:d=2 hl=2 l= 30 cons: SEQUENCE + 73:d=3 hl=2 l= 13 prim: UTCTIME :110123184058Z + 88:d=3 hl=2 l= 13 prim: UTCTIME :110123204058Z + 103:d=2 hl=2 l= 28 cons: SEQUENCE + 105:d=3 hl=2 l= 26 cons: SET + 107:d=4 hl=2 l= 24 cons: SEQUENCE + 109:d=5 hl=2 l= 3 prim: OBJECT :commonName + 114:d=5 hl=2 l= 17 prim: PRINTABLESTRING :www.s4ku5skci.net + 133:d=2 hl=3 l= 159 cons: SEQUENCE + 136:d=3 hl=2 l= 13 cons: SEQUENCE + 138:d=4 hl=2 l= 9 prim: OBJECT :rsaEncryption + 149:d=4 hl=2 l= 0 prim: NULL + 151:d=3 hl=3 l= 141 prim: BIT STRING + 295:d=1 hl=2 l= 13 cons: SEQUENCE + 297:d=2 hl=2 l= 9 prim: OBJECT :sha1WithRSAEncryption + 308:d=2 hl=2 l= 0 prim: NULL + 310:d=1 hl=3 l= 129 prim: BIT STRING + +I propose that we match OpenSSL's default self-signed certificates. I hypothesise +that they are the most common self-signed certificates. If this turns out not +to be the case, then we should use whatever the most common turns out to be. + +Certificate serial numbers + +Currently our generated certificate serial number is set to the number of +seconds since the epoch at the time of the certificate's creation. I propose +that we should ensure that our serial numbers are unrelated to the epoch, +since the generation methods are potentially recognizable as Tor-related. + +Instead, I propose that we use a randomly generated number that is +subsequently hashed with SHA-512 and then truncate the data to eight bytes[1]. + +Random sixteen byte values appear to be the high bound for serial number as +issued by Verisign and DigiCert. RapidSSL appears to be three bytes in length. +Others common byte lengths appear to be between one and four bytes. The default +OpenSSL certificates are eight bytes and we should use this length with our +self-signed certificates. + +This randomly generated serial number field may now serve as a covert channel +that signals to the client that the OR will not support TLS renegotiation; this +means that the client can expect to perform a V3 TLS handshake setup. +Otherwise, if the serial number is a reasonable time since the epoch, we should +assume the OR is using an earlier protocol version and hence that it expects +renegotiation. + +We also have a need to signal properties with our certificates for a possible +v3 handshake in the future. Therefore I propose that we match OpenSSL default +self-signed certificates (a 64-bit random number), but reserve the two least- +significant bits for signaling. For the moment, these two bits will be zero. + +This means that an attacker may be able to identify Tor certificates from default +OpenSSL certificates with a 75% probability. + +As a security note, care must be taken to ensure that supporting this +covert channel will not lead to an attacker having a method to downgrade client +behavior. This shouldn't be a risk because the TLS Finished message hashes over +all the bytes of the handshake, including the certificates. + +Certificate fingerprinting issues expressed as base64 encoding + +It appears that all deployed Tor certificates have the following strings in +common: + +MIIB +CCA +gAwIBAgIETU +ANBgkqhkiG9w0BAQUFADA +YDVQQDEx +3d3cu + +As expected these values correspond to specific ASN.1 OBJECT IDENTIFIER (OID) +properties (sha1WithRSAEncryption, commonName, etc) of how we generate our +certificates. + +As an illustrated example of the common bytes of all certificates used within +the Tor network within a single one hour window, I have replaced the actual +value with a wild card ('.') character here: + +-----BEGIN CERTIFICATE----- +MIIB..CCA..gAwIBAgIETU....ANBgkqhkiG9w0BAQUFADA.M..w..YDVQQDEx.3 +d3cu............................................................ +................................................................ +................................................................ +................................................................ +................................................................ +................................................................ +................................................................ +................................................................ +........................... <--- Variable length and padding +-----END CERTIFICATE----- + +This fine ascii art only illustrates the bytes that absolutely match in all +cases. In many cases, it's likely that there is a high probability for a given +byte to be only a small subset of choices. + +Using the above strings, the EFF's certificate observatory may trivially +discover all known relays, known bridges and unknown bridges in a single SQL +query. I propose that we ensure that we test our certificates to ensure that +they do not have these kinds of statistical similarities without ensuring +overlap with a very large cross section of the internet's certificates. + +Certificate dating and validity issues + +TLS certificates found in the wild are generally found to be long-lived; +they are frequently old and often even expired. The current Tor certificate +validity time is a very small time window starting at generation time and +ending shortly thereafter, as defined in or.h by MAX_SSL_KEY_LIFETIME +(2*60*60). + +I propose that the certificate validity time length is extended to a period of +twelve Earth months, possibly with a small random skew to be determined by the +implementer. Tor should randomly set the start date in the past or some +currently unspecified window of time before the current date. This would +more closely track the typical distribution of non-Tor TLS certificate +expiration times. + +The certificate values, such as expiration, should not be used for anything +relating to security; for example, if the OR presents an expired TLS +certificate, this does not imply that the client should terminate the +connection (as would be appropriate for an ordinary TLS implementation). +Rather, I propose we use a TOFU style expiration policy - the certificate +should never be trusted for more than a two hour window from first sighting. + +This policy should have two major impacts. The first is that an adversary will +have to perform a differential analysis of all certificates for a given IP +address rather than a single check. The second is that the server expiration +time is enforced by the client and confirmed by keys rotating in the consensus. + +The expiration time should not be a fixed time that is simple to calculate by +any Deep Packet Inspection device or it will become a new Tor TLS setup +fingerprint. + +Proposed certificate form + +The following output from openssl asn1parse results from the proposed +certificate generation algorithm. It matches the results of generating a +default self-signed certificate: + + 0:d=0 hl=4 l= 513 cons: SEQUENCE + 4:d=1 hl=4 l= 362 cons: SEQUENCE + 8:d=2 hl=2 l= 9 prim: INTEGER :DBF6B3B864FF7478 + 19:d=2 hl=2 l= 13 cons: SEQUENCE + 21:d=3 hl=2 l= 9 prim: OBJECT :sha1WithRSAEncryption + 32:d=3 hl=2 l= 0 prim: NULL + 34:d=2 hl=2 l= 69 cons: SEQUENCE + 36:d=3 hl=2 l= 11 cons: SET + 38:d=4 hl=2 l= 9 cons: SEQUENCE + 40:d=5 hl=2 l= 3 prim: OBJECT :countryName + 45:d=5 hl=2 l= 2 prim: PRINTABLESTRING :AU + 49:d=3 hl=2 l= 19 cons: SET + 51:d=4 hl=2 l= 17 cons: SEQUENCE + 53:d=5 hl=2 l= 3 prim: OBJECT :stateOrProvinceName + 58:d=5 hl=2 l= 10 prim: PRINTABLESTRING :Some-State + 70:d=3 hl=2 l= 33 cons: SET + 72:d=4 hl=2 l= 31 cons: SEQUENCE + 74:d=5 hl=2 l= 3 prim: OBJECT :organizationName + 79:d=5 hl=2 l= 24 prim: PRINTABLESTRING :Internet Widgits Pty Ltd + 105:d=2 hl=2 l= 30 cons: SEQUENCE + 107:d=3 hl=2 l= 13 prim: UTCTIME :110217011237Z + 122:d=3 hl=2 l= 13 prim: UTCTIME :120217011237Z + 137:d=2 hl=2 l= 69 cons: SEQUENCE + 139:d=3 hl=2 l= 11 cons: SET + 141:d=4 hl=2 l= 9 cons: SEQUENCE + 143:d=5 hl=2 l= 3 prim: OBJECT :countryName + 148:d=5 hl=2 l= 2 prim: PRINTABLESTRING :AU + 152:d=3 hl=2 l= 19 cons: SET + 154:d=4 hl=2 l= 17 cons: SEQUENCE + 156:d=5 hl=2 l= 3 prim: OBJECT :stateOrProvinceName + 161:d=5 hl=2 l= 10 prim: PRINTABLESTRING :Some-State + 173:d=3 hl=2 l= 33 cons: SET + 175:d=4 hl=2 l= 31 cons: SEQUENCE + 177:d=5 hl=2 l= 3 prim: OBJECT :organizationName + 182:d=5 hl=2 l= 24 prim: PRINTABLESTRING :Internet Widgits Pty Ltd + 208:d=2 hl=3 l= 159 cons: SEQUENCE + 211:d=3 hl=2 l= 13 cons: SEQUENCE + 213:d=4 hl=2 l= 9 prim: OBJECT :rsaEncryption + 224:d=4 hl=2 l= 0 prim: NULL + 226:d=3 hl=3 l= 141 prim: BIT STRING + 370:d=1 hl=2 l= 13 cons: SEQUENCE + 372:d=2 hl=2 l= 9 prim: OBJECT :sha1WithRSAEncryption + 383:d=2 hl=2 l= 0 prim: NULL + 385:d=1 hl=3 l= 129 prim: BIT STRING + + +Custom Certificates + +It should be possible for a Tor relay operator to use a specifically supplied +certificate and secret key. This will allow a relay or bridge operator to use a +certificate signed by any member of any geographically relevant certificate +authority racket; it will also allow for any other user-supplied certificate. +This may be desirable in some kinds of filtered networks or when attempting to +avoid attracting suspicion by blending in with the TLS web server certificate +crowd. + +Problematic Diffie–Hellman parameters + +We currently send a static Diffie–Hellman parameter, prime p (or “prime p +outlaw”) as specified in RFC2409 as part of the TLS Server Hello response. + +The use of this prime in TLS negotiations may, as a result, be filtered and +effectively banned by certain networks. We do not have to use this particular +prime in all cases. + +While amusing to have the power to make specific prime numbers into a new class +of numbers (cf. imaginary, irrational, illegal [3]) - our new friend prime p +outlaw is not required. + +The use of this prime in TLS negotiations may, as a result, be filtered and +effectively banned by certain networks. We do not have to use this particular +prime in all cases. + +I propose that the function to initialize and generate DH parameters be +split into two functions. + +First, init_dh_param() should be used only for OR-to-OR DH setup and +communication. Second, it is proposed that we create a new function +init_tls_dh_param() that will have a two-stage development process. + +The first stage init_tls_dh_param() will use the same prime that +Apache2.x [4] sends (or “dh1024_apache_p”), and this change should be +made immediately. This is a known good and safe prime number (p-1 / 2 +is also prime) that is currently not known to be blocked. + +The second stage init_tls_dh_param() should randomly generate a new prime on a +regular basis; this is designed to make the prime difficult to outlaw or +filter. Call this a shape-shifting or "Rakshasa" prime. This should be added +to the 0.2.3.x branch of Tor. This prime can be generated at setup or execution +time and probably does not need to be stored on disk. Rakshasa primes only +need to be generated by Tor relays as Tor clients will never send them. Such +a prime should absolutely not be shared between different Tor relays nor +should it ever be static after the 0.2.3.x release. + +As a security precaution, care must be taken to ensure that we do not generate +weak primes or known filtered primes. Both weak and filtered primes will +undermine the TLS connection security properties. OpenSSH solves this issue +dynamically in RFC 4419 [5] and may provide a solution that works reasonably +well for Tor. More research in this area including the applicability of +Miller-Rabin or AKS primality tests[6] will need to be analyzed and probably +added to Tor. + +Practical key size + +Currently we use a 1024 bit long RSA modulus. I propose that we increase the +RSA key size to 2048 as an additional channel to signal support for the V3 +handshake setup. 2048 appears to be the most common key size[0] above 1024. +Additionally, the increase in modulus size provides a reasonable security boost +with regard to key security properties. + +The implementer should increase the 1024 bit RSA modulus to 2048 bits. + +Possible future filtering nightmares + +At some point it may cost effective or politically feasible for a network +filter to simply block all signed or self-signed certificates without a known +valid CA trust chain. This will break many applications on the internet and +hopefully, our option for custom certificates will ensure that this step is +simply avoided by the censors. + +The Rakshasa prime approach may cause censors to specifically allow only +certain known and accepted DH parameters. + + +Appendix: Other issues + +What other obvious TLS certificate issues exist? What other static values are +present in the Tor TLS setup process? + +[0] http://archives.seul.org/or/dev/Jan-2011/msg00051.html +[1] http://archives.seul.org/or/dev/Feb-2011/msg00016.html +[2] http://archives.seul.org/or/dev/Feb-2011/msg00039.html +[3] To be fair this is hardly a new class of numbers. History is rife with + similar examples of inane authoritarian attempts at mathematical secrecy. + Probably the most dramatic example is the story of the pupil Hipassus of + Metapontum, pupil of the famous Pythagoras, who, legend goes, proved the + fact that Root2 cannot be expressed as a fraction of whole numbers (now + called an irrational number) and was assassinated for revealing this + secret. Further reading on the subject may be found on the Wikipedia: + http://en.wikipedia.org/wiki/Hippasus + +[4] httpd-2.2.17/modules/ss/ssl_engine_dh.c +[5] http://tools.ietf.org/html/rfc4419 +[6] http://archives.seul.org/or/dev/Jan-2011/msg00037.html diff --git a/proposals/ideas/xxx-draft-spec-for-TLS-normalization.txt b/proposals/ideas/xxx-draft-spec-for-TLS-normalization.txt deleted file mode 100644 index 16484e6..0000000 --- a/proposals/ideas/xxx-draft-spec-for-TLS-normalization.txt +++ /dev/null @@ -1,360 +0,0 @@ -Filename: xxx-draft-spec-for-TLS-normalization.txt -Title: Draft spec for TLS certificate and handshake normalization -Author: Jacob Appelbaum, Gladys Shufflebottom -Created: 16-Feb-2011 -Status: Draft - - - Draft spec for TLS certificate and handshake normalization - - - Overview - -Scope - -This is a document that proposes improvements to problems with Tor's -current TLS (Transport Layer Security) certificates and handshake that will -reduce the distinguishability of Tor traffic from other encrypted traffic that -uses TLS. It also addresses some of the possible fingerprinting attacks -possible against the current Tor TLS protocol setup process. - -Motivation and history - -Censorship is an arms race and this is a step forward in the defense -of Tor. This proposal outlines ideas to make it more difficult to -fingerprint and block Tor traffic. - -Goals - -This proposal intends to normalize or remove easy-to-predict or static -values in the Tor TLS certificates and with the Tor TLS setup process. -These values can be used as criteria for the automated classification of -encrypted traffic as Tor traffic. Network observers should not be able -to trivially detect Tor merely by receiving or observing the certificate -used or advertised by a Tor relay. I also propose the creation of -a hard-to-detect covert channel through which a server can signal that it -supports the third version ("V3") of the Tor handshake protocol. - -Non-Goals - -This document is not intended to solve all of the possible active or passive -Tor fingerprinting problems. This document focuses on removing distinctive -and predictable features of TLS protocol negotiation; we do not attempt to -make guarantees about resisting other kinds of fingerprinting of Tor -traffic, such as fingerprinting techniques related to timing or volume of -transmitted data. - - Implementation details - - -Certificate Issues - -The CN or commonName ASN1 field - -Tor generates certificates with a predictable commonName field; the -field is within a given range of values that is specific to Tor. -Additionally, the generated host names have other undesirable properties. -The host names typically do not resolve in the DNS because the domain -names referred to are generated at random. Although they are syntatically -valid, they usually refer to domains that have never been registered by -any domain name registrar. - -An example of the current commonName field: CN=www.s4ku5skci.net - -An example of OpenSSL’s asn1parse over a typical Tor certificate: - - 0:d=0 hl=4 l= 438 cons: SEQUENCE - 4:d=1 hl=4 l= 287 cons: SEQUENCE - 8:d=2 hl=2 l= 3 cons: cont [ 0 ] - 10:d=3 hl=2 l= 1 prim: INTEGER :02 - 13:d=2 hl=2 l= 4 prim: INTEGER :4D3C763A - 19:d=2 hl=2 l= 13 cons: SEQUENCE - 21:d=3 hl=2 l= 9 prim: OBJECT :sha1WithRSAEncryption - 32:d=3 hl=2 l= 0 prim: NULL - 34:d=2 hl=2 l= 35 cons: SEQUENCE - 36:d=3 hl=2 l= 33 cons: SET - 38:d=4 hl=2 l= 31 cons: SEQUENCE - 40:d=5 hl=2 l= 3 prim: OBJECT :commonName - 45:d=5 hl=2 l= 24 prim: PRINTABLESTRING :www.vsbsvwu5b4soh4wg.net - 71:d=2 hl=2 l= 30 cons: SEQUENCE - 73:d=3 hl=2 l= 13 prim: UTCTIME :110123184058Z - 88:d=3 hl=2 l= 13 prim: UTCTIME :110123204058Z - 103:d=2 hl=2 l= 28 cons: SEQUENCE - 105:d=3 hl=2 l= 26 cons: SET - 107:d=4 hl=2 l= 24 cons: SEQUENCE - 109:d=5 hl=2 l= 3 prim: OBJECT :commonName - 114:d=5 hl=2 l= 17 prim: PRINTABLESTRING :www.s4ku5skci.net - 133:d=2 hl=3 l= 159 cons: SEQUENCE - 136:d=3 hl=2 l= 13 cons: SEQUENCE - 138:d=4 hl=2 l= 9 prim: OBJECT :rsaEncryption - 149:d=4 hl=2 l= 0 prim: NULL - 151:d=3 hl=3 l= 141 prim: BIT STRING - 295:d=1 hl=2 l= 13 cons: SEQUENCE - 297:d=2 hl=2 l= 9 prim: OBJECT :sha1WithRSAEncryption - 308:d=2 hl=2 l= 0 prim: NULL - 310:d=1 hl=3 l= 129 prim: BIT STRING - -I propose that we match OpenSSL's default self-signed certificates. I hypothesise -that they are the most common self-signed certificates. If this turns out not -to be the case, then we should use whatever the most common turns out to be. - -Certificate serial numbers - -Currently our generated certificate serial number is set to the number of -seconds since the epoch at the time of the certificate's creation. I propose -that we should ensure that our serial numbers are unrelated to the epoch, -since the generation methods are potentially recognizable as Tor-related. - -Instead, I propose that we use a randomly generated number that is -subsequently hashed with SHA-512 and then truncate the data to eight bytes[1]. - -Random sixteen byte values appear to be the high bound for serial number as -issued by Verisign and DigiCert. RapidSSL appears to be three bytes in length. -Others common byte lengths appear to be between one and four bytes. The default -OpenSSL certificates are eight bytes and we should use this length with our -self-signed certificates. - -This randomly generated serial number field may now serve as a covert channel -that signals to the client that the OR will not support TLS renegotiation; this -means that the client can expect to perform a V3 TLS handshake setup. -Otherwise, if the serial number is a reasonable time since the epoch, we should -assume the OR is using an earlier protocol version and hence that it expects -renegotiation. - -We also have a need to signal properties with our certificates for a possible -v3 handshake in the future. Therefore I propose that we match OpenSSL default -self-signed certificates (a 64-bit random number), but reserve the two least- -significant bits for signaling. For the moment, these two bits will be zero. - -This means that an attacker may be able to identify Tor certificates from default -OpenSSL certificates with a 75% probability. - -As a security note, care must be taken to ensure that supporting this -covert channel will not lead to an attacker having a method to downgrade client -behavior. This shouldn't be a risk because the TLS Finished message hashes over -all the bytes of the handshake, including the certificates. - -Certificate fingerprinting issues expressed as base64 encoding - -It appears that all deployed Tor certificates have the following strings in -common: - -MIIB -CCA -gAwIBAgIETU -ANBgkqhkiG9w0BAQUFADA -YDVQQDEx -3d3cu - -As expected these values correspond to specific ASN.1 OBJECT IDENTIFIER (OID) -properties (sha1WithRSAEncryption, commonName, etc) of how we generate our -certificates. - -As an illustrated example of the common bytes of all certificates used within -the Tor network within a single one hour window, I have replaced the actual -value with a wild card ('.') character here: - ------BEGIN CERTIFICATE----- -MIIB..CCA..gAwIBAgIETU....ANBgkqhkiG9w0BAQUFADA.M..w..YDVQQDEx.3 -d3cu............................................................ -................................................................ -................................................................ -................................................................ -................................................................ -................................................................ -................................................................ -................................................................ -........................... <--- Variable length and padding ------END CERTIFICATE----- - -This fine ascii art only illustrates the bytes that absolutely match in all -cases. In many cases, it's likely that there is a high probability for a given -byte to be only a small subset of choices. - -Using the above strings, the EFF's certificate observatory may trivially -discover all known relays, known bridges and unknown bridges in a single SQL -query. I propose that we ensure that we test our certificates to ensure that -they do not have these kinds of statistical similarities without ensuring -overlap with a very large cross section of the internet's certificates. - -Certificate dating and validity issues - -TLS certificates found in the wild are generally found to be long-lived; -they are frequently old and often even expired. The current Tor certificate -validity time is a very small time window starting at generation time and -ending shortly thereafter, as defined in or.h by MAX_SSL_KEY_LIFETIME -(2*60*60). - -I propose that the certificate validity time length is extended to a period of -twelve Earth months, possibly with a small random skew to be determined by the -implementer. Tor should randomly set the start date in the past or some -currently unspecified window of time before the current date. This would -more closely track the typical distribution of non-Tor TLS certificate -expiration times. - -The certificate values, such as expiration, should not be used for anything -relating to security; for example, if the OR presents an expired TLS -certificate, this does not imply that the client should terminate the -connection (as would be appropriate for an ordinary TLS implementation). -Rather, I propose we use a TOFU style expiration policy - the certificate -should never be trusted for more than a two hour window from first sighting. - -This policy should have two major impacts. The first is that an adversary will -have to perform a differential analysis of all certificates for a given IP -address rather than a single check. The second is that the server expiration -time is enforced by the client and confirmed by keys rotating in the consensus. - -The expiration time should not be a fixed time that is simple to calculate by -any Deep Packet Inspection device or it will become a new Tor TLS setup -fingerprint. - -Proposed certificate form - -The following output from openssl asn1parse results from the proposed -certificate generation algorithm. It matches the results of generating a -default self-signed certificate: - - 0:d=0 hl=4 l= 513 cons: SEQUENCE - 4:d=1 hl=4 l= 362 cons: SEQUENCE - 8:d=2 hl=2 l= 9 prim: INTEGER :DBF6B3B864FF7478 - 19:d=2 hl=2 l= 13 cons: SEQUENCE - 21:d=3 hl=2 l= 9 prim: OBJECT :sha1WithRSAEncryption - 32:d=3 hl=2 l= 0 prim: NULL - 34:d=2 hl=2 l= 69 cons: SEQUENCE - 36:d=3 hl=2 l= 11 cons: SET - 38:d=4 hl=2 l= 9 cons: SEQUENCE - 40:d=5 hl=2 l= 3 prim: OBJECT :countryName - 45:d=5 hl=2 l= 2 prim: PRINTABLESTRING :AU - 49:d=3 hl=2 l= 19 cons: SET - 51:d=4 hl=2 l= 17 cons: SEQUENCE - 53:d=5 hl=2 l= 3 prim: OBJECT :stateOrProvinceName - 58:d=5 hl=2 l= 10 prim: PRINTABLESTRING :Some-State - 70:d=3 hl=2 l= 33 cons: SET - 72:d=4 hl=2 l= 31 cons: SEQUENCE - 74:d=5 hl=2 l= 3 prim: OBJECT :organizationName - 79:d=5 hl=2 l= 24 prim: PRINTABLESTRING :Internet Widgits Pty Ltd - 105:d=2 hl=2 l= 30 cons: SEQUENCE - 107:d=3 hl=2 l= 13 prim: UTCTIME :110217011237Z - 122:d=3 hl=2 l= 13 prim: UTCTIME :120217011237Z - 137:d=2 hl=2 l= 69 cons: SEQUENCE - 139:d=3 hl=2 l= 11 cons: SET - 141:d=4 hl=2 l= 9 cons: SEQUENCE - 143:d=5 hl=2 l= 3 prim: OBJECT :countryName - 148:d=5 hl=2 l= 2 prim: PRINTABLESTRING :AU - 152:d=3 hl=2 l= 19 cons: SET - 154:d=4 hl=2 l= 17 cons: SEQUENCE - 156:d=5 hl=2 l= 3 prim: OBJECT :stateOrProvinceName - 161:d=5 hl=2 l= 10 prim: PRINTABLESTRING :Some-State - 173:d=3 hl=2 l= 33 cons: SET - 175:d=4 hl=2 l= 31 cons: SEQUENCE - 177:d=5 hl=2 l= 3 prim: OBJECT :organizationName - 182:d=5 hl=2 l= 24 prim: PRINTABLESTRING :Internet Widgits Pty Ltd - 208:d=2 hl=3 l= 159 cons: SEQUENCE - 211:d=3 hl=2 l= 13 cons: SEQUENCE - 213:d=4 hl=2 l= 9 prim: OBJECT :rsaEncryption - 224:d=4 hl=2 l= 0 prim: NULL - 226:d=3 hl=3 l= 141 prim: BIT STRING - 370:d=1 hl=2 l= 13 cons: SEQUENCE - 372:d=2 hl=2 l= 9 prim: OBJECT :sha1WithRSAEncryption - 383:d=2 hl=2 l= 0 prim: NULL - 385:d=1 hl=3 l= 129 prim: BIT STRING - - -Custom Certificates - -It should be possible for a Tor relay operator to use a specifically supplied -certificate and secret key. This will allow a relay or bridge operator to use a -certificate signed by any member of any geographically relevant certificate -authority racket; it will also allow for any other user-supplied certificate. -This may be desirable in some kinds of filtered networks or when attempting to -avoid attracting suspicion by blending in with the TLS web server certificate -crowd. - -Problematic Diffie–Hellman parameters - -We currently send a static Diffie–Hellman parameter, prime p (or “prime p -outlaw”) as specified in RFC2409 as part of the TLS Server Hello response. - -The use of this prime in TLS negotiations may, as a result, be filtered and -effectively banned by certain networks. We do not have to use this particular -prime in all cases. - -While amusing to have the power to make specific prime numbers into a new class -of numbers (cf. imaginary, irrational, illegal [3]) - our new friend prime p -outlaw is not required. - -The use of this prime in TLS negotiations may, as a result, be filtered and -effectively banned by certain networks. We do not have to use this particular -prime in all cases. - -I propose that the function to initialize and generate DH parameters be -split into two functions. - -First, init_dh_param() should be used only for OR-to-OR DH setup and -communication. Second, it is proposed that we create a new function -init_tls_dh_param() that will have a two-stage development process. - -The first stage init_tls_dh_param() will use the same prime that -Apache2.x [4] sends (or “dh1024_apache_p”), and this change should be -made immediately. This is a known good and safe prime number (p-1 / 2 -is also prime) that is currently not known to be blocked. - -The second stage init_tls_dh_param() should randomly generate a new prime on a -regular basis; this is designed to make the prime difficult to outlaw or -filter. Call this a shape-shifting or "Rakshasa" prime. This should be added -to the 0.2.3.x branch of Tor. This prime can be generated at setup or execution -time and probably does not need to be stored on disk. Rakshasa primes only -need to be generated by Tor relays as Tor clients will never send them. Such -a prime should absolutely not be shared between different Tor relays nor -should it ever be static after the 0.2.3.x release. - -As a security precaution, care must be taken to ensure that we do not generate -weak primes or known filtered primes. Both weak and filtered primes will -undermine the TLS connection security properties. OpenSSH solves this issue -dynamically in RFC 4419 [5] and may provide a solution that works reasonably -well for Tor. More research in this area including the applicability of -Miller-Rabin or AKS primality tests[6] will need to be analyzed and probably -added to Tor. - -Practical key size - -Currently we use a 1024 bit long RSA modulus. I propose that we increase the -RSA key size to 2048 as an additional channel to signal support for the V3 -handshake setup. 2048 appears to be the most common key size[0] above 1024. -Additionally, the increase in modulus size provides a reasonable security boost -with regard to key security properties. - -The implementer should increase the 1024 bit RSA modulus to 2048 bits. - -Possible future filtering nightmares - -At some point it may cost effective or politically feasible for a network -filter to simply block all signed or self-signed certificates without a known -valid CA trust chain. This will break many applications on the internet and -hopefully, our option for custom certificates will ensure that this step is -simply avoided by the censors. - -The Rakshasa prime approach may cause censors to specifically allow only -certain known and accepted DH parameters. - - -Appendix: Other issues - -What other obvious TLS certificate issues exist? What other static values are -present in the Tor TLS setup process? - -[0] http://archives.seul.org/or/dev/Jan-2011/msg00051.html -[1] http://archives.seul.org/or/dev/Feb-2011/msg00016.html -[2] http://archives.seul.org/or/dev/Feb-2011/msg00039.html -[3] To be fair this is hardly a new class of numbers. History is rife with - similar examples of inane authoritarian attempts at mathematical secrecy. - Probably the most dramatic example is the story of the pupil Hipassus of - Metapontum, pupil of the famous Pythagoras, who, legend goes, proved the - fact that Root2 cannot be expressed as a fraction of whole numbers (now - called an irrational number) and was assassinated for revealing this - secret. Further reading on the subject may be found on the Wikipedia: - http://en.wikipedia.org/wiki/Hippasus - -[4] httpd-2.2.17/modules/ss/ssl_engine_dh.c -[5] http://tools.ietf.org/html/rfc4419 -[6] http://archives.seul.org/or/dev/Jan-2011/msg00037.html

1 0

[torspec/master] TLS certificate and parameter normalization [DRAFT] as prop 179
by ioerror＠torproject.org 03 Mar '11

03 Mar '11

commit 3a85a3afac30dc8a2433a6e06508ac57c964f820 Author: Jacob Appelbaum <jacob(a)appelbaum.net> Date: Wed Mar 2 16:00:42 2011 -0800 TLS certificate and parameter normalization [DRAFT] as prop 179 --- proposals/000-index.txt | 2 + .../179-TLS-cert-and-parameter-normalization.txt | 360 ++++++++++++++++++++ .../xxx-TLS-cert-and-parameter-normalization.txt | 360 -------------------- 3 files changed, 362 insertions(+), 360 deletions(-) diff --git a/proposals/000-index.txt b/proposals/000-index.txt index b642703..be9b4e2 100644 --- a/proposals/000-index.txt +++ b/proposals/000-index.txt @@ -99,6 +99,7 @@ Proposals by number: 176 Proposed version-3 link handshake for Tor [OPEN] 177 Abstaining from votes on individual flags [OPEN] 178 Require majority of authorities to vote for consensus parameters [OPEN] +179 TLS certificate and parameter normalization [DRAFT] Proposals by status: @@ -112,6 +113,7 @@ Proposals by status: 149 Using data from NETINFO cells [for 0.2.1.x] 170 Configuration options regarding circuit building 175 Automatically promoting Tor clients to nodes + 179 TLS certificate and parameter normalization NEEDS-REVISION: 131 Help users to verify they are using Tor OPEN: diff --git a/proposals/179-TLS-cert-and-parameter-normalization.txt b/proposals/179-TLS-cert-and-parameter-normalization.txt new file mode 100644 index 0000000..acad79a --- /dev/null +++ b/proposals/179-TLS-cert-and-parameter-normalization.txt @@ -0,0 +1,360 @@ +Filename: 179-TLS-cert-and-parameter-normalization.txt +Title: TLS certificate and parameter normalization +Author: Jacob Appelbaum, Gladys Shufflebottom +Created: 16-Feb-2011 +Status: Draft + + + Draft spec for TLS certificate and handshake normalization + + + Overview + +Scope + +This is a document that proposes improvements to problems with Tor's +current TLS (Transport Layer Security) certificates and handshake that will +reduce the distinguishability of Tor traffic from other encrypted traffic that +uses TLS. It also addresses some of the possible fingerprinting attacks +possible against the current Tor TLS protocol setup process. + +Motivation and history + +Censorship is an arms race and this is a step forward in the defense +of Tor. This proposal outlines ideas to make it more difficult to +fingerprint and block Tor traffic. + +Goals + +This proposal intends to normalize or remove easy-to-predict or static +values in the Tor TLS certificates and with the Tor TLS setup process. +These values can be used as criteria for the automated classification of +encrypted traffic as Tor traffic. Network observers should not be able +to trivially detect Tor merely by receiving or observing the certificate +used or advertised by a Tor relay. I also propose the creation of +a hard-to-detect covert channel through which a server can signal that it +supports the third version ("V3") of the Tor handshake protocol. + +Non-Goals + +This document is not intended to solve all of the possible active or passive +Tor fingerprinting problems. This document focuses on removing distinctive +and predictable features of TLS protocol negotiation; we do not attempt to +make guarantees about resisting other kinds of fingerprinting of Tor +traffic, such as fingerprinting techniques related to timing or volume of +transmitted data. + + Implementation details + + +Certificate Issues + +The CN or commonName ASN1 field + +Tor generates certificates with a predictable commonName field; the +field is within a given range of values that is specific to Tor. +Additionally, the generated host names have other undesirable properties. +The host names typically do not resolve in the DNS because the domain +names referred to are generated at random. Although they are syntatically +valid, they usually refer to domains that have never been registered by +any domain name registrar. + +An example of the current commonName field: CN=www.s4ku5skci.net + +An example of OpenSSL’s asn1parse over a typical Tor certificate: + + 0:d=0 hl=4 l= 438 cons: SEQUENCE + 4:d=1 hl=4 l= 287 cons: SEQUENCE + 8:d=2 hl=2 l= 3 cons: cont [ 0 ] + 10:d=3 hl=2 l= 1 prim: INTEGER :02 + 13:d=2 hl=2 l= 4 prim: INTEGER :4D3C763A + 19:d=2 hl=2 l= 13 cons: SEQUENCE + 21:d=3 hl=2 l= 9 prim: OBJECT :sha1WithRSAEncryption + 32:d=3 hl=2 l= 0 prim: NULL + 34:d=2 hl=2 l= 35 cons: SEQUENCE + 36:d=3 hl=2 l= 33 cons: SET + 38:d=4 hl=2 l= 31 cons: SEQUENCE + 40:d=5 hl=2 l= 3 prim: OBJECT :commonName + 45:d=5 hl=2 l= 24 prim: PRINTABLESTRING :www.vsbsvwu5b4soh4wg.net + 71:d=2 hl=2 l= 30 cons: SEQUENCE + 73:d=3 hl=2 l= 13 prim: UTCTIME :110123184058Z + 88:d=3 hl=2 l= 13 prim: UTCTIME :110123204058Z + 103:d=2 hl=2 l= 28 cons: SEQUENCE + 105:d=3 hl=2 l= 26 cons: SET + 107:d=4 hl=2 l= 24 cons: SEQUENCE + 109:d=5 hl=2 l= 3 prim: OBJECT :commonName + 114:d=5 hl=2 l= 17 prim: PRINTABLESTRING :www.s4ku5skci.net + 133:d=2 hl=3 l= 159 cons: SEQUENCE + 136:d=3 hl=2 l= 13 cons: SEQUENCE + 138:d=4 hl=2 l= 9 prim: OBJECT :rsaEncryption + 149:d=4 hl=2 l= 0 prim: NULL + 151:d=3 hl=3 l= 141 prim: BIT STRING + 295:d=1 hl=2 l= 13 cons: SEQUENCE + 297:d=2 hl=2 l= 9 prim: OBJECT :sha1WithRSAEncryption + 308:d=2 hl=2 l= 0 prim: NULL + 310:d=1 hl=3 l= 129 prim: BIT STRING + +I propose that we match OpenSSL's default self-signed certificates. I hypothesise +that they are the most common self-signed certificates. If this turns out not +to be the case, then we should use whatever the most common turns out to be. + +Certificate serial numbers + +Currently our generated certificate serial number is set to the number of +seconds since the epoch at the time of the certificate's creation. I propose +that we should ensure that our serial numbers are unrelated to the epoch, +since the generation methods are potentially recognizable as Tor-related. + +Instead, I propose that we use a randomly generated number that is +subsequently hashed with SHA-512 and then truncate the data to eight bytes[1]. + +Random sixteen byte values appear to be the high bound for serial number as +issued by Verisign and DigiCert. RapidSSL appears to be three bytes in length. +Others common byte lengths appear to be between one and four bytes. The default +OpenSSL certificates are eight bytes and we should use this length with our +self-signed certificates. + +This randomly generated serial number field may now serve as a covert channel +that signals to the client that the OR will not support TLS renegotiation; this +means that the client can expect to perform a V3 TLS handshake setup. +Otherwise, if the serial number is a reasonable time since the epoch, we should +assume the OR is using an earlier protocol version and hence that it expects +renegotiation. + +We also have a need to signal properties with our certificates for a possible +v3 handshake in the future. Therefore I propose that we match OpenSSL default +self-signed certificates (a 64-bit random number), but reserve the two least- +significant bits for signaling. For the moment, these two bits will be zero. + +This means that an attacker may be able to identify Tor certificates from default +OpenSSL certificates with a 75% probability. + +As a security note, care must be taken to ensure that supporting this +covert channel will not lead to an attacker having a method to downgrade client +behavior. This shouldn't be a risk because the TLS Finished message hashes over +all the bytes of the handshake, including the certificates. + +Certificate fingerprinting issues expressed as base64 encoding + +It appears that all deployed Tor certificates have the following strings in +common: + +MIIB +CCA +gAwIBAgIETU +ANBgkqhkiG9w0BAQUFADA +YDVQQDEx +3d3cu + +As expected these values correspond to specific ASN.1 OBJECT IDENTIFIER (OID) +properties (sha1WithRSAEncryption, commonName, etc) of how we generate our +certificates. + +As an illustrated example of the common bytes of all certificates used within +the Tor network within a single one hour window, I have replaced the actual +value with a wild card ('.') character here: + +-----BEGIN CERTIFICATE----- +MIIB..CCA..gAwIBAgIETU....ANBgkqhkiG9w0BAQUFADA.M..w..YDVQQDEx.3 +d3cu............................................................ +................................................................ +................................................................ +................................................................ +................................................................ +................................................................ +................................................................ +................................................................ +........................... <--- Variable length and padding +-----END CERTIFICATE----- + +This fine ascii art only illustrates the bytes that absolutely match in all +cases. In many cases, it's likely that there is a high probability for a given +byte to be only a small subset of choices. + +Using the above strings, the EFF's certificate observatory may trivially +discover all known relays, known bridges and unknown bridges in a single SQL +query. I propose that we ensure that we test our certificates to ensure that +they do not have these kinds of statistical similarities without ensuring +overlap with a very large cross section of the internet's certificates. + +Certificate dating and validity issues + +TLS certificates found in the wild are generally found to be long-lived; +they are frequently old and often even expired. The current Tor certificate +validity time is a very small time window starting at generation time and +ending shortly thereafter, as defined in or.h by MAX_SSL_KEY_LIFETIME +(2*60*60). + +I propose that the certificate validity time length is extended to a period of +twelve Earth months, possibly with a small random skew to be determined by the +implementer. Tor should randomly set the start date in the past or some +currently unspecified window of time before the current date. This would +more closely track the typical distribution of non-Tor TLS certificate +expiration times. + +The certificate values, such as expiration, should not be used for anything +relating to security; for example, if the OR presents an expired TLS +certificate, this does not imply that the client should terminate the +connection (as would be appropriate for an ordinary TLS implementation). +Rather, I propose we use a TOFU style expiration policy - the certificate +should never be trusted for more than a two hour window from first sighting. + +This policy should have two major impacts. The first is that an adversary will +have to perform a differential analysis of all certificates for a given IP +address rather than a single check. The second is that the server expiration +time is enforced by the client and confirmed by keys rotating in the consensus. + +The expiration time should not be a fixed time that is simple to calculate by +any Deep Packet Inspection device or it will become a new Tor TLS setup +fingerprint. + +Proposed certificate form + +The following output from openssl asn1parse results from the proposed +certificate generation algorithm. It matches the results of generating a +default self-signed certificate: + + 0:d=0 hl=4 l= 513 cons: SEQUENCE + 4:d=1 hl=4 l= 362 cons: SEQUENCE + 8:d=2 hl=2 l= 9 prim: INTEGER :DBF6B3B864FF7478 + 19:d=2 hl=2 l= 13 cons: SEQUENCE + 21:d=3 hl=2 l= 9 prim: OBJECT :sha1WithRSAEncryption + 32:d=3 hl=2 l= 0 prim: NULL + 34:d=2 hl=2 l= 69 cons: SEQUENCE + 36:d=3 hl=2 l= 11 cons: SET + 38:d=4 hl=2 l= 9 cons: SEQUENCE + 40:d=5 hl=2 l= 3 prim: OBJECT :countryName + 45:d=5 hl=2 l= 2 prim: PRINTABLESTRING :AU + 49:d=3 hl=2 l= 19 cons: SET + 51:d=4 hl=2 l= 17 cons: SEQUENCE + 53:d=5 hl=2 l= 3 prim: OBJECT :stateOrProvinceName + 58:d=5 hl=2 l= 10 prim: PRINTABLESTRING :Some-State + 70:d=3 hl=2 l= 33 cons: SET + 72:d=4 hl=2 l= 31 cons: SEQUENCE + 74:d=5 hl=2 l= 3 prim: OBJECT :organizationName + 79:d=5 hl=2 l= 24 prim: PRINTABLESTRING :Internet Widgits Pty Ltd + 105:d=2 hl=2 l= 30 cons: SEQUENCE + 107:d=3 hl=2 l= 13 prim: UTCTIME :110217011237Z + 122:d=3 hl=2 l= 13 prim: UTCTIME :120217011237Z + 137:d=2 hl=2 l= 69 cons: SEQUENCE + 139:d=3 hl=2 l= 11 cons: SET + 141:d=4 hl=2 l= 9 cons: SEQUENCE + 143:d=5 hl=2 l= 3 prim: OBJECT :countryName + 148:d=5 hl=2 l= 2 prim: PRINTABLESTRING :AU + 152:d=3 hl=2 l= 19 cons: SET + 154:d=4 hl=2 l= 17 cons: SEQUENCE + 156:d=5 hl=2 l= 3 prim: OBJECT :stateOrProvinceName + 161:d=5 hl=2 l= 10 prim: PRINTABLESTRING :Some-State + 173:d=3 hl=2 l= 33 cons: SET + 175:d=4 hl=2 l= 31 cons: SEQUENCE + 177:d=5 hl=2 l= 3 prim: OBJECT :organizationName + 182:d=5 hl=2 l= 24 prim: PRINTABLESTRING :Internet Widgits Pty Ltd + 208:d=2 hl=3 l= 159 cons: SEQUENCE + 211:d=3 hl=2 l= 13 cons: SEQUENCE + 213:d=4 hl=2 l= 9 prim: OBJECT :rsaEncryption + 224:d=4 hl=2 l= 0 prim: NULL + 226:d=3 hl=3 l= 141 prim: BIT STRING + 370:d=1 hl=2 l= 13 cons: SEQUENCE + 372:d=2 hl=2 l= 9 prim: OBJECT :sha1WithRSAEncryption + 383:d=2 hl=2 l= 0 prim: NULL + 385:d=1 hl=3 l= 129 prim: BIT STRING + + +Custom Certificates + +It should be possible for a Tor relay operator to use a specifically supplied +certificate and secret key. This will allow a relay or bridge operator to use a +certificate signed by any member of any geographically relevant certificate +authority racket; it will also allow for any other user-supplied certificate. +This may be desirable in some kinds of filtered networks or when attempting to +avoid attracting suspicion by blending in with the TLS web server certificate +crowd. + +Problematic Diffie–Hellman parameters + +We currently send a static Diffie–Hellman parameter, prime p (or “prime p +outlaw”) as specified in RFC2409 as part of the TLS Server Hello response. + +The use of this prime in TLS negotiations may, as a result, be filtered and +effectively banned by certain networks. We do not have to use this particular +prime in all cases. + +While amusing to have the power to make specific prime numbers into a new class +of numbers (cf. imaginary, irrational, illegal [3]) - our new friend prime p +outlaw is not required. + +The use of this prime in TLS negotiations may, as a result, be filtered and +effectively banned by certain networks. We do not have to use this particular +prime in all cases. + +I propose that the function to initialize and generate DH parameters be +split into two functions. + +First, init_dh_param() should be used only for OR-to-OR DH setup and +communication. Second, it is proposed that we create a new function +init_tls_dh_param() that will have a two-stage development process. + +The first stage init_tls_dh_param() will use the same prime that +Apache2.x [4] sends (or “dh1024_apache_p”), and this change should be +made immediately. This is a known good and safe prime number (p-1 / 2 +is also prime) that is currently not known to be blocked. + +The second stage init_tls_dh_param() should randomly generate a new prime on a +regular basis; this is designed to make the prime difficult to outlaw or +filter. Call this a shape-shifting or "Rakshasa" prime. This should be added +to the 0.2.3.x branch of Tor. This prime can be generated at setup or execution +time and probably does not need to be stored on disk. Rakshasa primes only +need to be generated by Tor relays as Tor clients will never send them. Such +a prime should absolutely not be shared between different Tor relays nor +should it ever be static after the 0.2.3.x release. + +As a security precaution, care must be taken to ensure that we do not generate +weak primes or known filtered primes. Both weak and filtered primes will +undermine the TLS connection security properties. OpenSSH solves this issue +dynamically in RFC 4419 [5] and may provide a solution that works reasonably +well for Tor. More research in this area including the applicability of +Miller-Rabin or AKS primality tests[6] will need to be analyzed and probably +added to Tor. + +Practical key size + +Currently we use a 1024 bit long RSA modulus. I propose that we increase the +RSA key size to 2048 as an additional channel to signal support for the V3 +handshake setup. 2048 appears to be the most common key size[0] above 1024. +Additionally, the increase in modulus size provides a reasonable security boost +with regard to key security properties. + +The implementer should increase the 1024 bit RSA modulus to 2048 bits. + +Possible future filtering nightmares + +At some point it may cost effective or politically feasible for a network +filter to simply block all signed or self-signed certificates without a known +valid CA trust chain. This will break many applications on the internet and +hopefully, our option for custom certificates will ensure that this step is +simply avoided by the censors. + +The Rakshasa prime approach may cause censors to specifically allow only +certain known and accepted DH parameters. + + +Appendix: Other issues + +What other obvious TLS certificate issues exist? What other static values are +present in the Tor TLS setup process? + +[0] http://archives.seul.org/or/dev/Jan-2011/msg00051.html +[1] http://archives.seul.org/or/dev/Feb-2011/msg00016.html +[2] http://archives.seul.org/or/dev/Feb-2011/msg00039.html +[3] To be fair this is hardly a new class of numbers. History is rife with + similar examples of inane authoritarian attempts at mathematical secrecy. + Probably the most dramatic example is the story of the pupil Hipassus of + Metapontum, pupil of the famous Pythagoras, who, legend goes, proved the + fact that Root2 cannot be expressed as a fraction of whole numbers (now + called an irrational number) and was assassinated for revealing this + secret. Further reading on the subject may be found on the Wikipedia: + http://en.wikipedia.org/wiki/Hippasus + +[4] httpd-2.2.17/modules/ss/ssl_engine_dh.c +[5] http://tools.ietf.org/html/rfc4419 +[6] http://archives.seul.org/or/dev/Jan-2011/msg00037.html diff --git a/proposals/ideas/xxx-TLS-cert-and-parameter-normalization.txt b/proposals/ideas/xxx-TLS-cert-and-parameter-normalization.txt deleted file mode 100644 index 74704b1..0000000 --- a/proposals/ideas/xxx-TLS-cert-and-parameter-normalization.txt +++ /dev/null @@ -1,360 +0,0 @@ -Filename: xxx-TLS-cert-and-parameter-normalization.txt -Title: TLS certificate and parameter normalization -Author: Jacob Appelbaum, Gladys Shufflebottom -Created: 16-Feb-2011 -Status: Draft - - - Draft spec for TLS certificate and handshake normalization - - - Overview - -Scope - -This is a document that proposes improvements to problems with Tor's -current TLS (Transport Layer Security) certificates and handshake that will -reduce the distinguishability of Tor traffic from other encrypted traffic that -uses TLS. It also addresses some of the possible fingerprinting attacks -possible against the current Tor TLS protocol setup process. - -Motivation and history - -Censorship is an arms race and this is a step forward in the defense -of Tor. This proposal outlines ideas to make it more difficult to -fingerprint and block Tor traffic. - -Goals - -This proposal intends to normalize or remove easy-to-predict or static -values in the Tor TLS certificates and with the Tor TLS setup process. -These values can be used as criteria for the automated classification of -encrypted traffic as Tor traffic. Network observers should not be able -to trivially detect Tor merely by receiving or observing the certificate -used or advertised by a Tor relay. I also propose the creation of -a hard-to-detect covert channel through which a server can signal that it -supports the third version ("V3") of the Tor handshake protocol. - -Non-Goals - -This document is not intended to solve all of the possible active or passive -Tor fingerprinting problems. This document focuses on removing distinctive -and predictable features of TLS protocol negotiation; we do not attempt to -make guarantees about resisting other kinds of fingerprinting of Tor -traffic, such as fingerprinting techniques related to timing or volume of -transmitted data. - - Implementation details - - -Certificate Issues - -The CN or commonName ASN1 field - -Tor generates certificates with a predictable commonName field; the -field is within a given range of values that is specific to Tor. -Additionally, the generated host names have other undesirable properties. -The host names typically do not resolve in the DNS because the domain -names referred to are generated at random. Although they are syntatically -valid, they usually refer to domains that have never been registered by -any domain name registrar. - -An example of the current commonName field: CN=www.s4ku5skci.net - -An example of OpenSSL’s asn1parse over a typical Tor certificate: - - 0:d=0 hl=4 l= 438 cons: SEQUENCE - 4:d=1 hl=4 l= 287 cons: SEQUENCE - 8:d=2 hl=2 l= 3 cons: cont [ 0 ] - 10:d=3 hl=2 l= 1 prim: INTEGER :02 - 13:d=2 hl=2 l= 4 prim: INTEGER :4D3C763A - 19:d=2 hl=2 l= 13 cons: SEQUENCE - 21:d=3 hl=2 l= 9 prim: OBJECT :sha1WithRSAEncryption - 32:d=3 hl=2 l= 0 prim: NULL - 34:d=2 hl=2 l= 35 cons: SEQUENCE - 36:d=3 hl=2 l= 33 cons: SET - 38:d=4 hl=2 l= 31 cons: SEQUENCE - 40:d=5 hl=2 l= 3 prim: OBJECT :commonName - 45:d=5 hl=2 l= 24 prim: PRINTABLESTRING :www.vsbsvwu5b4soh4wg.net - 71:d=2 hl=2 l= 30 cons: SEQUENCE - 73:d=3 hl=2 l= 13 prim: UTCTIME :110123184058Z - 88:d=3 hl=2 l= 13 prim: UTCTIME :110123204058Z - 103:d=2 hl=2 l= 28 cons: SEQUENCE - 105:d=3 hl=2 l= 26 cons: SET - 107:d=4 hl=2 l= 24 cons: SEQUENCE - 109:d=5 hl=2 l= 3 prim: OBJECT :commonName - 114:d=5 hl=2 l= 17 prim: PRINTABLESTRING :www.s4ku5skci.net - 133:d=2 hl=3 l= 159 cons: SEQUENCE - 136:d=3 hl=2 l= 13 cons: SEQUENCE - 138:d=4 hl=2 l= 9 prim: OBJECT :rsaEncryption - 149:d=4 hl=2 l= 0 prim: NULL - 151:d=3 hl=3 l= 141 prim: BIT STRING - 295:d=1 hl=2 l= 13 cons: SEQUENCE - 297:d=2 hl=2 l= 9 prim: OBJECT :sha1WithRSAEncryption - 308:d=2 hl=2 l= 0 prim: NULL - 310:d=1 hl=3 l= 129 prim: BIT STRING - -I propose that we match OpenSSL's default self-signed certificates. I hypothesise -that they are the most common self-signed certificates. If this turns out not -to be the case, then we should use whatever the most common turns out to be. - -Certificate serial numbers - -Currently our generated certificate serial number is set to the number of -seconds since the epoch at the time of the certificate's creation. I propose -that we should ensure that our serial numbers are unrelated to the epoch, -since the generation methods are potentially recognizable as Tor-related. - -Instead, I propose that we use a randomly generated number that is -subsequently hashed with SHA-512 and then truncate the data to eight bytes[1]. - -Random sixteen byte values appear to be the high bound for serial number as -issued by Verisign and DigiCert. RapidSSL appears to be three bytes in length. -Others common byte lengths appear to be between one and four bytes. The default -OpenSSL certificates are eight bytes and we should use this length with our -self-signed certificates. - -This randomly generated serial number field may now serve as a covert channel -that signals to the client that the OR will not support TLS renegotiation; this -means that the client can expect to perform a V3 TLS handshake setup. -Otherwise, if the serial number is a reasonable time since the epoch, we should -assume the OR is using an earlier protocol version and hence that it expects -renegotiation. - -We also have a need to signal properties with our certificates for a possible -v3 handshake in the future. Therefore I propose that we match OpenSSL default -self-signed certificates (a 64-bit random number), but reserve the two least- -significant bits for signaling. For the moment, these two bits will be zero. - -This means that an attacker may be able to identify Tor certificates from default -OpenSSL certificates with a 75% probability. - -As a security note, care must be taken to ensure that supporting this -covert channel will not lead to an attacker having a method to downgrade client -behavior. This shouldn't be a risk because the TLS Finished message hashes over -all the bytes of the handshake, including the certificates. - -Certificate fingerprinting issues expressed as base64 encoding - -It appears that all deployed Tor certificates have the following strings in -common: - -MIIB -CCA -gAwIBAgIETU -ANBgkqhkiG9w0BAQUFADA -YDVQQDEx -3d3cu - -As expected these values correspond to specific ASN.1 OBJECT IDENTIFIER (OID) -properties (sha1WithRSAEncryption, commonName, etc) of how we generate our -certificates. - -As an illustrated example of the common bytes of all certificates used within -the Tor network within a single one hour window, I have replaced the actual -value with a wild card ('.') character here: - ------BEGIN CERTIFICATE----- -MIIB..CCA..gAwIBAgIETU....ANBgkqhkiG9w0BAQUFADA.M..w..YDVQQDEx.3 -d3cu............................................................ -................................................................ -................................................................ -................................................................ -................................................................ -................................................................ -................................................................ -................................................................ -........................... <--- Variable length and padding ------END CERTIFICATE----- - -This fine ascii art only illustrates the bytes that absolutely match in all -cases. In many cases, it's likely that there is a high probability for a given -byte to be only a small subset of choices. - -Using the above strings, the EFF's certificate observatory may trivially -discover all known relays, known bridges and unknown bridges in a single SQL -query. I propose that we ensure that we test our certificates to ensure that -they do not have these kinds of statistical similarities without ensuring -overlap with a very large cross section of the internet's certificates. - -Certificate dating and validity issues - -TLS certificates found in the wild are generally found to be long-lived; -they are frequently old and often even expired. The current Tor certificate -validity time is a very small time window starting at generation time and -ending shortly thereafter, as defined in or.h by MAX_SSL_KEY_LIFETIME -(2*60*60). - -I propose that the certificate validity time length is extended to a period of -twelve Earth months, possibly with a small random skew to be determined by the -implementer. Tor should randomly set the start date in the past or some -currently unspecified window of time before the current date. This would -more closely track the typical distribution of non-Tor TLS certificate -expiration times. - -The certificate values, such as expiration, should not be used for anything -relating to security; for example, if the OR presents an expired TLS -certificate, this does not imply that the client should terminate the -connection (as would be appropriate for an ordinary TLS implementation). -Rather, I propose we use a TOFU style expiration policy - the certificate -should never be trusted for more than a two hour window from first sighting. - -This policy should have two major impacts. The first is that an adversary will -have to perform a differential analysis of all certificates for a given IP -address rather than a single check. The second is that the server expiration -time is enforced by the client and confirmed by keys rotating in the consensus. - -The expiration time should not be a fixed time that is simple to calculate by -any Deep Packet Inspection device or it will become a new Tor TLS setup -fingerprint. - -Proposed certificate form - -The following output from openssl asn1parse results from the proposed -certificate generation algorithm. It matches the results of generating a -default self-signed certificate: - - 0:d=0 hl=4 l= 513 cons: SEQUENCE - 4:d=1 hl=4 l= 362 cons: SEQUENCE - 8:d=2 hl=2 l= 9 prim: INTEGER :DBF6B3B864FF7478 - 19:d=2 hl=2 l= 13 cons: SEQUENCE - 21:d=3 hl=2 l= 9 prim: OBJECT :sha1WithRSAEncryption - 32:d=3 hl=2 l= 0 prim: NULL - 34:d=2 hl=2 l= 69 cons: SEQUENCE - 36:d=3 hl=2 l= 11 cons: SET - 38:d=4 hl=2 l= 9 cons: SEQUENCE - 40:d=5 hl=2 l= 3 prim: OBJECT :countryName - 45:d=5 hl=2 l= 2 prim: PRINTABLESTRING :AU - 49:d=3 hl=2 l= 19 cons: SET - 51:d=4 hl=2 l= 17 cons: SEQUENCE - 53:d=5 hl=2 l= 3 prim: OBJECT :stateOrProvinceName - 58:d=5 hl=2 l= 10 prim: PRINTABLESTRING :Some-State - 70:d=3 hl=2 l= 33 cons: SET - 72:d=4 hl=2 l= 31 cons: SEQUENCE - 74:d=5 hl=2 l= 3 prim: OBJECT :organizationName - 79:d=5 hl=2 l= 24 prim: PRINTABLESTRING :Internet Widgits Pty Ltd - 105:d=2 hl=2 l= 30 cons: SEQUENCE - 107:d=3 hl=2 l= 13 prim: UTCTIME :110217011237Z - 122:d=3 hl=2 l= 13 prim: UTCTIME :120217011237Z - 137:d=2 hl=2 l= 69 cons: SEQUENCE - 139:d=3 hl=2 l= 11 cons: SET - 141:d=4 hl=2 l= 9 cons: SEQUENCE - 143:d=5 hl=2 l= 3 prim: OBJECT :countryName - 148:d=5 hl=2 l= 2 prim: PRINTABLESTRING :AU - 152:d=3 hl=2 l= 19 cons: SET - 154:d=4 hl=2 l= 17 cons: SEQUENCE - 156:d=5 hl=2 l= 3 prim: OBJECT :stateOrProvinceName - 161:d=5 hl=2 l= 10 prim: PRINTABLESTRING :Some-State - 173:d=3 hl=2 l= 33 cons: SET - 175:d=4 hl=2 l= 31 cons: SEQUENCE - 177:d=5 hl=2 l= 3 prim: OBJECT :organizationName - 182:d=5 hl=2 l= 24 prim: PRINTABLESTRING :Internet Widgits Pty Ltd - 208:d=2 hl=3 l= 159 cons: SEQUENCE - 211:d=3 hl=2 l= 13 cons: SEQUENCE - 213:d=4 hl=2 l= 9 prim: OBJECT :rsaEncryption - 224:d=4 hl=2 l= 0 prim: NULL - 226:d=3 hl=3 l= 141 prim: BIT STRING - 370:d=1 hl=2 l= 13 cons: SEQUENCE - 372:d=2 hl=2 l= 9 prim: OBJECT :sha1WithRSAEncryption - 383:d=2 hl=2 l= 0 prim: NULL - 385:d=1 hl=3 l= 129 prim: BIT STRING - - -Custom Certificates - -It should be possible for a Tor relay operator to use a specifically supplied -certificate and secret key. This will allow a relay or bridge operator to use a -certificate signed by any member of any geographically relevant certificate -authority racket; it will also allow for any other user-supplied certificate. -This may be desirable in some kinds of filtered networks or when attempting to -avoid attracting suspicion by blending in with the TLS web server certificate -crowd. - -Problematic Diffie–Hellman parameters - -We currently send a static Diffie–Hellman parameter, prime p (or “prime p -outlaw”) as specified in RFC2409 as part of the TLS Server Hello response. - -The use of this prime in TLS negotiations may, as a result, be filtered and -effectively banned by certain networks. We do not have to use this particular -prime in all cases. - -While amusing to have the power to make specific prime numbers into a new class -of numbers (cf. imaginary, irrational, illegal [3]) - our new friend prime p -outlaw is not required. - -The use of this prime in TLS negotiations may, as a result, be filtered and -effectively banned by certain networks. We do not have to use this particular -prime in all cases. - -I propose that the function to initialize and generate DH parameters be -split into two functions. - -First, init_dh_param() should be used only for OR-to-OR DH setup and -communication. Second, it is proposed that we create a new function -init_tls_dh_param() that will have a two-stage development process. - -The first stage init_tls_dh_param() will use the same prime that -Apache2.x [4] sends (or “dh1024_apache_p”), and this change should be -made immediately. This is a known good and safe prime number (p-1 / 2 -is also prime) that is currently not known to be blocked. - -The second stage init_tls_dh_param() should randomly generate a new prime on a -regular basis; this is designed to make the prime difficult to outlaw or -filter. Call this a shape-shifting or "Rakshasa" prime. This should be added -to the 0.2.3.x branch of Tor. This prime can be generated at setup or execution -time and probably does not need to be stored on disk. Rakshasa primes only -need to be generated by Tor relays as Tor clients will never send them. Such -a prime should absolutely not be shared between different Tor relays nor -should it ever be static after the 0.2.3.x release. - -As a security precaution, care must be taken to ensure that we do not generate -weak primes or known filtered primes. Both weak and filtered primes will -undermine the TLS connection security properties. OpenSSH solves this issue -dynamically in RFC 4419 [5] and may provide a solution that works reasonably -well for Tor. More research in this area including the applicability of -Miller-Rabin or AKS primality tests[6] will need to be analyzed and probably -added to Tor. - -Practical key size - -Currently we use a 1024 bit long RSA modulus. I propose that we increase the -RSA key size to 2048 as an additional channel to signal support for the V3 -handshake setup. 2048 appears to be the most common key size[0] above 1024. -Additionally, the increase in modulus size provides a reasonable security boost -with regard to key security properties. - -The implementer should increase the 1024 bit RSA modulus to 2048 bits. - -Possible future filtering nightmares - -At some point it may cost effective or politically feasible for a network -filter to simply block all signed or self-signed certificates without a known -valid CA trust chain. This will break many applications on the internet and -hopefully, our option for custom certificates will ensure that this step is -simply avoided by the censors. - -The Rakshasa prime approach may cause censors to specifically allow only -certain known and accepted DH parameters. - - -Appendix: Other issues - -What other obvious TLS certificate issues exist? What other static values are -present in the Tor TLS setup process? - -[0] http://archives.seul.org/or/dev/Jan-2011/msg00051.html -[1] http://archives.seul.org/or/dev/Feb-2011/msg00016.html -[2] http://archives.seul.org/or/dev/Feb-2011/msg00039.html -[3] To be fair this is hardly a new class of numbers. History is rife with - similar examples of inane authoritarian attempts at mathematical secrecy. - Probably the most dramatic example is the story of the pupil Hipassus of - Metapontum, pupil of the famous Pythagoras, who, legend goes, proved the - fact that Root2 cannot be expressed as a fraction of whole numbers (now - called an irrational number) and was assassinated for revealing this - secret. Further reading on the subject may be found on the Wikipedia: - http://en.wikipedia.org/wiki/Hippasus - -[4] httpd-2.2.17/modules/ss/ssl_engine_dh.c -[5] http://tools.ietf.org/html/rfc4419 -[6] http://archives.seul.org/or/dev/Jan-2011/msg00037.html

1 0

r24292: {website} Updated page content to accurately reflect features for Orbo (website/trunk/docs/en)
by Nathan Freitas 02 Mar '11

02 Mar '11

Author: n8fr8 Date: 2011-03-02 21:16:07 +0000 (Wed, 02 Mar 2011) New Revision: 24292 Modified: website/trunk/docs/en/android.wml Log: Updated page content to accurately reflect features for Orbot 1.0.4.1. Modified information on related apps, supported devices, roms, etc. Modified: website/trunk/docs/en/android.wml =================================================================== --- website/trunk/docs/en/android.wml 2011-03-02 21:15:28 UTC (rev 24291) +++ website/trunk/docs/en/android.wml 2011-03-02 21:16:07 UTC (rev 24292) @@ -25,7 +25,7 @@ Orbot contains Tor, libevent and privoxy. Orbot provides a local HTTP proxy and the standard SOCKS4A/SOCKS5 proxy interfaces into the Tor network. Orbot has the ability to transparently torify all of the TCP traffic on your Android - device when it has the correct permissions. + device when it has the correct permissions and system libraries. <a id="QrCode"></a> @@ -96,26 +96,22 @@ - For standard Android 1.x devices (G1, MyTouch3G, Hero, Droid Eris, Cliq, Moment): + For standard Android 1.x devices: <ul> - <li>The “ProxySurf” browser available in the Android Market allows for use - of a proxy. Simply set the HTTP Proxy to '127.0.0.1' and port '8118'. - This only proxies some traffic and should not be considered secure. + <li>The Orweb browser available in the Android Market integrates directly with Orbot, and offers a number of other privacy-oriented features. </li> - <li>For Instant Messsaging, try “Beem” in the market, and set the Proxy to SOCKS5 '127.0.0.1' and port '9050'.</li> + <li>For Instant Messsaging, try <a href="https://guardianproject.info/apps/gibber">Gibberbot</a>, which includes support for connecting via Tor and Off-the-Record encryption.</li> </ul> - For Android 2.x devices: Droid, Nexus One + For Android 2.x devices: <ul> - <li>You must root your device for Orbot to transparently proxy all TCP traffic.</li> - <li>For non-modified and non-rooted phones, you'll want to manually configure - your specific applications.</li> - <li>If you root your device, whether it is 1.x or 2.x based, Orbot will - automatically, transparently proxy all web traffic on port 80 and 443 - and all DNS requests. This includes the built-in Browser, Gmail, YouTube, - Maps and any other application that uses standard web traffic.</li> + <li>You must root your device and update the firmware to an iptables-capable ROM for Orbot to transparently proxy all TCP traffic.</li> + <li>For non-modified and non-rooted phones, you'll want to manually configure HTTP or SOCKS proxy settings for specific applications.</li> + <li>If you root your device and install an iptables-capable ROM (such as <a href="http://cyanogenmod.com">Cyanogen<a/>), Orbot can transparently proxy traffic on an app-by-app basis through Tor.</li> + <li>You can also install Firefox for Android from the market with the <a href="https://guardianproject.info/apps/proxymob">ProxyMob Add-on</a> or install the text-only “NDBrowser” from the Android Market. Both of these solutions all routing of web access through Tor on standard, un-rooted devices. + <li>For Instant Messaging, try <a href="https://guardianproject.info/apps/gibber">Gibberbot</a>, which includes support for connecting via Tor and Off-the-Record encryption.</li> </ul> <a id="Source"></a>

1 0