- tor-commits - lists.torproject.org

[metrics-web/master] Use metrics-lib to parse rsync'ed torperf files.
by karsten＠torproject.org 01 Jun '12

01 Jun '12

commit ccb8bfe5be8d41bea824239f54fddf64ba0f9a41 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jun 1 14:35:19 2012 +0200 Use metrics-lib to parse rsync'ed torperf files. --- config.template | 2 +- src/org/torproject/ernie/cron/Configuration.java | 2 +- .../torproject/ernie/cron/TorperfProcessor.java | 99 ++++++++++---------- 3 files changed, 53 insertions(+), 50 deletions(-) diff --git a/config.template b/config.template index 97cebf8..528b36f 100644 --- a/config.template +++ b/config.template @@ -43,7 +43,7 @@ #ImportWriteTorperfStats 0 # ## Relative path to directory to import torperf results from -#TorperfDirectory torperf/ +#TorperfDirectory in/torperf/ # ## Process GetTor stats and import them into the database #ProcessGetTorStats 0 diff --git a/src/org/torproject/ernie/cron/Configuration.java b/src/org/torproject/ernie/cron/Configuration.java index 92a11b2..5cdadaf 100644 --- a/src/org/torproject/ernie/cron/Configuration.java +++ b/src/org/torproject/ernie/cron/Configuration.java @@ -29,7 +29,7 @@ public class Configuration { private String relayDescriptorRawFilesDirectory = "pg-import/"; private boolean writeBridgeStats = false; private boolean importWriteTorperfStats = false; - private String torperfDirectory = "torperf/"; + private String torperfDirectory = "in/torperf/"; private boolean processGetTorStats = false; private String getTorDirectory = "in/gettor/"; private String exoneraTorDatabaseJdbc = "jdbc:postgresql:" diff --git a/src/org/torproject/ernie/cron/TorperfProcessor.java b/src/org/torproject/ernie/cron/TorperfProcessor.java index 377fede..ff64fb0 100644 --- a/src/org/torproject/ernie/cron/TorperfProcessor.java +++ b/src/org/torproject/ernie/cron/TorperfProcessor.java @@ -24,12 +24,17 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedMap; -import java.util.Stack; import java.util.TimeZone; import java.util.TreeMap; import java.util.logging.Level; import java.util.logging.Logger; +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.DescriptorFile; +import org.torproject.descriptor.DescriptorReader; +import org.torproject.descriptor.DescriptorSourceFactory; +import org.torproject.descriptor.TorperfResult; + public class TorperfProcessor { public TorperfProcessor(File torperfDirectory, File statsDirectory, String connectionURL) { @@ -80,58 +85,56 @@ public class TorperfProcessor { } if (torperfDirectory.exists()) { logger.fine("Importing files in " + torperfDirectory + "/..."); - Stack<File> filesInInputDir = new Stack<File>(); - filesInInputDir.add(torperfDirectory); - while (!filesInInputDir.isEmpty()) { - File pop = filesInInputDir.pop(); - if (pop.isDirectory()) { - for (File f : pop.listFiles()) { - filesInInputDir.add(f); + DescriptorReader descriptorReader = + DescriptorSourceFactory.createDescriptorReader(); + descriptorReader.addDirectory(torperfDirectory); + descriptorReader.setExcludeFiles(new File(statsDirectory, + "torperf-history")); + Iterator<DescriptorFile> descriptorFiles = + descriptorReader.readDescriptors(); + while (descriptorFiles.hasNext()) { + DescriptorFile descriptorFile = descriptorFiles.next(); + if (descriptorFile.getException() != null) { + logger.log(Level.FINE, "Error parsing file.", + descriptorFile.getException()); + continue; + } + for (Descriptor descriptor : descriptorFile.getDescriptors()) { + if (!(descriptor instanceof TorperfResult)) { + continue; } - } else { - String source = pop.getName().substring(0, - pop.getName().indexOf(".")); - String size = pop.getName().split("-")[1]; - long receivedBytes = 1L; - if (pop.getName().endsWith("kb.data")) { - receivedBytes *= 1024L; - } else if (pop.getName().endsWith("mb.data")) { - receivedBytes *= 1024L * 1024L; + TorperfResult result = (TorperfResult) descriptor; + String source = result.getSource(); + long fileSize = result.getFileSize(); + if (fileSize == 51200) { + source += "-50kb"; + } else if (fileSize == 1048576) { + source += "-1mb"; + } else if (fileSize == 5242880) { + source += "-5mb"; } else { - // not a valid .data file + logger.fine("Unexpected file size '" + fileSize + + "'. Skipping."); continue; } - receivedBytes *= Long.parseLong(size.substring(0, - size.length() - "xb.data".length())); - BufferedReader br = new BufferedReader(new FileReader(pop)); - String line = null; - while ((line = br.readLine()) != null) { - String[] parts = line.split(" "); - // remove defective lines as they occurred on gabelmoo - if (parts.length >= 20 && parts[0].length() == 10) { - long startSec = Long.parseLong(parts[0]); - String dateTime = formatter.format(startSec * 1000L); - long completeMillis = Long.parseLong(parts[16]) - * 1000L + Long.parseLong(parts[17]) / 1000L - - Long.parseLong(parts[0]) * 1000L - + Long.parseLong(parts[1]) / 1000L; - String key = source + "," + dateTime; - String value = key; - if ((parts.length == 20 && parts[16].equals("0")) || - (parts.length >= 21 && parts[20].equals("1"))) { - value += ",-2"; // -2 for timeout - } else if (Long.parseLong(parts[19]) < receivedBytes) { - value += ",-1"; // -1 for failure - } else { - value += "," + completeMillis; - } - if (!rawObs.containsKey(key)) { - rawObs.put(key, value); - addedRawObs++; - } - } + String dateTime = formatter.format(result.getStartMillis()); + long completeMillis = result.getDataCompleteMillis() + - result.getStartMillis(); + String key = source + "," + dateTime; + String value = key; + if ((result.didTimeout() == null && + result.getDataCompleteMillis() < 1) || + (result.didTimeout() != null && result.didTimeout())) { + value += ",-2"; // -2 for timeout + } else if (result.getReadBytes() < fileSize) { + value += ",-1"; // -1 for failure + } else { + value += "," + completeMillis; + } + if (!rawObs.containsKey(key)) { + rawObs.put(key, value); + addedRawObs++; } - br.close(); } } logger.fine("Finished importing files in " + torperfDirectory

1 0

[metrics-web/master] Change default gettor directory.
by karsten＠torproject.org 01 Jun '12

01 Jun '12

commit 43bdeb612f016a117f4c5d67c5f1a63cb56db589 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jun 1 14:32:59 2012 +0200 Change default gettor directory. --- config.template | 2 +- src/org/torproject/ernie/cron/Configuration.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config.template b/config.template index 9760b56..97cebf8 100644 --- a/config.template +++ b/config.template @@ -49,7 +49,7 @@ #ProcessGetTorStats 0 # ## Relative path to directory where to find GetTor stats -#GetTorDirectory gettor/ +#GetTorDirectory in/gettor/ # ## JDBC string for ExoneraTor database #ExoneraTorDatabaseJdbc jdbc:postgresql://localhost/exonerator?user=metrics&password=password diff --git a/src/org/torproject/ernie/cron/Configuration.java b/src/org/torproject/ernie/cron/Configuration.java index 82a004b..92a11b2 100644 --- a/src/org/torproject/ernie/cron/Configuration.java +++ b/src/org/torproject/ernie/cron/Configuration.java @@ -31,7 +31,7 @@ public class Configuration { private boolean importWriteTorperfStats = false; private String torperfDirectory = "torperf/"; private boolean processGetTorStats = false; - private String getTorDirectory = "gettor/"; + private String getTorDirectory = "in/gettor/"; private String exoneraTorDatabaseJdbc = "jdbc:postgresql:" + "//localhost/exonerator?user=metrics&password=password"; private String exoneraTorImportDirectory = "exonerator-import/";

1 0

[metrics-db/master] Provide torperf files via rsync.
by karsten＠torproject.org 01 Jun '12

01 Jun '12

commit 1b62b6cdd9c9f7d8b76e24196577d6777ee53692 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jun 1 14:02:54 2012 +0200 Provide torperf files via rsync. --- src/org/torproject/ernie/db/Main.java | 2 + src/org/torproject/ernie/db/RsyncDataProvider.java | 23 ++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/org/torproject/ernie/db/Main.java b/src/org/torproject/ernie/db/Main.java index c0f100a..3fd6656 100644 --- a/src/org/torproject/ernie/db/Main.java +++ b/src/org/torproject/ernie/db/Main.java @@ -155,6 +155,8 @@ public class Main { config.getDownloadExitList(), !config.getDownloadGetTorStats() ? null : new File(config.getGetTorDirectory()), + !config.getProcessTorperfFiles() ? null : + new File(config.getTorperfOutputDirectory()), new File(config.getRsyncDirectory())); } diff --git a/src/org/torproject/ernie/db/RsyncDataProvider.java b/src/org/torproject/ernie/db/RsyncDataProvider.java index dac0249..4cb8e99 100644 --- a/src/org/torproject/ernie/db/RsyncDataProvider.java +++ b/src/org/torproject/ernie/db/RsyncDataProvider.java @@ -22,7 +22,7 @@ public class RsyncDataProvider { File sanitizedBridgesWriteDirectory, File sanitizedAssignmentsDirectory, boolean downloadExitList, File getTorDirectory, - File rsyncDirectory) { + File torperfOutputDirectory, File rsyncDirectory) { /* Initialize logger. */ Logger logger = Logger.getLogger(RsyncDataProvider.class.getName()); @@ -154,6 +154,26 @@ public class RsyncDataProvider { + fileNamesInRsync.size() + " files left in " + rsyncDirectory.getAbsolutePath() + "."); + /* Copy Torperf files. */ + if (torperfOutputDirectory != null) { + files.add(torperfOutputDirectory); + while (!files.isEmpty()) { + File pop = files.pop(); + if (pop.isDirectory()) { + files.addAll(Arrays.asList(pop.listFiles())); + } else if (pop.getName().endsWith(".tpf") && + pop.lastModified() >= cutOffMillis) { + String fileName = pop.getName(); + this.copyFile(pop, new File(rsyncDirectory, + "torperf/" + fileName)); + fileNamesInRsync.remove(pop.getName()); + } + } + } + logger.info("After copying Torperf files, there are still " + + fileNamesInRsync.size() + " files left in " + + rsyncDirectory.getAbsolutePath() + "."); + /* Copy GetTor stats. */ if (getTorDirectory != null) { String getTorFileName = "gettor_stats.txt"; @@ -165,7 +185,6 @@ public class RsyncDataProvider { fileNamesInRsync.remove(getTorFileName); } } - logger.info("After copying the GetTor stats file, there are still " + fileNamesInRsync.size() + " files left in " + rsyncDirectory.getAbsolutePath() + ".");

1 0

[metrics-web/master] Use metrics-lib to parse the GetTor stats file.
by karsten＠torproject.org 01 Jun '12

01 Jun '12

commit eea4d575f65799f8d57e1b42257e01e685d9309e Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jun 1 11:46:26 2012 +0200 Use metrics-lib to parse the GetTor stats file. --- src/org/torproject/ernie/cron/GetTorProcessor.java | 79 ++++++++----------- 1 files changed, 33 insertions(+), 46 deletions(-) diff --git a/src/org/torproject/ernie/cron/GetTorProcessor.java b/src/org/torproject/ernie/cron/GetTorProcessor.java index a9ce00b..34190b6 100644 --- a/src/org/torproject/ernie/cron/GetTorProcessor.java +++ b/src/org/torproject/ernie/cron/GetTorProcessor.java @@ -2,19 +2,16 @@ * See LICENSE for licensing information */ package org.torproject.ernie.cron; -import java.io.BufferedReader; import java.io.File; -import java.io.FileReader; -import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.HashMap; +import java.util.Iterator; import java.util.Map; import java.util.SortedMap; import java.util.SortedSet; @@ -23,6 +20,12 @@ import java.util.TreeSet; import java.util.logging.Level; import java.util.logging.Logger; +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.DescriptorFile; +import org.torproject.descriptor.DescriptorReader; +import org.torproject.descriptor.DescriptorSourceFactory; +import org.torproject.descriptor.GetTorStatistics; + public class GetTorProcessor { public GetTorProcessor(File getTorDirectory, String connectionURL) { @@ -38,53 +41,37 @@ public class GetTorProcessor { SortedSet<String> columns = new TreeSet<String>(); SortedMap<String, Map<String, Integer>> data = new TreeMap<String, Map<String, Integer>>(); - try { - logger.fine("Parsing GetTor stats..."); - BufferedReader br = new BufferedReader(new FileReader(getTorFile)); - String line = null; - while ((line = br.readLine()) != null) { - if (line.startsWith("@type ")) { - if (!line.startsWith("@type gettor 1.")) { - logger.warning("Wrong descriptor type: '" + line + "'. " - + "Aborting."); - break; + + logger.fine("Importing GetTor stats files in directory " + + getTorDirectory + "/..."); + DescriptorReader reader = + DescriptorSourceFactory.createDescriptorReader(); + reader.addDirectory(getTorDirectory); + Iterator<DescriptorFile> descriptorFiles = reader.readDescriptors(); + while (descriptorFiles.hasNext()) { + DescriptorFile descriptorFile = descriptorFiles.next(); + if (descriptorFile.getException() != null) { + logger.log(Level.WARNING, "Could not parse descriptor file '" + + descriptorFile.getFileName() + "'. Skipping.", + descriptorFile.getException()); + continue; + } + if (descriptorFile.getDescriptors() != null) { + for (Descriptor descriptor : descriptorFile.getDescriptors()) { + if (!(descriptor instanceof GetTorStatistics)) { + continue; } - continue; - } - String[] parts = line.split(" "); - String date = parts[0]; - try { - dateFormat.parse(date); - } catch (ParseException e) { - logger.warning("Illegal line in GetTor stats file: '" + line - + "'. Skipping."); - continue; - } - Map<String, Integer> obs = new HashMap<String, Integer>(); - for (int i = 2; i < parts.length; i++) { - String[] partParts = parts[i].split(":"); - if (partParts.length != 2) { - logger.warning("Illegal line in GetTor stats file: '" + line - + "'. Skipping."); - obs = null; - break; + GetTorStatistics stats = (GetTorStatistics) descriptor; + String date = dateFormat.format(stats.getDateMillis()); + Map<String, Integer> obs = new HashMap<String, Integer>(); + for (Map.Entry<String, Integer> e : + stats.getDownloadedPackages().entrySet()) { + columns.add(e.getKey().toLowerCase()); + obs.put(e.getKey().toLowerCase(), e.getValue()); } - String key = partParts[0].toLowerCase(); - Integer value = new Integer(partParts[1]); - columns.add(key); - obs.put(key, value); - } - if (obs != null) { data.put(date, obs); } } - br.close(); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed parsing GetTor stats!", e); - return; - } catch (NumberFormatException e) { - logger.log(Level.WARNING, "Failed parsing GetTor stats!", e); - return; } /* Write results to database. */

1 0

[metrics-lib/master] Support parsing GetTor statistics files.
by karsten＠torproject.org 01 Jun '12

01 Jun '12

commit a3d89ee7886a18f3bddc1efc66bf47e37c082d53 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jun 1 11:45:06 2012 +0200 Support parsing GetTor statistics files. --- .../torproject/descriptor/GetTorStatistics.java | 13 +++ .../torproject/descriptor/impl/DescriptorImpl.java | 3 + .../descriptor/impl/GetTorStatisticsImpl.java | 78 ++++++++++++++++++++ .../descriptor/impl/NetworkStatusEntryImpl.java | 2 +- .../torproject/descriptor/impl/ParseHelper.java | 27 ++++++- .../impl/RelayNetworkStatusConsensusImpl.java | 7 +- .../impl/RelayNetworkStatusVoteImpl.java | 3 +- 7 files changed, 126 insertions(+), 7 deletions(-) diff --git a/src/org/torproject/descriptor/GetTorStatistics.java b/src/org/torproject/descriptor/GetTorStatistics.java new file mode 100644 index 0000000..b6ba1e7 --- /dev/null +++ b/src/org/torproject/descriptor/GetTorStatistics.java @@ -0,0 +1,13 @@ +package org.torproject.descriptor; + +import java.util.SortedMap; + +public interface GetTorStatistics { + + /* Return the date of these GetTor statistics in milliseconds since + * 1970-01-01 00:00:00. */ + public long getDateMillis(); + + /* Return the number of downloaded packages. */ + public SortedMap<String, Integer> getDownloadedPackages(); +} diff --git a/src/org/torproject/descriptor/impl/DescriptorImpl.java b/src/org/torproject/descriptor/impl/DescriptorImpl.java index c1c4a9a..98be2d8 100644 --- a/src/org/torproject/descriptor/impl/DescriptorImpl.java +++ b/src/org/torproject/descriptor/impl/DescriptorImpl.java @@ -88,6 +88,9 @@ public abstract class DescriptorImpl implements Descriptor { } else if (firstLines.startsWith("@type torperf 1.0\n")) { parsedDescriptors.addAll(TorperfResultImpl.parseTorperfResults( rawDescriptorBytes, failUnrecognizedDescriptorLines)); + } else if (firstLines.startsWith("@type gettor 1.0\n")) { + parsedDescriptors.addAll(GetTorStatisticsImpl.parseGetTorStatistics( + rawDescriptorBytes, failUnrecognizedDescriptorLines)); } else { throw new DescriptorParseException("Could not detect descriptor " + "type in descriptor starting with '" + firstLines + "'."); diff --git a/src/org/torproject/descriptor/impl/GetTorStatisticsImpl.java b/src/org/torproject/descriptor/impl/GetTorStatisticsImpl.java new file mode 100644 index 0000000..202cf03 --- /dev/null +++ b/src/org/torproject/descriptor/impl/GetTorStatisticsImpl.java @@ -0,0 +1,78 @@ +/* Copyright 2012 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.descriptor.impl; + +import java.util.ArrayList; +import java.util.List; +import java.util.Scanner; +import java.util.SortedMap; +import java.util.TreeMap; + +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.GetTorStatistics; + +public class GetTorStatisticsImpl extends DescriptorImpl + implements GetTorStatistics { + + public static List<Descriptor> parseGetTorStatistics( + byte[] rawDescriptorBytes, boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { + if (rawDescriptorBytes.length == 0) { + throw new DescriptorParseException("Descriptor is empty."); + } + List<Descriptor> parsedDescriptors = new ArrayList<Descriptor>(); + String descriptorString = new String(rawDescriptorBytes); + Scanner s = new Scanner(descriptorString).useDelimiter("\n"); + while (s.hasNext()) { + String line = s.next(); + if (line.startsWith("@type gettor ")) { + String[] parts = line.split(" "); + if (parts.length != 3) { + throw new DescriptorParseException("Illegal line '" + line + + "'."); + } + String version = parts[2]; + if (!version.startsWith("1.")) { + throw new DescriptorParseException("Unsupported version in " + + " line '" + line + "'."); + } + } else { + parsedDescriptors.add(new GetTorStatisticsImpl(line.getBytes(), + failUnrecognizedDescriptorLines)); + } + } + return parsedDescriptors; + } + + protected GetTorStatisticsImpl(byte[] rawDescriptorBytes, + boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { + super(rawDescriptorBytes, failUnrecognizedDescriptorLines, false); + this.parseGetTorStatisticsLine(new String(rawDescriptorBytes)); + } + + private void parseGetTorStatisticsLine(String line) + throws DescriptorParseException { + if (line.isEmpty()) { + throw new DescriptorParseException("Blank lines are not allowed."); + } + String[] parts = line.split(" "); + if (parts.length < 3) { + throw new DescriptorParseException("Illegal GetTor statistics line " + + "'" + line + "'."); + } + this.dateMillis = ParseHelper.parseDateAtIndex(line, parts, 0); + this.downloadedPackages = ParseHelper.parseKeyValuePairs(line, parts, + 2, ":"); + } + + private long dateMillis; + public long getDateMillis() { + return this.dateMillis; + } + + private SortedMap<String, Integer> downloadedPackages; + public SortedMap<String, Integer> getDownloadedPackages() { + return new TreeMap<String, Integer>(this.downloadedPackages); + } +} diff --git a/src/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java b/src/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java index 108d563..423e21e 100644 --- a/src/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java +++ b/src/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java @@ -151,7 +151,7 @@ public class NetworkStatusEntryImpl implements NetworkStatusEntry { throws DescriptorParseException { this.parsedAtMostOnceKeyword("w"); SortedMap<String, Integer> pairs = ParseHelper.parseKeyValuePairs( - line, parts, 1); + line, parts, 1, "="); if (pairs.isEmpty()) { throw new DescriptorParseException("Illegal line '" + line + "'."); } diff --git a/src/org/torproject/descriptor/impl/ParseHelper.java b/src/org/torproject/descriptor/impl/ParseHelper.java index a3f907d..61d1eb0 100644 --- a/src/org/torproject/descriptor/impl/ParseHelper.java +++ b/src/org/torproject/descriptor/impl/ParseHelper.java @@ -149,6 +149,28 @@ public class ParseHelper { return result; } + public static long parseDateAtIndex(String line, String[] parts, + int dateIndex) throws DescriptorParseException { + if (dateIndex >= parts.length) { + throw new DescriptorParseException("Line '" + line + "' does not " + + "contain a date at the expected position."); + } + long result = -1L; + try { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + dateFormat.setLenient(false); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + result = dateFormat.parse(parts[dateIndex]).getTime(); + } catch (ParseException e) { + /* Leave result at -1L. */ + } + if (result < 0L || result / 1000L > (long) Integer.MAX_VALUE) { + throw new DescriptorParseException("Illegal date format in line '" + + line + "'."); + } + return result; + } + private static Pattern twentyByteHexPattern = Pattern.compile("^[0-9a-fA-F]{40}$"); public static String parseTwentyByteHexString(String line, @@ -161,11 +183,12 @@ public class ParseHelper { } public static SortedMap<String, Integer> parseKeyValuePairs(String line, - String[] parts, int startIndex) throws DescriptorParseException { + String[] parts, int startIndex, String separatorString) + throws DescriptorParseException { SortedMap<String, Integer> result = new TreeMap<String, Integer>(); for (int i = startIndex; i < parts.length; i++) { String pair = parts[i]; - String[] pairParts = pair.split("="); + String[] pairParts = pair.split(separatorString); if (pairParts.length != 2) { throw new DescriptorParseException("Illegal key-value pair in " + "line '" + line + "'."); diff --git a/src/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java b/src/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java index f296fa6..35cb316 100644 --- a/src/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java +++ b/src/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java @@ -257,13 +257,14 @@ public class RelayNetworkStatusConsensusImpl extends NetworkStatusImpl private void parseParamsLine(String line, String[] parts) throws DescriptorParseException { - this.consensusParams = ParseHelper.parseKeyValuePairs(line, parts, 1); + this.consensusParams = ParseHelper.parseKeyValuePairs(line, parts, 1, + "="); } private void parseBandwidthWeightsLine(String line, String[] parts) throws DescriptorParseException { - this.bandwidthWeights = ParseHelper.parseKeyValuePairs(line, parts, - 1); + this.bandwidthWeights = ParseHelper.parseKeyValuePairs(line, parts, 1, + "="); } private String consensusDigest; diff --git a/src/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java b/src/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java index d5b87c1..3695074 100644 --- a/src/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java +++ b/src/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java @@ -249,7 +249,8 @@ public class RelayNetworkStatusVoteImpl extends NetworkStatusImpl private void parseParamsLine(String line, String[] parts) throws DescriptorParseException { - this.consensusParams = ParseHelper.parseKeyValuePairs(line, parts, 1); + this.consensusParams = ParseHelper.parseKeyValuePairs(line, parts, 1, + "="); } private void parseDirSourceLine(String line, String[] parts)

1 0

[onionoo/master] Don't break when reading a broken history file.
by karsten＠torproject.org 01 Jun '12

01 Jun '12

commit 5951918390db5d48fd044e9dad9de47c69981b6b Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jun 1 11:42:43 2012 +0200 Don't break when reading a broken history file. --- .../torproject/onionoo/BandwidthDataWriter.java | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diff --git a/src/org/torproject/onionoo/BandwidthDataWriter.java b/src/org/torproject/onionoo/BandwidthDataWriter.java index 7052e3c..4f86371 100644 --- a/src/org/torproject/onionoo/BandwidthDataWriter.java +++ b/src/org/torproject/onionoo/BandwidthDataWriter.java @@ -143,6 +143,12 @@ public class BandwidthDataWriter { String line; while ((line = br.readLine()) != null) { String[] parts = line.split(" "); + if (parts.length != 6) { + System.err.println("Illegal line '" + line + "' in history " + + "file '" + historyFile.getAbsolutePath() + + "'. Skipping this line."); + continue; + } SortedMap<Long, long[]> history = parts[0].equals("r") ? readHistory : writeHistory; long startMillis = this.dateTimeFormat.parse(parts[1] + " "

1 0

[flashproxy/master] Got the order of addresses backwards in the public connector example.
by dcf＠torproject.org 01 Jun '12

01 Jun '12

commit 75ba0f771ed17bd51957090f12edf302b8ea2215 Author: David Fifield <david(a)bamsoftware.com> Date: Thu May 31 23:28:55 2012 -0700 Got the order of addresses backwards in the public connector example. --- README | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/README b/README index 5aa054f..c00804b 100644 --- a/README +++ b/README @@ -83,7 +83,7 @@ connector. This way is not as realistic because all your Tor traffic will first go to a public connector, which is at a fixed address and can be easily blocked. However this is an easy way to try out the system without having to do port forwarding. - $ tor ClientTransportPlugin "websocket socks4 127.0.0.1:9001" UseBridges 1 Bridge "websocket tor-facilitator.bamsoftware.com:9999" + $ tor ClientTransportPlugin "websocket socks4 tor-facilitator.bamsoftware.com:9999" UseBridges 1 Bridge "websocket 127.0.0.1:9001" === Troubleshooting

1 0

[torbutton/master] Update build process documentation to cover signing.
by mikeperry＠torproject.org 31 May '12

31 May '12

commit 9a695108478eb538b30a8bd04575921764f227a8 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Thu May 31 19:48:54 2012 -0700 Update build process documentation to cover signing. --- README.RELEASE | 17 ++++++++++------- 1 files changed, 10 insertions(+), 7 deletions(-) diff --git a/README.RELEASE b/README.RELEASE index aa6e9ca..eb6015a 100644 --- a/README.RELEASE +++ b/README.RELEASE @@ -59,19 +59,22 @@ torsocks scp ./pkg/torbutton-${VERSION}.xpi vescum:/srv/www-master.torproject.org/htdocs/dist/torbutton/torbutton-curre… torsocks scp ./pkg/torbutton-${VERSION}.xpi.asc vescum:/srv/www-master.torproject.org/htdocs/dist/torbutton/torbutton-curre… -#. Update + Sign update.rdf +#. Update update.rdf cd website-svn vim ./include/versions.wmi # Add sha1 and version vim ./torbutton/update.rdf # Add <li> and <description> w/ sha1 and version - # Sign the new update.rdf. - # Command line: - # http://blog.techno-barje.fr/post/2009/10/08/How-to-setup-a-mozilla-extensio… - # Gui: - # https://www.binaryturf.com/steps-release-extension-firefox-3/ +#. Sign the new update.rdf. - # Copy the signed update.rdf into place + # TODO: Currently manual due to offline signing. + # No need to fear the reaper, man. We can do online signing if we just put + # our minds to it: https://trac.torproject.org/projects/tor/ticket/6011 + + # Command line: http://blog.techno-barje.fr/post/2009/10/08/How-to-setup-a-mozilla-extensio… + # Gui: https://www.binaryturf.com/steps-release-extension-firefox-3/ + + # Copy the signed update.rdf into place in the website svn #. Update website

1 0

r25664: {website} New signed Torbutton update.rdf for 1.4.6. Oh man, I hope th (in website/trunk: include torbutton)
by Mike Perry 31 May '12

31 May '12

Author: mikeperry Date: 2012-06-01 02:42:55 +0000 (Fri, 01 Jun 2012) New Revision: 25664 Modified: website/trunk/include/versions.wmi website/trunk/torbutton/update.rdf Log: New signed Torbutton update.rdf for 1.4.6. Oh man, I hope this works... Modified: website/trunk/include/versions.wmi =================================================================== --- website/trunk/include/versions.wmi 2012-05-31 21:16:22 UTC (rev 25663) +++ website/trunk/include/versions.wmi 2012-06-01 02:42:55 UTC (rev 25664) @@ -107,8 +107,8 @@ <define-tag file-source-alpha whitespace=delete>tor-<version-alpha>.tar.gz</define-tag> <define-tag package-source-alpha whitespace=delete>../dist/<file-source-alpha></define-tag> -<define-tag version-torbutton whitespace=delete>1.4.5.1 (17 Dec 2011)</define-tag> -<define-tag version-hash-torbutton whitespace=delete>sha1:0002e2d61613504e41d8b773c30ff6307b8286da</define-tag> +<define-tag version-torbutton whitespace=delete>1.4.6 (31 May 2012)</define-tag> +<define-tag version-hash-torbutton whitespace=delete>sha1:eedadda6e5aacb8dabe2118e6c14c4dd4441fed0</define-tag> <define-tag version-torbutton-alpha whitespace=delete>None</define-tag> <define-tag version-hash-torbutton-alpha whitespace=delete>sha1:c7323cd408ebee28ee23b6a91fe056d59de668f5</define-tag> Modified: website/trunk/torbutton/update.rdf =================================================================== (Binary files differ)

1 0

[torbutton/master] Release script + doc updates.
by mikeperry＠torproject.org 31 May '12

31 May '12

commit 03b74562a4b2137769582816c1dc83b3d792bd77 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Thu May 31 15:17:37 2012 -0700 Release script + doc updates. --- README.RELEASE | 14 ++++++++++++-- trans_tools/validate.py | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/README.RELEASE b/README.RELEASE index f386760..aa6e9ca 100644 --- a/README.RELEASE +++ b/README.RELEASE @@ -5,7 +5,7 @@ #. Decide the version number. - export VERSION="1.4.5.1" + export VERSION="1.4.6" vim ./src/install.rdf # edit <em:version> tag to reflect version #. Test upgrades of candidate XPI in latest Firefox and TBB product matrix @@ -59,12 +59,22 @@ torsocks scp ./pkg/torbutton-${VERSION}.xpi vescum:/srv/www-master.torproject.org/htdocs/dist/torbutton/torbutton-curre… torsocks scp ./pkg/torbutton-${VERSION}.xpi.asc vescum:/srv/www-master.torproject.org/htdocs/dist/torbutton/torbutton-curre… -#. Update website +#. Update + Sign update.rdf cd website-svn vim ./include/versions.wmi # Add sha1 and version vim ./torbutton/update.rdf # Add <li> and <description> w/ sha1 and version + # Sign the new update.rdf. + # Command line: + # http://blog.techno-barje.fr/post/2009/10/08/How-to-setup-a-mozilla-extensio… + # Gui: + # https://www.binaryturf.com/steps-release-extension-firefox-3/ + + # Copy the signed update.rdf into place + +#. Update website + torsocks svn up . torsocks svn commit . diff --git a/trans_tools/validate.py b/trans_tools/validate.py index 983ed94..179e7b0 100755 --- a/trans_tools/validate.py +++ b/trans_tools/validate.py @@ -1,4 +1,4 @@ -#! /bin/env python +#!/usr/bin/python from __future__ import unicode_literals, print_function

1 0