tor-commits January 2016

tor-commits@lists.torproject.org

19 participants
1449 discussions

[meek/master] Increase version number to 0.22.
by dcf＠torproject.org 11 Jan '16

11 Jan '16

commit b60dc093c98c650f3a8ab1445da8b9185791cbcc Author: David Fifield <david(a)bamsoftware.com> Date: Mon Jan 11 12:50:07 2016 -0800 Increase version number to 0.22. --- meek-server/meek-server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meek-server/meek-server.go b/meek-server/meek-server.go index a44d1c5..1eab1e5 100644 --- a/meek-server/meek-server.go +++ b/meek-server/meek-server.go @@ -30,7 +30,7 @@ import ( ) const ( - programVersion = "0.21" + programVersion = "0.22" ptMethodName = "meek" // Reject session ids shorter than this, as a weak defense against

1 0

[tor/master] Make the touch_file unit test work around FS/system time sync issues
by nickm＠torproject.org 11 Jan '16

11 Jan '16

commit 16840e52e594e221069efc9bf7b65ebfb5e2d639 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Jan 11 10:03:00 2016 -0500 Make the touch_file unit test work around FS/system time sync issues Sometimes you can call time() and then touch a file, and have the second come out a little before the first. See #18025 for way more information than you necessarily wanted. --- src/test/test_util.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/test/test_util.c b/src/test/test_util.c index 863105f..37f7d93 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -4612,18 +4612,23 @@ test_util_touch_file(void *arg) struct stat st; write_bytes_to_file(fname, "abc", 3, 1); tt_int_op(0, OP_EQ, stat(fname, &st)); - tt_i64_op(st.st_mtime, OP_GE, now); + /* A subtle point: the filesystem time is not necessarily equal to the + * system clock time, since one can be using a monotonic clock, or coarse + * monotonic clock, or whatever. So we might wind up with an mtime a few + * microseconds ago. Let's just give it a lot of wiggle room. */ + tt_i64_op(st.st_mtime, OP_GE, now - 1); const time_t five_sec_ago = now - 5; struct utimbuf u = { five_sec_ago, five_sec_ago }; tt_int_op(0, OP_EQ, utime(fname, &u)); tt_int_op(0, OP_EQ, stat(fname, &st)); + /* Let's hope that utime/stat give the same second as a round-trip? */ tt_i64_op(st.st_mtime, OP_EQ, five_sec_ago); /* Finally we can touch the file */ tt_int_op(0, OP_EQ, touch_file(fname)); tt_int_op(0, OP_EQ, stat(fname, &st)); - tt_i64_op(st.st_mtime, OP_GE, now); + tt_i64_op(st.st_mtime, OP_GE, now-1); done: ;

1 0

[metrics-web/master] Stop updating deprecated connbidirect.csv.
by karsten＠torproject.org 11 Jan '16

11 Jan '16

commit ff9ec67b046ce37a1d06834edb3bd980bce98a9c Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Jan 11 09:27:57 2016 +0100 Stop updating deprecated connbidirect.csv. This removes another psql dependency. --- modules/legacy/db/tordir.sql | 19 -- .../legacy/src/org/torproject/ernie/cron/Main.java | 13 - .../cron/performance/PerformanceStatsImporter.java | 271 -------------------- shared/bin/50-run-legacy-stats.sh | 1 - website/web/WEB-INF/connbidirect-data.jsp | 1 + 5 files changed, 1 insertion(+), 304 deletions(-) diff --git a/modules/legacy/db/tordir.sql b/modules/legacy/db/tordir.sql index 2c97829..04d31c0 100644 --- a/modules/legacy/db/tordir.sql +++ b/modules/legacy/db/tordir.sql @@ -104,19 +104,6 @@ CREATE TABLE consensus ( CONSTRAINT consensus_pkey PRIMARY KEY (validafter) ); --- TABLE connbidirect --- Contain conn-bi-direct stats strings -CREATE TABLE connbidirect ( - source CHARACTER(40) NOT NULL, - statsend TIMESTAMP WITHOUT TIME ZONE NOT NULL, - seconds INTEGER NOT NULL, - belownum BIGINT NOT NULL, - readnum BIGINT NOT NULL, - writenum BIGINT NOT NULL, - bothnum BIGINT NOT NULL, - CONSTRAINT connbidirect_pkey PRIMARY KEY (source, statsend) -); - -- TABLE network_size CREATE TABLE network_size ( date DATE NOT NULL, @@ -952,9 +939,3 @@ UNION ALL current_date - 3) ORDER BY 1, 2, 3; --- View for exporting connbidirect statistics. -CREATE VIEW stats_connbidirect AS -SELECT DATE(statsend) AS date, source, belownum AS below, readnum AS read, - writenum AS write, bothnum AS "both" FROM connbidirect - WHERE DATE(statsend) < current_date - 1 ORDER BY 1, 2; - diff --git a/modules/legacy/src/org/torproject/ernie/cron/Main.java b/modules/legacy/src/org/torproject/ernie/cron/Main.java index 9bd2d34..fb0697c 100644 --- a/modules/legacy/src/org/torproject/ernie/cron/Main.java +++ b/modules/legacy/src/org/torproject/ernie/cron/Main.java @@ -6,7 +6,6 @@ import java.io.File; import java.util.logging.Logger; import org.torproject.ernie.cron.network.ConsensusStatsFileHandler; -import org.torproject.ernie.cron.performance.PerformanceStatsImporter; import org.torproject.ernie.cron.performance.TorperfProcessor; /** @@ -53,18 +52,6 @@ public class Main { rddi.importRelayDescriptors(); } rddi.closeConnection(); - - // Import conn-bi-direct statistics. - PerformanceStatsImporter psi = new PerformanceStatsImporter( - config.getWriteRelayDescriptorDatabase() ? - config.getRelayDescriptorDatabaseJDBC() : null, - config.getWriteRelayDescriptorsRawFiles() ? - config.getRelayDescriptorRawFilesDirectory() : null, - new File(config.getDirectoryArchivesDirectory()), - statsDirectory, - config.getKeepDirectoryArchiveImportHistory()); - psi.importRelayDescriptors(); - psi.closeConnection(); } // Prepare consensus stats file handler (used for stats on running diff --git a/modules/legacy/src/org/torproject/ernie/cron/performance/PerformanceStatsImporter.java b/modules/legacy/src/org/torproject/ernie/cron/performance/PerformanceStatsImporter.java deleted file mode 100644 index 815b37f..0000000 --- a/modules/legacy/src/org/torproject/ernie/cron/performance/PerformanceStatsImporter.java +++ /dev/null @@ -1,271 +0,0 @@ -/* Copyright 2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.ernie.cron.performance; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Timestamp; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Iterator; -import java.util.TimeZone; -import java.util.logging.Level; -import java.util.logging.Logger; - -import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorFile; -import org.torproject.descriptor.DescriptorReader; -import org.torproject.descriptor.DescriptorSourceFactory; -import org.torproject.descriptor.ExtraInfoDescriptor; - -public class PerformanceStatsImporter { - - /** - * How many records to commit with each database transaction. - */ - private final long autoCommitCount = 500; - - /** - * Keep track of the number of records committed before each transaction - */ - private int rbsCount = 0; - - /** - * Relay descriptor database connection. - */ - private Connection conn; - - /** - * Prepared statement to check whether a given conn-bi-direct stats - * string has been imported into the database before. - */ - private PreparedStatement psBs; - - /** - * Prepared statement to insert a conn-bi-direct stats string into the - * database. - */ - private PreparedStatement psB; - - /** - * Logger for this class. - */ - private Logger logger; - - /** - * Directory for writing raw import files. - */ - private String rawFilesDirectory; - - /** - * Raw import file containing conn-bi-direct stats strings. - */ - private BufferedWriter connBiDirectOut; - - /** - * Date format to parse timestamps. - */ - private SimpleDateFormat dateTimeFormat; - - private boolean importIntoDatabase; - private boolean writeRawImportFiles; - - private File archivesDirectory; - private File statsDirectory; - private boolean keepImportHistory; - - /** - * Initialize database importer by connecting to the database and - * preparing statements. - */ - public PerformanceStatsImporter(String connectionURL, - String rawFilesDirectory, File archivesDirectory, - File statsDirectory, boolean keepImportHistory) { - - if (archivesDirectory == null || - statsDirectory == null) { - throw new IllegalArgumentException(); - } - this.archivesDirectory = archivesDirectory; - this.statsDirectory = statsDirectory; - this.keepImportHistory = keepImportHistory; - - /* Initialize logger. */ - this.logger = Logger.getLogger( - PerformanceStatsImporter.class.getName()); - - if (connectionURL != null) { - try { - /* Connect to database. */ - this.conn = DriverManager.getConnection(connectionURL); - - /* Turn autocommit off */ - this.conn.setAutoCommit(false); - - /* Prepare statements. */ - this.psBs = conn.prepareStatement("SELECT COUNT(*) " - + "FROM connbidirect WHERE source = ? AND statsend = ?"); - this.psB = conn.prepareStatement("INSERT INTO connbidirect " - + "(source, statsend, seconds, belownum, readnum, writenum, " - + "bothnum) VALUES (?, ?, ?, ?, ?, ?, ?)"); - this.importIntoDatabase = true; - } catch (SQLException e) { - this.logger.log(Level.WARNING, "Could not connect to database or " - + "prepare statements.", e); - } - } - - /* Remember where we want to write raw import files. */ - if (rawFilesDirectory != null) { - this.rawFilesDirectory = rawFilesDirectory; - this.writeRawImportFiles = true; - } - - /* Initialize date format, so that we can format timestamps. */ - this.dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - this.dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - } - - /** - * Insert a conn-bi-direct stats string into the database. - */ - private void addConnBiDirect(String source, long statsEndMillis, - long seconds, long below, long read, long write, long both) { - String statsEnd = this.dateTimeFormat.format(statsEndMillis); - if (this.importIntoDatabase) { - try { - Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC")); - Timestamp statsEndTimestamp = new Timestamp(statsEndMillis); - this.psBs.setString(1, source); - this.psBs.setTimestamp(2, statsEndTimestamp, cal); - ResultSet rs = psBs.executeQuery(); - rs.next(); - if (rs.getInt(1) == 0) { - this.psB.clearParameters(); - this.psB.setString(1, source); - this.psB.setTimestamp(2, statsEndTimestamp, cal); - this.psB.setLong(3, seconds); - this.psB.setLong(4, below); - this.psB.setLong(5, read); - this.psB.setLong(6, write); - this.psB.setLong(7, both); - this.psB.executeUpdate(); - rbsCount++; - if (rbsCount % autoCommitCount == 0) { - this.conn.commit(); - } - } - } catch (SQLException e) { - this.logger.log(Level.WARNING, "Could not add conn-bi-direct " - + "stats string. We won't make any further SQL requests in " - + "this execution.", e); - this.importIntoDatabase = false; - } - } - if (this.writeRawImportFiles) { - try { - if (this.connBiDirectOut == null) { - new File(rawFilesDirectory).mkdirs(); - this.connBiDirectOut = new BufferedWriter(new FileWriter( - rawFilesDirectory + "/connbidirect.sql")); - this.connBiDirectOut.write(" COPY connbidirect (source, " - + "statsend, seconds, belownum, readnum, writenum, " - + "bothnum) FROM stdin;\n"); - } - this.connBiDirectOut.write(source + "\t" + statsEnd + "\t" - + seconds + "\t" + below + "\t" + read + "\t" + write + "\t" - + both + "\n"); - } catch (IOException e) { - this.logger.log(Level.WARNING, "Could not write conn-bi-direct " - + "stats string to raw database import file. We won't make " - + "any further attempts to write raw import files in this " - + "execution.", e); - this.writeRawImportFiles = false; - } - } - } - - public void importRelayDescriptors() { - if (archivesDirectory.exists()) { - logger.fine("Importing files in directory " + archivesDirectory - + "/..."); - DescriptorReader reader = - DescriptorSourceFactory.createDescriptorReader(); - reader.addDirectory(archivesDirectory); - if (keepImportHistory) { - reader.setExcludeFiles(new File(statsDirectory, - "performance-stats-relay-descriptor-history")); - } - Iterator<DescriptorFile> descriptorFiles = reader.readDescriptors(); - while (descriptorFiles.hasNext()) { - DescriptorFile descriptorFile = descriptorFiles.next(); - if (descriptorFile.getDescriptors() != null) { - for (Descriptor descriptor : descriptorFile.getDescriptors()) { - if (descriptor instanceof ExtraInfoDescriptor) { - this.addExtraInfoDescriptor( - (ExtraInfoDescriptor) descriptor); - } - } - } - } - } - - logger.info("Finished importing relay descriptors."); - } - - private void addExtraInfoDescriptor(ExtraInfoDescriptor descriptor) { - if (descriptor.getConnBiDirectStatsEndMillis() >= 0L) { - this.addConnBiDirect(descriptor.getFingerprint(), - descriptor.getConnBiDirectStatsEndMillis(), - descriptor.getConnBiDirectStatsIntervalLength(), - descriptor.getConnBiDirectBelow(), - descriptor.getConnBiDirectRead(), - descriptor.getConnBiDirectWrite(), - descriptor.getConnBiDirectBoth()); - } - } - - /** - * Close the relay descriptor database connection. - */ - public void closeConnection() { - - /* Log stats about imported descriptors. */ - this.logger.info(String.format("Finished importing relay " - + "descriptors: %d conn-bi-direct stats lines", rbsCount)); - - /* Commit any stragglers before closing. */ - if (this.conn != null) { - try { - this.conn.commit(); - } catch (SQLException e) { - this.logger.log(Level.WARNING, "Could not commit final records " - + "to database", e); - } - try { - this.conn.close(); - } catch (SQLException e) { - this.logger.log(Level.WARNING, "Could not close database " - + "connection.", e); - } - } - - /* Close raw import files. */ - try { - if (this.connBiDirectOut != null) { - this.connBiDirectOut.write("\\.\n"); - this.connBiDirectOut.close(); - } - } catch (IOException e) { - this.logger.log(Level.WARNING, "Could not close one or more raw " - + "database import files.", e); - } - } -} diff --git a/shared/bin/50-run-legacy-stats.sh b/shared/bin/50-run-legacy-stats.sh index 75fe66c..0fd5699 100755 --- a/shared/bin/50-run-legacy-stats.sh +++ b/shared/bin/50-run-legacy-stats.sh @@ -5,6 +5,5 @@ psql -U metrics tordir -c 'SELECT * FROM refresh_all();' mkdir -p stats psql -c 'COPY (SELECT * FROM stats_servers) TO STDOUT WITH CSV HEADER;' tordir > stats/servers.csv psql -c 'COPY (SELECT * FROM stats_bandwidth) TO STDOUT WITH CSV HEADER;' tordir > stats/bandwidth.csv -psql -c 'COPY (SELECT * FROM stats_connbidirect) TO STDOUT WITH CSV HEADER;' tordir > stats/connbidirect.csv cd ../../ diff --git a/website/web/WEB-INF/connbidirect-data.jsp b/website/web/WEB-INF/connbidirect-data.jsp index 8c32fc5..1be8831 100644 --- a/website/web/WEB-INF/connbidirect-data.jsp +++ b/website/web/WEB-INF/connbidirect-data.jsp @@ -18,6 +18,7 @@ As of August 25, 2015, this page and the linked data file have been replaced by <a href="connbidirect2-data.html">this page and the data file linked from there</a>. +Starting on January 11, 2016, the linked data file is not updated anymore. This page and the linked data file will be removed in the future.

1 0

[metrics-web/master] Remove psql dependency for producing torperf.csv.
by karsten＠torproject.org 11 Jan '16

11 Jan '16

commit 9c22116574b8dd79e82bb8c59314bce569794f71 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun Jan 10 17:38:49 2016 +0100 Remove psql dependency for producing torperf.csv. --- modules/legacy/db/tordir.sql | 24 --- .../legacy/src/org/torproject/ernie/cron/Main.java | 2 +- .../ernie/cron/performance/TorperfProcessor.java | 167 +++++--------------- shared/bin/50-run-legacy-stats.sh | 1 - 4 files changed, 41 insertions(+), 153 deletions(-) diff --git a/modules/legacy/db/tordir.sql b/modules/legacy/db/tordir.sql index e5f02f3..2c97829 100644 --- a/modules/legacy/db/tordir.sql +++ b/modules/legacy/db/tordir.sql @@ -807,20 +807,6 @@ CREATE TABLE dirreq_stats ( PRIMARY KEY (source, statsend, seconds, country) ); --- TABLE torperf_stats --- Quantiles and medians of daily torperf results. -CREATE TABLE torperf_stats ( - "date" DATE NOT NULL, - source CHARACTER VARYING(32) NOT NULL, - q1 INTEGER NOT NULL, - md INTEGER NOT NULL, - q3 INTEGER NOT NULL, - timeouts INTEGER NOT NULL, - failures INTEGER NOT NULL, - requests INTEGER NOT NULL, - CONSTRAINT torperf_stats_pkey PRIMARY KEY("date", source) -); - -- Refresh all statistics in the database. CREATE OR REPLACE FUNCTION refresh_all() RETURNS INTEGER AS $$ BEGIN @@ -966,16 +952,6 @@ UNION ALL current_date - 3) ORDER BY 1, 2, 3; --- View for exporting torperf statistics. -CREATE VIEW stats_torperf AS -SELECT date, CASE WHEN source LIKE '%-50kb' THEN 50 * 1024 - WHEN source LIKE '%-1mb' THEN 1024 * 1024 - WHEN source LIKE '%-5mb' THEN 5 * 1024 * 1024 END AS size, - CASE WHEN source NOT LIKE 'all-%' - THEN split_part(source, '-', 1) END AS source, q1, md, q3, timeouts, - failures, requests FROM torperf_stats WHERE date < current_date - 1 - ORDER BY 1, 2, 3; - -- View for exporting connbidirect statistics. CREATE VIEW stats_connbidirect AS SELECT DATE(statsend) AS date, source, belownum AS below, readnum AS read, diff --git a/modules/legacy/src/org/torproject/ernie/cron/Main.java b/modules/legacy/src/org/torproject/ernie/cron/Main.java index 5d561a6..9bd2d34 100644 --- a/modules/legacy/src/org/torproject/ernie/cron/Main.java +++ b/modules/legacy/src/org/torproject/ernie/cron/Main.java @@ -88,7 +88,7 @@ public class Main { // Import and process torperf stats if (config.getImportWriteTorperfStats()) { new TorperfProcessor(new File(config.getTorperfDirectory()), - statsDirectory, config.getRelayDescriptorDatabaseJDBC()); + statsDirectory); } // Remove lock file diff --git a/modules/legacy/src/org/torproject/ernie/cron/performance/TorperfProcessor.java b/modules/legacy/src/org/torproject/ernie/cron/performance/TorperfProcessor.java index d7322db..ed3a0af 100644 --- a/modules/legacy/src/org/torproject/ernie/cron/performance/TorperfProcessor.java +++ b/modules/legacy/src/org/torproject/ernie/cron/performance/TorperfProcessor.java @@ -8,21 +8,12 @@ import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.SortedMap; import java.util.TimeZone; import java.util.TreeMap; @@ -36,8 +27,7 @@ import org.torproject.descriptor.DescriptorSourceFactory; import org.torproject.descriptor.TorperfResult; public class TorperfProcessor { - public TorperfProcessor(File torperfDirectory, File statsDirectory, - String connectionURL) { + public TorperfProcessor(File torperfDirectory, File statsDirectory) { if (torperfDirectory == null || statsDirectory == null) { throw new IllegalArgumentException(); @@ -45,7 +35,7 @@ public class TorperfProcessor { Logger logger = Logger.getLogger(TorperfProcessor.class.getName()); File rawFile = new File(statsDirectory, "torperf-raw"); - File statsFile = new File(statsDirectory, "torperf-stats"); + File statsFile = new File(statsDirectory, "torperf.csv"); SortedMap<String, String> rawObs = new TreeMap<String, String>(); SortedMap<String, String> stats = new TreeMap<String, String>(); int addedRawObs = 0; @@ -76,7 +66,9 @@ public class TorperfProcessor { BufferedReader br = new BufferedReader(new FileReader(statsFile)); String line = br.readLine(); // ignore header while ((line = br.readLine()) != null) { - String key = line.split(",")[0] + "," + line.split(",")[1]; + String[] parts = line.split(","); + String key = String.format("%s,%s,%s", parts[0], parts[1], + parts[2]); stats.put(key, line); } br.close(); @@ -165,18 +157,27 @@ public class TorperfProcessor { long q1 = dlTimes.get(dlTimes.size() / 4 - 1); long md = dlTimes.get(dlTimes.size() / 2 - 1); long q3 = dlTimes.get(dlTimes.size() * 3 / 4 - 1); - stats.put(tempSourceDate, tempSourceDate + "," + q1 + "," - + md + "," + q3 + "," + timeouts + "," + failures + "," - + requests); - String allSourceDate = "all" + tempSourceDate.substring( - tempSourceDate.indexOf("-")); - if (dlTimesAllSources.containsKey(allSourceDate)) { - dlTimesAllSources.get(allSourceDate).addAll(dlTimes); + String[] tempParts = tempSourceDate.split("[-,]", 3); + String tempDate = tempParts[2]; + int tempSize = Integer.parseInt( + tempParts[1].substring(0, tempParts[1].length() - 2)) + * 1024 * (tempParts[1].endsWith("mb") ? 1024 : 1); + String tempSource = tempParts[0]; + String tempDateSizeSource = String.format("%s,%d,%s", + tempDate, tempSize, tempSource); + stats.put(tempDateSizeSource, + String.format("%s,%s,%s,%s,%s,%s,%s", + tempDateSizeSource, q1, md, q3, timeouts, failures, + requests)); + String allDateSizeSource = String.format("%s,%d,", + tempDate, tempSize); + if (dlTimesAllSources.containsKey(allDateSizeSource)) { + dlTimesAllSources.get(allDateSizeSource).addAll(dlTimes); } else { - dlTimesAllSources.put(allSourceDate, dlTimes); + dlTimesAllSources.put(allDateSizeSource, dlTimes); } - if (statusesAllSources.containsKey(allSourceDate)) { - long[] status = statusesAllSources.get(allSourceDate); + if (statusesAllSources.containsKey(allDateSizeSource)) { + long[] status = statusesAllSources.get(allDateSizeSource); status[0] += timeouts; status[1] += failures; status[2] += requests; @@ -185,7 +186,7 @@ public class TorperfProcessor { status[0] = timeouts; status[1] = failures; status[2] = requests; - statusesAllSources.put(allSourceDate, status); + statusesAllSources.put(allDateSizeSource, status); } } dlTimes = new ArrayList<Long>(); @@ -212,19 +213,20 @@ public class TorperfProcessor { bw.close(); for (Map.Entry<String, List<Long>> e : dlTimesAllSources.entrySet()) { - String allSourceDate = e.getKey(); + String allDateSizeSource = e.getKey(); dlTimes = e.getValue(); Collections.sort(dlTimes); long q1 = dlTimes.get(dlTimes.size() / 4 - 1); long md = dlTimes.get(dlTimes.size() / 2 - 1); long q3 = dlTimes.get(dlTimes.size() * 3 / 4 - 1); - long[] status = statusesAllSources.get(allSourceDate); + long[] status = statusesAllSources.get(allDateSizeSource); timeouts = status[0]; failures = status[1]; requests = status[2]; - stats.put(allSourceDate, allSourceDate + "," + q1 + "," + md - + "," + q3 + "," + timeouts + "," + failures + "," - + requests); + stats.put(allDateSizeSource, + String.format("%s,%s,%s,%s,%s,%s,%s", + allDateSizeSource, q1, md, q3, timeouts, failures, + requests)); } logger.fine("Finished writing file " + rawFile.getAbsolutePath() + "."); @@ -232,11 +234,18 @@ public class TorperfProcessor { if (stats.size() > 0) { logger.fine("Writing file " + statsFile.getAbsolutePath() + "..."); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + String yesterday = dateFormat.format(System.currentTimeMillis() + - 86400000L); statsFile.getParentFile().mkdirs(); BufferedWriter bw = new BufferedWriter(new FileWriter(statsFile)); - bw.append("source,date,q1,md,q3,timeouts,failures,requests\n"); + bw.append("date,size,source,q1,md,q3,timeouts,failures," + + "requests\n"); for (String s : stats.values()) { - bw.append(s + "\n"); + if (s.compareTo(yesterday) < 0) { + bw.append(s + "\n"); + } } bw.close(); logger.fine("Finished writing file " + statsFile.getAbsolutePath() @@ -273,102 +282,6 @@ public class TorperfProcessor { dumpStats.append("\n" + lastSource + " " + lastKnownObservation); } logger.info(dumpStats.toString()); - - /* Write results to database. */ - if (connectionURL != null) { - try { - Map<String, String> insertRows = new HashMap<String, String>(); - insertRows.putAll(stats); - Set<String> updateRows = new HashSet<String>(); - Connection conn = DriverManager.getConnection(connectionURL); - conn.setAutoCommit(false); - Statement statement = conn.createStatement(); - ResultSet rs = statement.executeQuery( - "SELECT date, source, q1, md, q3, timeouts, failures, " - + "requests FROM torperf_stats"); - while (rs.next()) { - String date = rs.getDate(1).toString(); - String source = rs.getString(2); - String key = source + "," + date; - if (insertRows.containsKey(key)) { - String insertRow = insertRows.remove(key); - String[] newStats = insertRow.split(","); - long newQ1 = Long.parseLong(newStats[2]); - long newMd = Long.parseLong(newStats[3]); - long newQ3 = Long.parseLong(newStats[4]); - long newTimeouts = Long.parseLong(newStats[5]); - long newFailures = Long.parseLong(newStats[6]); - long newRequests = Long.parseLong(newStats[7]); - long oldQ1 = rs.getLong(3); - long oldMd = rs.getLong(4); - long oldQ3 = rs.getLong(5); - long oldTimeouts = rs.getLong(6); - long oldFailures = rs.getLong(7); - long oldRequests = rs.getLong(8); - if (newQ1 != oldQ1 || newMd != oldMd || newQ3 != oldQ3 || - newTimeouts != oldTimeouts || - newFailures != oldFailures || - newRequests != oldRequests) { - updateRows.add(insertRow); - } - } - } - PreparedStatement psU = conn.prepareStatement( - "UPDATE torperf_stats SET q1 = ?, md = ?, q3 = ?, " - + "timeouts = ?, failures = ?, requests = ? " - + "WHERE date = ? AND source = ?"); - for (String row : updateRows) { - String[] newStats = row.split(","); - String source = newStats[0]; - java.sql.Date date = java.sql.Date.valueOf(newStats[1]); - long q1 = Long.parseLong(newStats[2]); - long md = Long.parseLong(newStats[3]); - long q3 = Long.parseLong(newStats[4]); - long timeouts = Long.parseLong(newStats[5]); - long failures = Long.parseLong(newStats[6]); - long requests = Long.parseLong(newStats[7]); - psU.clearParameters(); - psU.setLong(1, q1); - psU.setLong(2, md); - psU.setLong(3, q3); - psU.setLong(4, timeouts); - psU.setLong(5, failures); - psU.setLong(6, requests); - psU.setDate(7, date); - psU.setString(8, source); - psU.executeUpdate(); - } - PreparedStatement psI = conn.prepareStatement( - "INSERT INTO torperf_stats (q1, md, q3, timeouts, failures, " - + "requests, date, source) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"); - for (String row : insertRows.values()) { - String[] newStats = row.split(","); - String source = newStats[0]; - java.sql.Date date = java.sql.Date.valueOf(newStats[1]); - long q1 = Long.parseLong(newStats[2]); - long md = Long.parseLong(newStats[3]); - long q3 = Long.parseLong(newStats[4]); - long timeouts = Long.parseLong(newStats[5]); - long failures = Long.parseLong(newStats[6]); - long requests = Long.parseLong(newStats[7]); - psI.clearParameters(); - psI.setLong(1, q1); - psI.setLong(2, md); - psI.setLong(3, q3); - psI.setLong(4, timeouts); - psI.setLong(5, failures); - psI.setLong(6, requests); - psI.setDate(7, date); - psI.setString(8, source); - psI.executeUpdate(); - } - conn.commit(); - conn.close(); - } catch (SQLException e) { - logger.log(Level.WARNING, "Failed to add torperf stats to " - + "database.", e); - } - } } } diff --git a/shared/bin/50-run-legacy-stats.sh b/shared/bin/50-run-legacy-stats.sh index b2b9f1a..75fe66c 100755 --- a/shared/bin/50-run-legacy-stats.sh +++ b/shared/bin/50-run-legacy-stats.sh @@ -5,7 +5,6 @@ psql -U metrics tordir -c 'SELECT * FROM refresh_all();' mkdir -p stats psql -c 'COPY (SELECT * FROM stats_servers) TO STDOUT WITH CSV HEADER;' tordir > stats/servers.csv psql -c 'COPY (SELECT * FROM stats_bandwidth) TO STDOUT WITH CSV HEADER;' tordir > stats/bandwidth.csv -psql -c 'COPY (SELECT * FROM stats_torperf) TO STDOUT WITH CSV HEADER;' tordir > stats/torperf.csv psql -c 'COPY (SELECT * FROM stats_connbidirect) TO STDOUT WITH CSV HEADER;' tordir > stats/connbidirect.csv cd ../../

1 0

[torspec/master] Fix a sentence that teor pointed out where
by nickm＠torproject.org 11 Jan '16

11 Jan '16

commit 9cd6b0ef6955d425499c30e8584bceee402ca8fd Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Jan 11 09:10:37 2016 -0500 Fix a sentence that teor pointed out where --- proposals/264-subprotocol-versions.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/proposals/264-subprotocol-versions.txt b/proposals/264-subprotocol-versions.txt index d0626f0..f8f5188 100644 --- a/proposals/264-subprotocol-versions.txt +++ b/proposals/264-subprotocol-versions.txt @@ -260,7 +260,8 @@ Status: Open Because all relays currently on the network are 0.2.4.19 or later, we can require 0.2.4.19, and use 0.2.4.19 as the minimal version so we - we don't need to do code archeology to determine which number + we don't need to do code archeology to determine how many + no-longer-relevant versions of each protocol once existed. Adding new protocol types is pretty cheap, given compression.

1 0

[tor/master] Try a little harder to only use SecureZeroMemory when it's present
by nickm＠torproject.org 11 Jan '16

11 Jan '16

commit 1d6dd288e1c084a5118785899cca910e8c69fbb1 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Jan 11 09:02:42 2016 -0500 Try a little harder to only use SecureZeroMemory when it's present We could be using AC_CHECK_FUNC_DECL too, but it shouldn't be needed. --- configure.ac | 2 ++ src/common/compat_openssl.h | 10 ++++++++-- src/common/crypto.c | 4 +++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index b62b4d3..6d2312d 100644 --- a/configure.ac +++ b/configure.ac @@ -376,6 +376,8 @@ AM_CONDITIONAL(THREADS_PTHREADS, test "$bwin32" = "false") AC_CHECK_FUNCS( _NSGetEnviron \ + RtlSecureZeroMemory \ + SecureZeroMemory \ accept4 \ backtrace \ backtrace_symbols_fd \ diff --git a/src/common/compat_openssl.h b/src/common/compat_openssl.h index 5825ff7..9c98181 100644 --- a/src/common/compat_openssl.h +++ b/src/common/compat_openssl.h @@ -19,8 +19,14 @@ #error "We require OpenSSL >= 1.0.0" #endif -#if OPENSSL_VERSION_NUMBER < OPENSSL_V_SERIES(1,1,0) || \ - defined(LIBRESSL_VERSION_NUMBER) +#if OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0) && \ + ! defined(LIBRESSL_VERSION_NUMBER) +/* We define this macro if we're trying to build with the majorly refactored + * API in OpenSSL 1.1 */ +#define OPENSSL_1_1_API +#endif + +#ifndef OPENSSL_1_1_API #define OPENSSL_VERSION SSLEAY_VERSION #define OpenSSL_version(v) SSLeay_version(v) #define OpenSSL_version_num() SSLeay() diff --git a/src/common/crypto.c b/src/common/crypto.c index 2f498ac..9cc5ee0 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -2960,9 +2960,11 @@ memwipe(void *mem, uint8_t byte, size_t sz) * have this function call "memset". A smart compiler could inline it, then * eliminate dead memsets, and declare itself to be clever. */ -#ifdef _WIN32 +#if defined(SecureZeroMemory) || defined(HAVE_SECUREZEROMEMORY) /* Here's what you do on windows. */ SecureZeroMemory(mem,sz); +#elif defined(HAVE_RTLSECUREZEROMEMORY) + RtlSecureZeroMemory(mem,sz); #elif defined(HAVE_EXPLICIT_BZERO) /* The BSDs provide this. */ explicit_bzero(mem, sz);

1 0

[tor/master] Add family argument to get_interface_addresses_raw (and subfunctions).
by nickm＠torproject.org 11 Jan '16

11 Jan '16

commit 44497e9ebc5231dcdb6a392093f85990fe13542e Author: rl1987 <rl1987(a)sdf.lonestar.org> Date: Sun Jan 3 15:35:45 2016 +0100 Add family argument to get_interface_addresses_raw (and subfunctions). --- changes/feature17950 | 5 +++++ src/common/address.c | 44 +++++++++++++++++++++++--------------------- src/common/address.h | 15 ++++++++++----- src/test/test_address.c | 11 ++++++----- 4 files changed, 44 insertions(+), 31 deletions(-) diff --git a/changes/feature17950 b/changes/feature17950 new file mode 100644 index 0000000..5ea83ec --- /dev/null +++ b/changes/feature17950 @@ -0,0 +1,5 @@ + o Minor features: + - Add a family argument to get_interface_addresses_raw() and + subfunctions to make network interface address interogation more + efficient. Now Tor can specifically ask for IPv4, IPv6 or both + types of interfaces from the operating system. Resolves ticket 17950. diff --git a/src/common/address.c b/src/common/address.c index 69a8098..1e6bb26 100644 --- a/src/common/address.c +++ b/src/common/address.c @@ -1274,7 +1274,7 @@ typedef ULONG (WINAPI *GetAdaptersAddresses_fn_t)( * into smartlist of tor_addr_t structures. */ STATIC smartlist_t * -ifaddrs_to_smartlist(const struct ifaddrs *ifa) +ifaddrs_to_smartlist(const struct ifaddrs *ifa, sa_family_t family) { smartlist_t *result = smartlist_new(); const struct ifaddrs *i; @@ -1285,8 +1285,7 @@ ifaddrs_to_smartlist(const struct ifaddrs *ifa) continue; if (!i->ifa_addr) continue; - if (i->ifa_addr->sa_family != AF_INET && - i->ifa_addr->sa_family != AF_INET6) + if (family != AF_UNSPEC && i->ifa_addr->sa_family != family) continue; if (tor_addr_from_sockaddr(&tmp, i->ifa_addr, NULL) < 0) continue; @@ -1301,7 +1300,7 @@ ifaddrs_to_smartlist(const struct ifaddrs *ifa) * tor_addr_t structures. */ STATIC smartlist_t * -get_interface_addresses_ifaddrs(int severity) +get_interface_addresses_ifaddrs(int severity, sa_family_t family) { /* Most free Unixy systems provide getifaddrs, which gives us a linked list @@ -1314,7 +1313,7 @@ get_interface_addresses_ifaddrs(int severity) return NULL; } - result = ifaddrs_to_smartlist(ifa); + result = ifaddrs_to_smartlist(ifa, family); freeifaddrs(ifa); @@ -1356,7 +1355,7 @@ ip_adapter_addresses_to_smartlist(const IP_ADAPTER_ADDRESSES *addresses) * tor_addr_t structures. */ STATIC smartlist_t * -get_interface_addresses_win32(int severity) +get_interface_addresses_win32(int severity, sa_family_t family) { /* Windows XP began to provide GetAdaptersAddresses. Windows 2000 had a @@ -1390,7 +1389,7 @@ get_interface_addresses_win32(int severity) /* Guess how much space we need. */ size = 15*1024; addresses = tor_malloc(size); - res = fn(AF_UNSPEC, FLAGS, NULL, addresses, &size); + res = fn(family, FLAGS, NULL, addresses, &size); if (res == ERROR_BUFFER_OVERFLOW) { /* we didn't guess that we needed enough space; try again */ tor_free(addresses); @@ -1464,7 +1463,7 @@ ifreq_to_smartlist(char *buf, size_t buflen) * tor_addr_t structures. */ STATIC smartlist_t * -get_interface_addresses_ioctl(int severity) +get_interface_addresses_ioctl(int severity, sa_family_t family) { /* Some older unixy systems make us use ioctl(SIOCGIFCONF) */ struct ifconf ifc; @@ -1473,7 +1472,14 @@ get_interface_addresses_ioctl(int severity) /* This interface, AFAICT, only supports AF_INET addresses, * except on AIX. For Solaris, we could use SIOCGLIFCONF. */ - fd = socket(AF_INET, SOCK_DGRAM, 0); + + /* FIXME: for now, we bail out if family is not AF_INET since + * ioctl() technique supports non-IPv4 interface addresses on + * a small number of niche systems only. */ + if (family != AF_INET) + return NULL; + + fd = socket(family, SOCK_DGRAM, 0); if (fd < 0) { tor_log(severity, LD_NET, "socket failed: %s", strerror(errno)); goto done; @@ -1508,21 +1514,23 @@ get_interface_addresses_ioctl(int severity) /** Try to ask our network interfaces what addresses they are bound to. * Return a new smartlist of tor_addr_t on success, and NULL on failure. * (An empty smartlist indicates that we successfully learned that we have no - * addresses.) Log failure messages at severity. */ + * addresses.) Log failure messages at severity. Only return the + * interface addresses of requested family and ignore the addresses + * of other address families. */ MOCK_IMPL(smartlist_t *, -get_interface_addresses_raw,(int severity)) +get_interface_addresses_raw,(int severity, sa_family_t family)) { smartlist_t *result = NULL; #if defined(HAVE_IFADDRS_TO_SMARTLIST) - if ((result = get_interface_addresses_ifaddrs(severity))) + if ((result = get_interface_addresses_ifaddrs(severity, family))) return result; #endif #if defined(HAVE_IP_ADAPTER_TO_SMARTLIST) - if ((result = get_interface_addresses_win32(severity))) + if ((result = get_interface_addresses_win32(severity, family))) return result; #endif #if defined(HAVE_IFCONF_TO_SMARTLIST) - if ((result = get_interface_addresses_ioctl(severity))) + if ((result = get_interface_addresses_ioctl(severity, family))) return result; #endif (void) severity; @@ -1686,15 +1694,9 @@ MOCK_IMPL(smartlist_t *,get_interface_address6_list,(int severity, tor_addr_t addr; /* Try to do this the smart way if possible. */ - if ((addrs = get_interface_addresses_raw(severity))) { + if ((addrs = get_interface_addresses_raw(severity, family))) { SMARTLIST_FOREACH_BEGIN(addrs, tor_addr_t *, a) { - if (family != AF_UNSPEC && family != tor_addr_family(a)) { - SMARTLIST_DEL_CURRENT(addrs, a); - tor_free(a); - continue; - } - if (tor_addr_is_loopback(a) || tor_addr_is_multicast(a)) { SMARTLIST_DEL_CURRENT(addrs, a); diff --git a/src/common/address.h b/src/common/address.h index 684ba65..7609013 100644 --- a/src/common/address.h +++ b/src/common/address.h @@ -311,26 +311,31 @@ get_interface_address_list(int severity, int include_internal) tor_addr_port_t *tor_addr_port_new(const tor_addr_t *addr, uint16_t port); #ifdef ADDRESS_PRIVATE -MOCK_DECL(smartlist_t *,get_interface_addresses_raw,(int severity)); +MOCK_DECL(smartlist_t *,get_interface_addresses_raw,(int severity, + sa_family_t family)); MOCK_DECL(int,get_interface_address6_via_udp_socket_hack,(int severity, sa_family_t family, tor_addr_t *addr)); #ifdef HAVE_IFADDRS_TO_SMARTLIST -STATIC smartlist_t *ifaddrs_to_smartlist(const struct ifaddrs *ifa); -STATIC smartlist_t *get_interface_addresses_ifaddrs(int severity); +STATIC smartlist_t *ifaddrs_to_smartlist(const struct ifaddrs *ifa, + sa_family_t family); +STATIC smartlist_t *get_interface_addresses_ifaddrs(int severity, + sa_family_t family); #endif #ifdef HAVE_IP_ADAPTER_TO_SMARTLIST STATIC smartlist_t *ip_adapter_addresses_to_smartlist( const IP_ADAPTER_ADDRESSES *addresses); -STATIC smartlist_t *get_interface_addresses_win32(int severity); +STATIC smartlist_t *get_interface_addresses_win32(int severity, + sa_family_t family); #endif #ifdef HAVE_IFCONF_TO_SMARTLIST STATIC smartlist_t *ifreq_to_smartlist(char *ifr, size_t buflen); -STATIC smartlist_t *get_interface_addresses_ioctl(int severity); +STATIC smartlist_t *get_interface_addresses_ioctl(int severity, + sa_family_t family); #endif #endif // ADDRESS_PRIVATE diff --git a/src/test/test_address.c b/src/test/test_address.c index 9f3d81c..3ddf247 100644 --- a/src/test/test_address.c +++ b/src/test/test_address.c @@ -220,7 +220,7 @@ test_address_ifaddrs_to_smartlist(void *arg) ifa_ipv6->ifa_dstaddr = NULL; ifa_ipv6->ifa_data = NULL; - smartlist = ifaddrs_to_smartlist(ifa); + smartlist = ifaddrs_to_smartlist(ifa, AF_UNSPEC); tt_assert(smartlist); tt_assert(smartlist_len(smartlist) == 3); @@ -281,7 +281,7 @@ test_address_get_if_addrs_ifaddrs(void *arg) (void)arg; - results = get_interface_addresses_ifaddrs(LOG_ERR); + results = get_interface_addresses_ifaddrs(LOG_ERR, AF_UNSPEC); tt_assert(results); /* Some FreeBSD jails don't have localhost IP address. Instead, they only @@ -314,7 +314,7 @@ test_address_get_if_addrs_win32(void *arg) (void)arg; - results = get_interface_addresses_win32(LOG_ERR); + results = get_interface_addresses_win32(LOG_ERR, AF_UNSPEC); tt_int_op(smartlist_len(results),>=,1); tt_assert(smartlist_contains_localhost_tor_addr(results)); @@ -511,7 +511,7 @@ test_address_get_if_addrs_ioctl(void *arg) (void)arg; - result = get_interface_addresses_ioctl(LOG_ERR); + result = get_interface_addresses_ioctl(LOG_ERR, AF_INET); /* On an IPv6-only system, this will fail and return NULL tt_assert(result); @@ -845,9 +845,10 @@ test_address_get_if_addrs6_list_no_internal(void *arg) static int called_get_interface_addresses_raw = 0; static smartlist_t * -mock_get_interface_addresses_raw_fail(int severity) +mock_get_interface_addresses_raw_fail(int severity, sa_family_t family) { (void)severity; + (void)family; called_get_interface_addresses_raw++; return smartlist_new();

1 0

[tor/master] Tweak ioctl case.
by nickm＠torproject.org 11 Jan '16

11 Jan '16

commit 680d0701e53fe583b2a048b4a3a60f6fea176b64 Author: rl1987 <rl1987(a)sdf.lonestar.org> Date: Wed Jan 6 11:47:31 2016 +0100 Tweak ioctl case. --- src/common/address.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/common/address.c b/src/common/address.c index 1e6bb26..f12f140 100644 --- a/src/common/address.c +++ b/src/common/address.c @@ -1473,10 +1473,13 @@ get_interface_addresses_ioctl(int severity, sa_family_t family) /* This interface, AFAICT, only supports AF_INET addresses, * except on AIX. For Solaris, we could use SIOCGLIFCONF. */ - /* FIXME: for now, we bail out if family is not AF_INET since + /* Bail out if family is neither AF_INET nor AF_UNSPEC since * ioctl() technique supports non-IPv4 interface addresses on - * a small number of niche systems only. */ - if (family != AF_INET) + * a small number of niche systems only. If family is AF_UNSPEC, + * fall back to getting AF_INET addresses only. */ + if (family == AF_UNSPEC) + family = AF_INET; + else if (family != AF_INET) return NULL; fd = socket(family, SOCK_DGRAM, 0);

1 0

[tor/master] Re-add the removed address family check.
by nickm＠torproject.org 11 Jan '16

11 Jan '16

commit fd26c1d994e88fbfc8546551e59633679cae5c78 Author: rl1987 <rl1987(a)sdf.lonestar.org> Date: Sat Jan 9 15:03:54 2016 +0100 Re-add the removed address family check. --- src/common/address.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/common/address.c b/src/common/address.c index f12f140..3ffa15e 100644 --- a/src/common/address.c +++ b/src/common/address.c @@ -1285,6 +1285,9 @@ ifaddrs_to_smartlist(const struct ifaddrs *ifa, sa_family_t family) continue; if (!i->ifa_addr) continue; + if (i->ifa_addr->sa_family != AF_INET && + i->ifa_addr->sa_family != AF_INET6) + continue; if (family != AF_UNSPEC && i->ifa_addr->sa_family != family) continue; if (tor_addr_from_sockaddr(&tmp, i->ifa_addr, NULL) < 0)

1 0

[tor/master] Merge remote-tracking branch 'rl1987/feature17950'
by nickm＠torproject.org 11 Jan '16

11 Jan '16

commit d10ea49588701b29148a27c87701961227d63ba9 Merge: 8fc27ac fd26c1d Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Jan 11 08:54:51 2016 -0500 Merge remote-tracking branch 'rl1987/feature17950' changes/feature17950 | 5 +++++ src/common/address.c | 46 +++++++++++++++++++++++++++------------------- src/common/address.h | 15 ++++++++++----- src/test/test_address.c | 11 ++++++----- 4 files changed, 48 insertions(+), 29 deletions(-)

1 0

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits January 2016