tor-commits May 2018

tor-commits@lists.torproject.org

17 participants
1514 discussions

[stem/master] Resume parsing proc connections via split
by atagar＠torproject.org 14 May '18

14 May '18

commit d0f402c6f0220087320c58cecb25b53d2bd6a156 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon May 14 09:43:03 2018 -0700 Resume parsing proc connections via split While back I put quite a bit of elbow grease into speeding up connection resolution. To this end commit 8da78a9 squeezed out a 30% improvement by parsing proc contents using indices rather than split. I was torn at the time since this made our code quite a bit more verbose and fragile, but all things being equal seemed worth the speed boost. However, in digging into connection resolution discrepancies Toralf spotted that when a relay has at least 10,000 connections the first column (sl) gets wider, throwing our parser off. Proc is clearly intended to be parsed via delimiters, not static indices. As such largely reverting commit 8da78a9. --- stem/util/proc.py | 62 +++++++++++-------------------------------------- test/unit/util/proc.py | 63 +++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 68 insertions(+), 57 deletions(-) diff --git a/stem/util/proc.py b/stem/util/proc.py index 6a316634..9969b59c 100644 --- a/stem/util/proc.py +++ b/stem/util/proc.py @@ -379,65 +379,29 @@ def connections(pid = None, user = None): continue # ipv6 proc contents are optional protocol = proc_file_path[10:].rstrip('6') # 'tcp' or 'udp' - is_tcp, is_ipv6 = protocol == 'tcp', proc_file_path.endswith('6') - title = '' + is_ipv6 = proc_file_path.endswith('6') try: with open(proc_file_path, 'rb') as proc_file: - title = proc_file.readline() - - if b'local_address' in title: - laddr_start = title.index(b'local_address') - laddr_end = laddr_start + (8 if not is_ipv6 else 32) - - lport_start = laddr_end + 1 - lport_end = lport_start + 4 - else: - raise IOError("title line missing 'local_address', %s" % title) - - if b'rem_address' in title or b'remote_address' in title: - raddr_start = title.index(b'rem_address') if b'rem_address' in title else title.index(b'remote_address') - raddr_end = raddr_start + (8 if not is_ipv6 else 32) - - rport_start = raddr_end + 1 - rport_end = rport_start + 4 - else: - raise IOError("title line missing 'remote_address', %s" % title) - - if b'st' in title: - status_start = title.index(b'st') - status_end = status_start + 2 - else: - raise IOError("title line missing 'st', %s" % title) - - if b'retrnsmt' in title and b'uid' in title: - # unlike the above fields uid is right aligned - uid_start = title.index(b'retrnsmt') + 9 - uid_end = title.index(b'uid') + 3 - elif b'retrnsmt' not in title: - raise IOError("title line missing 'retrnsmt', %s" % title) - else: - raise IOError("title line missing 'uid', %s" % title) - - if b'timeout' in title: - # inodes can lack a header, and are a dynamic size - inode_start = title.index(b'timeout') + 8 - else: - raise IOError("title line missing 'timeout', %s" % title) + proc_file.readline() # skip the first line for line in proc_file: - if inodes and line[inode_start:].split(b' ', 1)[0] not in inodes: + _, l_dst, r_dst, status, _, _, _, uid, _, inode = line.split()[:10] + + if inodes and inode not in inodes: continue - elif process_uid and line[uid_start:uid_end].strip() != process_uid: + elif process_uid and uid != process_uid: continue - elif is_tcp and line[status_start:status_end] != b'01': + elif protocol == 'tcp' and status != b'01': continue # skip tcp connections that aren't yet established - l_addr = _unpack_addr(line[laddr_start:laddr_end]) - l_port = int(line[lport_start:lport_end], 16) + div = l_dst.find(':') + l_addr = _unpack_addr(l_dst[:div]) + l_port = int(l_dst[div + 1:], 16) - r_addr = _unpack_addr(line[raddr_start:raddr_end]) - r_port = int(line[rport_start:rport_end], 16) + div = r_dst.find(':') + r_addr = _unpack_addr(r_dst[:div]) + r_port = int(r_dst[div + 1:], 16) if r_addr == '0.0.0.0' or r_addr == '0000:0000:0000:0000:0000:0000': continue # no address diff --git a/test/unit/util/proc.py b/test/unit/util/proc.py index 562d618a..7ee4ecda 100644 --- a/test/unit/util/proc.py +++ b/test/unit/util/proc.py @@ -33,6 +33,13 @@ TCP6_CONTENT = b"""\ 10: 0000000000000000FFFF00004B9E0905:1466 0000000000000000FFFF00002186364E:951E 01 00000000:00000000 02:00090E70 00000000 106 0 41512577 2 0000000000000000 26 4 31 10 -1 """ +SL_WIDTH_CHANGE = b"""\ + sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode + 0: 11111111:1111 22222233:2244 01 44444444:44444444 55:55555555 66666666 1111 8 99999999 + 1: 22222222:2222 33333344:3355 01 44444444:44444444 55:55555555 66666666 1111 8 88888888 +12675: 33333333:3333 44444455:4466 01 44444444:44444444 55:55555555 66666666 1111 8 77777777 +""".rstrip() + class TestProc(unittest.TestCase): @patch('stem.util.proc._get_line') @@ -216,9 +223,7 @@ class TestProc(unittest.TestCase): udp = TITLE_LINE + b'\n A: BBBBBBBB:BBBB CCCCCCCC:CCCC DD EEEEEEEE:EEEEEEEE FF:FFFFFFFF GGGGGGGG 1111 H IIIIIIII' path_exists_mock.side_effect = lambda param: { - '/proc/net/tcp': True, '/proc/net/tcp6': False, - '/proc/net/udp': True, '/proc/net/udp6': False }[param] @@ -227,12 +232,12 @@ class TestProc(unittest.TestCase): '/proc/net/udp': io.BytesIO(udp) }[param] - expected_results = [ + expected = [ Connection('17.17.17.17', 4369, '34.34.34.34', 8738, 'tcp', False), Connection('187.187.187.187', 48059, '204.204.204.204', 52428, 'udp', False), ] - self.assertEqual(expected_results, proc.connections(pid)) + self.assertEqual(expected, proc.connections(pid)) @patch('os.listdir') @patch('os.path.exists') @@ -265,12 +270,12 @@ class TestProc(unittest.TestCase): '/proc/net/udp': io.BytesIO(TITLE_LINE), }[param] - expected_results = [ + expected = [ Connection('2a01:04f8:0190:514a:0000:0000:0000:0002', 443, '2001:0638:a000:4140:0000:0000:ffff:0189', 40435, 'tcp', True), Connection('2a01:04f8:0190:514a:0000:0000:0000:0002', 443, '2001:0858:0002:0002:aabb:0000:563b:1526', 44469, 'tcp', True), ] - self.assertEqual(expected_results, proc.connections(pid = pid)) + self.assertEqual(expected, proc.connections(pid = pid)) @patch('os.path.exists') @patch('pwd.getpwnam') @@ -293,10 +298,52 @@ class TestProc(unittest.TestCase): '/proc/net/udp': io.BytesIO(TITLE_LINE), }[param] - expected_results = [ + expected = [ Connection('0000:0000:0000:0000:0000:ffff:0509:9e4b', 5222, '0000:0000:0000:0000:0000:ffff:4e36:8621', 38330, 'tcp', True), Connection('2a01:04f8:0190:514a:0000:0000:0000:0002', 5269, '2001:06f8:126f:0011:0000:0000:0000:0026', 50594, 'tcp', True), Connection('0000:0000:0000:0000:0000:ffff:0509:9e4b', 5222, '0000:0000:0000:0000:0000:ffff:4e36:8621', 38174, 'tcp', True), ] - self.assertEqual(expected_results, proc.connections(user = 'me')) + self.assertEqual(expected, proc.connections(user = 'me')) + + @patch('os.listdir') + @patch('os.path.exists') + @patch('os.readlink') + @patch('stem.util.proc.open', create = True) + def test_high_connection_count(self, open_mock, readlink_mock, path_exists_mock, listdir_mock): + """ + When we have over ten thousand connections the 'SL' column's width changes. + Checking that we account for this. + """ + + pid = 1111 + + listdir_mock.side_effect = lambda param: { + '/proc/%s/fd' % pid: ['1', '2', '3', '4'], + }[param] + + readlink_mock.side_effect = lambda param: { + '/proc/%s/fd/1' % pid: 'socket:[99999999]', + '/proc/%s/fd/2' % pid: 'socket:[88888888]', + '/proc/%s/fd/3' % pid: 'socket:[77777777]', + '/proc/%s/fd/4' % pid: 'pipe:[30303]', + '/proc/%s/fd/5' % pid: 'pipe:[40404]', + }[param] + + path_exists_mock.side_effect = lambda param: { + '/proc/net/tcp6': False, + '/proc/net/udp6': False + }[param] + + open_mock.side_effect = lambda param, mode: { + '/proc/net/tcp': io.BytesIO(SL_WIDTH_CHANGE), + '/proc/net/udp': io.BytesIO(TITLE_LINE) + }[param] + + expected = [ + Connection('17.17.17.17', 4369, '51.34.34.34', 8772, 'tcp', False), + Connection('34.34.34.34', 8738, '68.51.51.51', 13141, 'tcp', False), + Connection('51.51.51.51', 13107, '85.68.68.68', 17510, 'tcp', False), + ] + + self.assertEqual(expected, proc.connections(pid))

1 0

[orbot/master] update to 16.0.2-RC-1 build 16020041
by n8fr8＠torproject.org 14 May '18

14 May '18

commit 65bf70f281abff4b3d7096fd74cdf42434d28546 Author: n8fr8 <nathan(a)freitas.net> Date: Mon May 14 16:40:06 2018 -0400 update to 16.0.2-RC-1 build 16020041 --- app/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/build.gradle b/app/build.gradle index 6e689c53..7fd095b1 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -44,8 +44,8 @@ android { minSdkVersion 16 applicationId 'org.torproject.android' targetSdkVersion 27 - versionCode 16020013 - versionName '16.0.2-BETA-3' + versionCode 16020041 + versionName '16.0.2-RC-1' archivesBaseName = "Orbot-$versionName" } }

1 0

[orbot/master] update code to properly unzip entries from APK zip if needed
by n8fr8＠torproject.org 14 May '18

14 May '18

commit 4bdfb79a68845ce2677c40eadde36fd7e294d6ea Author: n8fr8 <nathan(a)freitas.net> Date: Mon May 14 12:44:45 2018 -0400 update code to properly unzip entries from APK zip if needed --- .../android/service/util/NativeLoader.java | 114 +++++++++++++++++++++ .../service/util/OtherResourceInstaller.java | 29 ++++-- 2 files changed, 136 insertions(+), 7 deletions(-) diff --git a/orbotservice/src/main/java/org/torproject/android/service/util/NativeLoader.java b/orbotservice/src/main/java/org/torproject/android/service/util/NativeLoader.java new file mode 100644 index 00000000..d6a25436 --- /dev/null +++ b/orbotservice/src/main/java/org/torproject/android/service/util/NativeLoader.java @@ -0,0 +1,114 @@ + +package org.torproject.android.service.util; + +import android.content.Context; +import android.os.Build; +import android.util.Log; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import java.util.zip.ZipInputStream; + +public class NativeLoader { + + private final static String TAG = "TorNativeLoader"; + + private static boolean loadFromZip(Context context, String libName, File destLocalFile, String folder) { + + + ZipFile zipFile = null; + ZipInputStream stream = null; + try { + zipFile = new ZipFile(context.getApplicationInfo().sourceDir); + ZipEntry entry = zipFile.getEntry("lib/" + folder + "/" + libName + ".so"); + if (entry == null) { + throw new Exception("Unable to find file in apk:" + "lib/" + folder + "/" + libName); + } + //the zip file entry is also zipped itself! + stream = new ZipInputStream(zipFile.getInputStream(entry)); + stream.getNextEntry(); + + OutputStream out = new FileOutputStream(destLocalFile); + byte[] buf = new byte[4096]; + int len; + while ((len = stream.read(buf)) > 0) { + Thread.yield(); + out.write(buf, 0, len); + } + out.close(); + + if (Build.VERSION.SDK_INT >= 9) { + destLocalFile.setReadable(true, false); + destLocalFile.setExecutable(true, false); + destLocalFile.setWritable(true); + } + + + return true; + } catch (Exception e) { + Log.e(TAG, e.getMessage()); + } finally { + if (stream != null) { + try { + stream.close(); + } catch (Exception e) { + Log.e(TAG, e.getMessage()); + } + } + if (zipFile != null) { + try { + zipFile.close(); + } catch (Exception e) { + Log.e(TAG, e.getMessage()); + } + } + } + return false; + } + + public static synchronized boolean initNativeLibs(Context context, String binaryName, File destLocalFile) { + + try { + String folder = null; + + try { + + if (Build.CPU_ABI.equalsIgnoreCase("armeabi-v7a")) { + folder = "armeabi-v7a"; + } else if (Build.CPU_ABI.startsWith("armeabi")) { + folder = "armeabi"; + } else if (Build.CPU_ABI.equalsIgnoreCase("x86")) { + folder = "x86"; + } else if (Build.CPU_ABI.equalsIgnoreCase("mips")) { + folder = "mips"; + } else { + folder = "armeabi"; + //FileLog.e("tmessages", "Unsupported arch: " + Build.CPU_ABI); + } + } catch (Exception e) { + // FileLog.e("tmessages", e); + Log.e(TAG, e.getMessage()); + folder = "armeabi"; + } + + + String javaArch = System.getProperty("os.arch"); + if (javaArch != null && javaArch.contains("686")) { + folder = "x86"; + } + + if (loadFromZip(context, binaryName, destLocalFile, folder)) { + return true; + } + + } catch (Throwable e) { + e.printStackTrace(); + } + + return false; + } +} diff --git a/orbotservice/src/main/java/org/torproject/android/service/util/OtherResourceInstaller.java b/orbotservice/src/main/java/org/torproject/android/service/util/OtherResourceInstaller.java index 672d6411..c02c5c54 100644 --- a/orbotservice/src/main/java/org/torproject/android/service/util/OtherResourceInstaller.java +++ b/orbotservice/src/main/java/org/torproject/android/service/util/OtherResourceInstaller.java @@ -72,22 +72,37 @@ public class OtherResourceInstaller implements TorServiceConstants { File libBinary = new File(getNativeLibraryDir(context),OBFSCLIENT_ASSET_KEY + ".so"); outFile = new File(installFolder, OBFSCLIENT_ASSET_KEY); + if (libBinary.exists()) { + if ((!outFile.exists()) || (libBinary.lastModified() > outFile.lastModified())) { + streamToFile(new FileInputStream(libBinary), outFile, false, true); + } - if ((!outFile.exists()) || (libBinary.lastModified() > outFile.lastModified())) { - streamToFile(new FileInputStream(libBinary), outFile, false, true); + setExecutable(outFile); + } + else + { + NativeLoader.initNativeLibs(context,OBFSCLIENT_ASSET_KEY,outFile); } - setExecutable(outFile); + if (!outFile.exists()) + return false; libBinary = new File(getNativeLibraryDir(context),PDNSD_ASSET_KEY + ".so"); outFile = new File(installFolder, PDNSD_ASSET_KEY); + if (libBinary.exists()) { + if ((!outFile.exists()) || (libBinary.lastModified() > outFile.lastModified())) { + streamToFile(new FileInputStream(libBinary), outFile, false, true); + } - if ((!outFile.exists()) || (libBinary.lastModified() > outFile.lastModified())) { - streamToFile(new FileInputStream(libBinary), outFile, false, true); + setExecutable(outFile); + } + else + { + NativeLoader.initNativeLibs(context,PDNSD_ASSET_KEY,outFile); } - setExecutable(outFile); - + if (!outFile.exists()) + return false; return true; }

1 0

[orbot/master] update to latest tor-android-binary library
by n8fr8＠torproject.org 14 May '18

14 May '18

commit 1dc740a76c815c42e26440bf8f4fd54ab8cd193f Author: n8fr8 <nathan(a)freitas.net> Date: Mon May 14 12:45:13 2018 -0400 update to latest tor-android-binary library --- orbotservice/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orbotservice/build.gradle b/orbotservice/build.gradle index 020ce563..aa35322f 100644 --- a/orbotservice/build.gradle +++ b/orbotservice/build.gradle @@ -27,7 +27,7 @@ android { dependencies { implementation project(':jsocksAndroid') - implementation 'org.torproject:tor-android-binary:0.3.3.5-rc-1' + implementation 'org.torproject:tor-android-binary:0.3.3.5-rc-3' implementation 'com.android.support:appcompat-v7:27.1.1' implementation 'com.jrummyapps:android-shell:1.0.1' implementation fileTree(dir: 'libs', include: ['.so'])

1 0

[orbot/master] update native build to support armeabi-v7a
by n8fr8＠torproject.org 14 May '18

14 May '18

commit 41e223fd3973a29c500dc05025ba9221ad4c4945 Author: n8fr8 <nathan(a)freitas.net> Date: Mon May 14 15:17:40 2018 -0400 update native build to support armeabi-v7a --- orbotservice/src/main/jni/Application.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orbotservice/src/main/jni/Application.mk b/orbotservice/src/main/jni/Application.mk index 20921781..9e5d89dd 100644 --- a/orbotservice/src/main/jni/Application.mk +++ b/orbotservice/src/main/jni/Application.mk @@ -1,4 +1,4 @@ -APP_ABI := armeabi x86 +APP_ABI := armeabi armeabi-v7a x86 APP_PLATFORM := android-16 APP_STL := stlport_static NDK_TOOLCHAIN_VERSION := 4.9

1 0

[orbot/master] add armeabi-v7a build to split APKs
by n8fr8＠torproject.org 14 May '18

14 May '18

commit 9a1e6fce8f008f8c4cd4c6cef13c21852d1ea61a Author: n8fr8 <nathan(a)freitas.net> Date: Mon May 14 12:45:02 2018 -0400 add armeabi-v7a build to split APKs --- app/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/build.gradle b/app/build.gradle index 1b23a441..5382fe6e 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -64,7 +64,7 @@ android { reset() // Specifies a list of ABIs that Gradle should create APKs for. - include "x86", "armeabi" + include "x86", "armeabi", "armeabi-v7a" // Specifies that we do not want to also generate a universal APK that includes all ABIs. universalApk true

1 0

[orbot/master] update to 16.0.2-BETA-2 build 16020012
by n8fr8＠torproject.org 14 May '18

14 May '18

commit f7f03f5ab7129019722656944d927b23e8f8b09d Author: n8fr8 <nathan(a)freitas.net> Date: Mon May 14 12:48:56 2018 -0400 update to 16.0.2-BETA-2 build 16020012 --- app/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/build.gradle b/app/build.gradle index 5382fe6e..ed8ce52b 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -44,8 +44,8 @@ android { minSdkVersion 16 applicationId 'org.torproject.android' targetSdkVersion 27 - versionCode 16020011 - versionName '16.0.2-BETA-1' + versionCode 16020012 + versionName '16.0.2-BETA-2' } }

1 0

[orbot/master] update gradle to produce APKs appropriately named
by n8fr8＠torproject.org 14 May '18

14 May '18

commit ae282939c2534c151f63c8d2f3f76bb1867bbfbc Author: n8fr8 <nathan(a)freitas.net> Date: Mon May 14 15:18:03 2018 -0400 update gradle to produce APKs appropriately named --- app/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/app/build.gradle b/app/build.gradle index ed8ce52b..6a23272b 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -46,6 +46,7 @@ android { targetSdkVersion 27 versionCode 16020012 versionName '16.0.2-BETA-2' + archivesBaseName = "Orbot-$versionName" } }

1 0

[orbot/master] update to 16.0.2-BETA-3 build 16020013
by n8fr8＠torproject.org 14 May '18

14 May '18

commit 18528bcf61cbfe62d65f2934ecc818717dd699e4 Author: n8fr8 <nathan(a)freitas.net> Date: Mon May 14 15:34:19 2018 -0400 update to 16.0.2-BETA-3 build 16020013 --- app/build.gradle | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/build.gradle b/app/build.gradle index 6a23272b..6e689c53 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -16,8 +16,8 @@ android { } buildTypes { release { - minifyEnabled true - shrinkResources true + minifyEnabled false + shrinkResources false proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.txt' } } @@ -44,8 +44,8 @@ android { minSdkVersion 16 applicationId 'org.torproject.android' targetSdkVersion 27 - versionCode 16020012 - versionName '16.0.2-BETA-2' + versionCode 16020013 + versionName '16.0.2-BETA-3' archivesBaseName = "Orbot-$versionName" } }

1 0

[torspec/master] Bandwidth-measurement file specification, as sent to tor-dev
by nickm＠torproject.org 14 May '18

14 May '18

commit 9465f9c0713b2185643a976ef14e0959d3885e80 Author: juga0 <juga(a)riseup.net> Date: Mon May 14 14:31:05 2018 -0400 Bandwidth-measurement file specification, as sent to tor-dev --- bandwidth-file-spec.txt | 412 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 412 insertions(+) diff --git a/bandwidth-file-spec.txt b/bandwidth-file-spec.txt new file mode 100644 index 0000000..0755329 --- /dev/null +++ b/bandwidth-file-spec.txt @@ -0,0 +1,412 @@ + Tor Bandwidth List Format + juga + teor + +1. Scope and preliminaries + + This document describes the format of Tor's Bandwidth List, + version 1.0.0, 1.1.0 and later. + It is new specification for the existing format 1.0.0. + Describes a new format 1.1.0, which is backwards compatible with + 1.0.0 parsers. + + Since Tor version 0.2.4.12-alpha the directory authorities use + the Bandwidth List file called "V3BandwidthsFile" generated by + Torflow [1]. The format is described in Torflow's README.spec.txt and + is considered to be version 1.0.0. + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL + NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and + "OPTIONAL" in this document are to be interpreted as described in + RFC 2119. + +1.2. Acknowledgements + + The original bandwidth generator (Torflow) and format was + created by mike. Teor suggested to write this specification while + contributing on pastly's new bandwidth generator implementation. + + This specification was revised after feedback from: + + Nick Mathewson (nickm) + Iain Learmonth (irl) + +1.3 Outline + + The Tor directory protocol (dir-spec.txt [3]) sections 3.4.1 + and 3.4.2, use the term bandwidth measurements, to refer to what + here is called Bandwidth List. + A Bandwidth List file contains information on relays' bandwidth + capacities and is produced by bandwidth generators, previously known + as bandwidth scanners. + +1.4. Format Versions + + 1.0.0 - The legacy fallback Bandwidth List format + + 1.1.0 - Adds KeyValue Lines to the Header List section, add KeyValues + to RelayLines and format versions. + + All Tor versions can consume format version 1.0.0. + All Tor versions can consume format version 1.1.0, + but they warn on additional header Lines. + [TODO: this might be fixed, and if it is fixed should be said which + version of Tor] + +2. Format details + + The Bandwidth List MUST contain the following sections: + - Header List (exactly once) + - Relays' Bandwidth List (zero or more times) + If it does not contain these sections, parsers SHOULD ignore the file. + +2.1. Definitions + + The following nonterminals are defined in Tor directory protocol + sections 1.2., 2.1.1., 2.1.3.: + + Int + SP (space) + NL (newline) + Keyword + ArgumentChar + nickname + hexdigest (a '$', followed by 40 hexadecimal characters + ([A-Fa-f0-9])) + + Nonterminal defined section 2 of version-spec.txt [4]: + + version_number + + We define the following nonterminals: + + Line ::= ArgumentChar* NL + RelayLine ::= KeyValue (SP KeyValue)* NL + KeyValue ::= Keyword "=" Value + Value ::= ArgumentCharValue+ + ArgumentCharValue ::= any printing ASCII character except NL and SP. + Terminator ::= "=====" + Timestamp ::= Int + Bandwidth ::= Int + MasterKey ::= a base64-encoded Ed25519 public key, with + padding characters omitted. + DateTime ::= "YYYY-MM-DDTHH:MM:SS", as in ISO 8601 + + Note that key_value and value are defined in Tor directory protocol + with different formats to KeyValue and Value here. + + All Lines in the file MUST be 510 characters or less, to allow for the + trailing newline and NULL characters. + The previous limit was 254 characters in Tor 0.2.6.2-alpha and + earlier. + The parser MAY ignore longer Lines. + [TODO: Change this restriction in 1.1.0 or later] + +2.2. Header List format + +Some header Lines MUST appear in specific positions, as documented +below. +All other Lines can appear in any order. +If a parser does not recognize any extra material in a header Line, +the Line MUST be ignored. +If a header Line does not conform to this format, the Line SHOULD be +ignored by parsers. + +It consists of: + + Timestamp NL + + [At start, exactly once.] + + The Unix Epoch time in seconds when the file was created. + It does not follow the KeyValue format for backwards + compatibility with version 1.0.0. + + "version=" version_number NL + + [In second position, zero or one time.] + + The specification document format version. + It uses semantic versioning [5]. + + This Line has been added in version 1.1.0 of this specification. + + Version 1.0.0 documents do not contain this Line, and the + version_number is considered to be "1.0.0". + + "software=" Value NL + + [Zero or one time.] + + The name of the software that created the document. + + This Line has been added in version 1.1.0 of this specification. + + Version 1.0.0 documents do not contain this Line, and the software + is considered to be "torflow". + + "software_version=" Value NL + + [Zero or one time.] + + The version of the software that created the document. + The version may be a version_number, a git commit, or some other + version scheme. + + This Line has been added in version 1.1.0 of this specification. + + "generator_started=" DateTime NL + + [Zero or one time.] + + The date and time timestamp in ISO 8601 format and UTC time zone + when the generator started. + + This Line has been added in version 1.1.0 of this specification. + + "earliest_bandwidth=" DateTime NL + + [Zero or one time.] + + The date and time timestamp in ISO 8601 format and UTC time zone + when the first relay bandwidth was obtained. + + This Line has been added in version 1.1.0 of this specification. + + KeyValue NL + + [Zero or more times.] + + There MUST NOT be multiple KeyValue header Lines with the same key. + If there are, the parser SHOULD choose an arbitrary Line. + + If a parser does not recognize a Keyword in a KeyValue Line, it + MUST be ignored. + + Future format versions may include additional KeyValue header Lines. + Additional header Lines will be accompanied by a minor version + increment. + + Implementations MAY add additional header Lines as needed. This + specification SHOULD be updated to avoid conflicting meanings for + the same header keys. + + Parsers MUST NOT rely on the order of these additional Lines. + + Additional header Lines MUST NOT use any keywords specified in the + relay measurements format. + If there are, the parser MAY ignore conflicting keywords. + + Terminator NL + + [Zero or one time.] + + The Header List section ends with this Terminator. + + In version 1.0.0, Header List ends when the first relay bandwidth + is found conforming to the next section. + Implementations of version 1.1.0 SHOULD include this Line. + +2.3. Relays' Bandwidth List format + +It consists of zero or more RelayLines with the relays' bandwidth +in arbitrary order. + +There MUST NOT be multiple KeyValue pairs with the same key in the same +RelayLine. +If there are, the parser SHOULD choose an arbitrary Value. + +There MUST NOT be multiple RelayLine per relay identity (node_id or +master_key_ed25519). +If there are, parsers SHOULD issue a warning and MAY choose an arbitrary +value or ignore both values. + +If a parser does not recognize any extra material in a RelayLine, +the extra material MUST be ignored. + +Each RelayLine MUST include the following KeyValue pairs: +In version 1.0.0, node_id MUST NOT be at the end of the Line. +In version 1.1.0, the KeyValue can be in any arbitrary order. +[TODO: list of Tor version that support it, when it's done] + + "node_id=" hexdigest + + [Exactly once.] + + The fingerprint for the relay's RSA identity key. + + "master_key_ed25519=" MasterKey + + [Zero or one time.] + + The relays's master Ed25519 key, base64 encoded, + without trailing "="s, to avoid ambiguity with KeyValue "=" + character. + + Implementations of version 1.1.0 SHOULD include both node_id and + master_key_ed25519. + Parsers SHOULD accept Lines that contain at least one of them. + + "bw=" Bandwidth + + [Exactly once.] + + The measured bandwidth of this relay. + + Tor accepts zero bandwidths, but they trigger bugs in older Tor + implementations. Therefore, implementations SHOULD NOT produce zero + bandwidths. Instead, they SHOULD use one as their minimum bandwidth. + If there are zero bandwidths, the parser MAY ignore them. + + Multiple measurements can be aggregated using an averaging scheme, + such as a mean, median, or decaying average. + + Torflow scales bandwidths to kilobytes per second. Other + implementations SHOULD use kilobytes per second for their initial + bandwidth scaling. + + If different implementations or configurations are used in votes for + the same network, their measurements MAY need further scaling. See + Appendix B for information about scaling, and one possible scaling + method. + + KeyValue + + [Zero or more times.] + + Future format versions may include additional KeyValue pairs on a + RelayLine. + Additional KeyValue pairs will be accompanied by a minor version + increment. + + Implementations MAY add additional relay KeyValue pairs as needed. + This specification SHOULD be updated to avoid conflicting meanings + for the same Keywords. + + Parsers MUST NOT rely on the order of these additional KeyValue + pairs. + + Additional KeyValue pairs MUST NOT use any keywords specified in the + header format. + If there are, the parser MAY ignore conflicting keywords. + +2.4. Implementation notes + +KeyValue pairs in RelayLines that current implementations generate. + +2.4.1. Simple Bandwidth Scanner + +Every RelayLine in sbws version 0.1.0 consists of: + + "node_id=" hexdigest SP + + As above. + + "bw=" Bandwidth SP + + As above. + + "nick=" nickname SP + + [Exactly once.] + + The relay nickname. + + "rtt=" Int SP + + [Exactly once.] + + The Round Trip Time in milliseconds to obtain 1 byte of data. + + "time=" DateTime NL + + [Exactly once.] + + The date and time timestamp in ISO 8601 format and UTC time zone + when the last bandwidth was obtained. + +2.4.2. Torflow + +Torflow RelayLines include node_id and bw, and other KeyValue pairs [2]. + +References: + +1. https://gitweb.torproject.org/torflow.git +2. https://gitweb.torproject.org/torflow.git/tree/NetworkScanners/BwAuthority/… +3. https://gitweb.torproject.org/torspec.git/tree/dir-spec.txt +4. https://gitweb.torproject.org/torspec.git/tree/version-spec.txt +5. https://semver.org/ + +A. Sample data + +The following has not been obtained from any real measurement. + +A.1. Generated by Torflow + +This an example version 1.0.0 document: + +1523911758 +node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 bw=760 nick=Test measured_at=1523911725 updated_at=1523911725 pid_error=4.11374090719 pid_error_sum=4.11374090719 pid_bw=57136645 pid_delta=2.12168374577 circ_fail=0.2 scanner=/filepath +node_id=$96C15995F30895689291F455587BD94CA427B6FC bw=189 nick=Test2 measured_at=1523911623 updated_at=1523911623 pid_error=3.96703337994 pid_error_sum=3.96703337994 pid_bw=47422125 pid_delta=2.65469736988 circ_fail=0.0 scanner=/filepath + +A.2. Generated by sbws version 0.1.X +[TODO: this needs to be implemented when this spec is finished] + +1523911758 +version=1.1.0 +software=sbws +software_version=0.1.0 +generator_started=2018-05-08T16:13:25 +earliest_bandwidth=2018-05-08T16:13:26 +==== +node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 master_key_ed25519=YaqV4vbvPYKucElk297eVdNArDz9HtIwUoIeo0+cVIpQ bw=760 nick=Test rtt=380 time=2018-05-08T16:13:26 +node_id=$96C15995F30895689291F455587BD94CA427B6FC master_key_ed25519=a6a+dZadrQBtfSbmQkP7j2ardCmLnm5NJ4ZzkvDxbo0I bw=189 nick=Test2 rtt=378 time=2018-05-08T16:13:36 + +B. Scaling bandwidths + +B.1. Scaling requirements + +Tor accepts zero bandwidths, but they trigger bugs in older Tor +implementations. Therefore, scaling methods SHOULD perform the +following checks: + * If the total bandwidth is zero, all relays should be given equal + bandwidths. + * If the scaled bandwidth is zero, it should be rounded up to one. + +Initial experiments indicate that scaling may not be needed for +torflow and sbws, because their measured bandwidths are similar +enough already. + +B.2. A linear scaling method + +If scaling is required, here is a simple linear bandwith scaling +method, which ensures that all bandwidth votes contain approximately +the same total bandwidth: + +1. Calculate the relay quota by dividing the total measured bandwidth + in all votes, by the number of relays with measured bandwidth + votes. In the public tor network, this is approximately 7500 as of + April 2018. The quota should be a consensus parameter, so it can be + adjusted for all generators on the network. + +2. Calculate a vote quota by multiplying the relay quota by the number + of relays this bandwidth authority has measured + bandwidths for. + +3. Calculate a scaling factor by dividing the vote quota by the + total unscaled measured bandwidth in this bandwidth + authority's upcoming vote. + +4. Multiply each unscaled measured bandwidth by the scaling + factor. + +Now, the total scaled bandwidth in the upcoming vote is +approximately equal to the quota. + +B.3. Quota changes + +If all generators are using scaling, the quota can be gradually +reduced or increased as needed. Smaller quotas decrease the size +of uncompressed consensuses, and may decrease the size of +consensus diffs and compressed consensuses. But if the relay +quota is too small, some relays may be over- or under-weighted.

1 0

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits May 2018