February 2018 - tor-commits - lists.torproject.org

[tor/release-0.3.3] backport to make channel_is_client() accurate
by nickm＠torproject.org 16 Feb '18

16 Feb '18

commit 8d5dcdbda2160b6375cb479827d8cc2f16e06777 Author: Roger Dingledine <arma(a)torproject.org> Date: Tue Jan 30 18:05:39 2018 -0500 backport to make channel_is_client() accurate This commit takes a piece of commit af8cadf3a9 and a piece of commit 46fe353f25, with the goal of making channel_is_client() be based on what sort of connection handshake the other side used, rather than seeing whether the other side ever sent a create_fast cell to us. --- changes/bug24898-029 | 7 +++++++ src/or/channeltls.c | 4 ++++ src/or/connection_or.c | 6 ++++++ 3 files changed, 17 insertions(+) diff --git a/changes/bug24898-029 b/changes/bug24898-029 new file mode 100644 index 000000000..f56dee5c7 --- /dev/null +++ b/changes/bug24898-029 @@ -0,0 +1,7 @@ + o Minor bugfixes: + - Make the internal channel_is_client() function look at what sort + of connection handshake the other side used, rather than whether + the other side ever sent a create_fast cell to us. Backports part + of the fixes from bug 24898. + + diff --git a/src/or/channeltls.c b/src/or/channeltls.c index 09cca95b6..31641c3db 100644 --- a/src/or/channeltls.c +++ b/src/or/channeltls.c @@ -1640,6 +1640,10 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) tor_assert(tor_digest_is_zero( (const char*)(chan->conn->handshake_state-> authenticated_peer_id))); + /* If the client never authenticated, it's a tor client or bridge + * relay, and we must not use it for EXTEND requests (nor could we, as + * there are no authenticated peer IDs) */ + channel_mark_client(TLS_CHAN_TO_BASE(chan)); channel_set_circid_type(TLS_CHAN_TO_BASE(chan), NULL, chan->conn->link_proto < MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS); diff --git a/src/or/connection_or.c b/src/or/connection_or.c index dadfdc438..8beedcae7 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -1880,6 +1880,12 @@ connection_or_set_state_open(or_connection_t *conn) connection_or_change_state(conn, OR_CONN_STATE_OPEN); control_event_or_conn_status(conn, OR_CONN_EVENT_CONNECTED, 0); + /* Link protocol 3 appeared in Tor 0.2.3.6-alpha, so any connection + * that uses an earlier link protocol should not be treated as a relay. */ + if (conn->link_proto < 3) { + channel_mark_client(TLS_CHAN_TO_BASE(conn->chan)); + } + or_handshake_state_free(conn->handshake_state); conn->handshake_state = NULL; connection_start_reading(TO_CONN(conn));

1 0

[tor/release-0.3.3] Merge branch 'maint-0.3.2' into maint-0.3.3
by nickm＠torproject.org 16 Feb '18

16 Feb '18

commit 9f1899fdfd611861bd1a254a6ba6df4ddf05f5da Merge: 8970058af c67adddac Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Feb 16 08:49:08 2018 -0500 Merge branch 'maint-0.3.2' into maint-0.3.3 changes/bug24898-029 | 6 ------ 1 file changed, 6 deletions(-)

1 0

[tor/release-0.3.3] Merge branch 'maint-0.3.2' into maint-0.3.3
by nickm＠torproject.org 16 Feb '18

16 Feb '18

commit 8970058af6822b14bc06e7a98230b8b13c0084a9 Merge: d662d4470 c8b087d90 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Feb 16 08:47:41 2018 -0500 Merge branch 'maint-0.3.2' into maint-0.3.3 changes/bug24898-029 | 6 ++++++ 1 file changed, 6 insertions(+)

1 0

[tor/release-0.3.2] Merge branch 'maint-0.2.9' into maint-0.3.1
by nickm＠torproject.org 16 Feb '18

16 Feb '18

commit 3930ffdf63425c344e14c45cff6780108cfa038b Merge: d21e5cfc2 2b99350ca Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Feb 16 08:47:12 2018 -0500 Merge branch 'maint-0.2.9' into maint-0.3.1 "ours" merge to avoid conflicts with the cherry-picked fix for 24898.

1 0

[tor/release-0.3.2] Remove changes file for 24898-029 backports in maint-0.3.2
by nickm＠torproject.org 16 Feb '18

16 Feb '18

commit c67adddacb56f90d77786820f29ca2b4b9f91cae Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Feb 16 08:48:46 2018 -0500 Remove changes file for 24898-029 backports in maint-0.3.2 These patches were already merged, and so don't need a changes file in these branches. --- changes/bug24898-029 | 6 ------ 1 file changed, 6 deletions(-) diff --git a/changes/bug24898-029 b/changes/bug24898-029 deleted file mode 100644 index b33f09384..000000000 --- a/changes/bug24898-029 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfixes (relay): - - Make the internal channel_is_client() function look at what sort - of connection handshake the other side used, rather than whether - the other side ever sent a create_fast cell to us. Backports part - of the fixes from bugs 22805 and 24898. -

1 0

[tor/release-0.3.2] Merge branch 'maint-0.3.2' into release-0.3.2
by nickm＠torproject.org 16 Feb '18

16 Feb '18

commit 7e664bf74ff4c2374d59d4a663799454bb86d10a Merge: 4bfbce9de c8b087d90 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Feb 16 08:47:41 2018 -0500 Merge branch 'maint-0.3.2' into release-0.3.2 changes/bug24898-029 | 6 ++++++ 1 file changed, 6 insertions(+)

1 0

[tor/master] Remove the return value of node_get_prim_orport() and node_get_prim_dirport()
by nickm＠torproject.org 16 Feb '18

16 Feb '18

commit c2fa743806accfbb119c925fbd8466b0f48b7ef7 Author: Neel Chauhan <neel(a)neelc.org> Date: Tue Feb 6 08:13:55 2018 -0500 Remove the return value of node_get_prim_orport() and node_get_prim_dirport() --- changes/ticket23873 | 6 ++++++ src/or/circuitbuild.c | 6 ++++-- src/or/nodelist.c | 26 +++++++++++--------------- src/or/nodelist.h | 4 ++-- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/changes/ticket23873 b/changes/ticket23873 new file mode 100644 index 000000000..ffe65e5ed --- /dev/null +++ b/changes/ticket23873 @@ -0,0 +1,6 @@ + o Code simplification and refactoring: + - We remove the return value of node_get_prim_orport() and + node_get_prim_dirport(), and introduce node_get_prim_orport() + in node_ipv6_or_preferred() and node_ipv6_dir_preferred() in + order to check for a null address. Closes ticket 23873. Patch + by Neel Chauhan. diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index 6d6fd033b..b3b543348 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -2866,8 +2866,10 @@ extend_info_from_node(const node_t *node, int for_direct_connect) valid_addr = fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0, &ap); - else - valid_addr = !node_get_prim_orport(node, &ap); + else { + node_get_prim_orport(node, &ap); + valid_addr = tor_addr_port_is_valid_ap(&ap, 0); + } if (valid_addr) log_debug(LD_CIRC, "using %s for %s", diff --git a/src/or/nodelist.c b/src/or/nodelist.c index 391b31d68..40b56f3d1 100644 --- a/src/or/nodelist.c +++ b/src/or/nodelist.c @@ -1456,9 +1456,11 @@ node_ipv6_or_preferred(const node_t *node) /* XX/teor - node->ipv6_preferred is set from * fascist_firewall_prefer_ipv6_orport() each time the consensus is loaded. */ + node_get_prim_orport(node, &ipv4_addr); if (!fascist_firewall_use_ipv6(options)) { return 0; - } else if (node->ipv6_preferred || node_get_prim_orport(node, &ipv4_addr)) { + } else if (node->ipv6_preferred || + !tor_addr_port_is_valid_ap(&ipv4_addr, 0)) { return node_has_ipv6_orport(node); } return 0; @@ -1469,14 +1471,12 @@ node_ipv6_or_preferred(const node_t *node) if (r && tor_addr_port_is_valid_ipv4h((r)->addr, (r)->port_field, 0)) { \ tor_addr_from_ipv4h(&(ap_out)->addr, (r)->addr); \ (ap_out)->port = (r)->port_field; \ - return 0; \ } \ STMT_END -/** Copy the primary (IPv4) OR port (IP address and TCP port) for - * node into *ap_out. Return 0 if a valid address and - * port was copied, else return non-zero.*/ -int +/** Copy the primary (IPv4) OR port (IP address and TCP port) for node + * into *ap_out. */ +void node_get_prim_orport(const node_t *node, tor_addr_port_t *ap_out) { node_assert_ok(node); @@ -1493,8 +1493,6 @@ node_get_prim_orport(const node_t *node, tor_addr_port_t *ap_out) RETURN_IPV4_AP(node->ri, or_port, ap_out); RETURN_IPV4_AP(node->rs, or_port, ap_out); /* Microdescriptors only have an IPv6 address */ - - return -1; } /** Copy the preferred OR port (IP address and TCP port) for @@ -1566,19 +1564,19 @@ node_ipv6_dir_preferred(const node_t *node) * so we can't use it to determine DirPort IPv6 preference. * This means that bridge clients will use IPv4 DirPorts by default. */ + node_get_prim_dirport(node, &ipv4_addr); if (!fascist_firewall_use_ipv6(options)) { return 0; - } else if (node_get_prim_dirport(node, &ipv4_addr) + } else if (!tor_addr_port_is_valid_ap(&ipv4_addr, 0) || fascist_firewall_prefer_ipv6_dirport(get_options())) { return node_has_ipv6_dirport(node); } return 0; } -/** Copy the primary (IPv4) Dir port (IP address and TCP port) for - * node into *ap_out. Return 0 if a valid address and - * port was copied, else return non-zero.*/ -int +/** Copy the primary (IPv4) Dir port (IP address and TCP port) for node + * into *ap_out. */ +void node_get_prim_dirport(const node_t *node, tor_addr_port_t *ap_out) { node_assert_ok(node); @@ -1590,8 +1588,6 @@ node_get_prim_dirport(const node_t *node, tor_addr_port_t *ap_out) RETURN_IPV4_AP(node->ri, dir_port, ap_out); RETURN_IPV4_AP(node->rs, dir_port, ap_out); /* Microdescriptors only have an IPv6 address */ - - return -1; } #undef RETURN_IPV4_AP diff --git a/src/or/nodelist.h b/src/or/nodelist.h index dc20eaf0a..ff26c8cc7 100644 --- a/src/or/nodelist.h +++ b/src/or/nodelist.h @@ -77,11 +77,11 @@ int node_has_ipv6_dirport(const node_t *node); /* Deprecated - use node_ipv6_or_preferred or node_ipv6_dir_preferred */ #define node_ipv6_preferred(node) node_ipv6_or_preferred(node) int node_ipv6_or_preferred(const node_t *node); -int node_get_prim_orport(const node_t *node, tor_addr_port_t *ap_out); +void node_get_prim_orport(const node_t *node, tor_addr_port_t *ap_out); void node_get_pref_orport(const node_t *node, tor_addr_port_t *ap_out); void node_get_pref_ipv6_orport(const node_t *node, tor_addr_port_t *ap_out); int node_ipv6_dir_preferred(const node_t *node); -int node_get_prim_dirport(const node_t *node, tor_addr_port_t *ap_out); +void node_get_prim_dirport(const node_t *node, tor_addr_port_t *ap_out); void node_get_pref_dirport(const node_t *node, tor_addr_port_t *ap_out); void node_get_pref_ipv6_dirport(const node_t *node, tor_addr_port_t *ap_out); int node_has_curve25519_onion_key(const node_t *node);

1 0

[tor/master] Compilation workaround for windows, which lacks O_SYNC
by nickm＠torproject.org 16 Feb '18

16 Feb '18

commit 200fc8c640341dde657bd5162b9468cbd07d2938 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Feb 16 08:16:12 2018 -0500 Compilation workaround for windows, which lacks O_SYNC Bug not in any released Tor. --- src/or/keypin.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/or/keypin.c b/src/or/keypin.c index 66525cbf9..52bdff816 100644 --- a/src/or/keypin.c +++ b/src/or/keypin.c @@ -289,6 +289,9 @@ static int keypin_journal_fd = -1; int keypin_open_journal(const char *fname) { +#ifndef O_SYNC +#define O_SYNC 0 +#endif int fd = tor_open_cloexec(fname, O_WRONLY|O_CREAT|O_BINARY|O_SYNC, 0600); if (fd < 0) goto err;

1 0

[tor/master] tor_zstd_format_version shouldn't be built when !HAVE_ZSTD
by nickm＠torproject.org 16 Feb '18

16 Feb '18

commit 5a9ada342fb17a7c3ba69efbdd548ec278b0e070 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Feb 16 08:06:01 2018 -0500 tor_zstd_format_version shouldn't be built when !HAVE_ZSTD Fixes bug 25276; bugfix not in any released Tor. --- src/common/compress_zstd.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/common/compress_zstd.c b/src/common/compress_zstd.c index 0c3e7104f..4e26c724e 100644 --- a/src/common/compress_zstd.c +++ b/src/common/compress_zstd.c @@ -58,6 +58,7 @@ tor_zstd_method_supported(void) #endif } +#ifdef HAVE_ZSTD /** Format a zstd version number as a string in buf. */ static void tor_zstd_format_version(char *buf, size_t buflen, unsigned version_number) @@ -68,6 +69,7 @@ tor_zstd_format_version(char *buf, size_t buflen, unsigned version_number) version_number / 100 % 100, version_number % 100); } +#endif #define VERSION_STR_MAX_LEN 16 /* more than enough space for 99.99.99 */

1 0

[metrics-lib/master] Add log line interfaces and access methods.
by karsten＠torproject.org 16 Feb '18

16 Feb '18

commit 59689a9fa4c162378f347902eb68e4c21ccf0043 Author: iwakeh <iwakeh(a)torproject.org> Date: Tue Feb 6 14:59:05 2018 +0000 Add log line interfaces and access methods. For both the general LogDescriptor and extension WebServerAccessLog. Include some new tests. --- .../org/torproject/descriptor/LogDescriptor.java | 12 ++++++++++ .../torproject/descriptor/{log => }/Method.java | 5 ++-- .../torproject/descriptor/WebServerAccessLog.java | 28 ++++++++++++++++++++++ .../descriptor/log/WebServerAccessLogImpl.java | 20 ++++++++++++++++ .../descriptor/log/WebServerAccessLogLine.java | 15 ++++++++++-- .../descriptor/log/LogDescriptorTest.java | 15 +++++++----- 6 files changed, 85 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/torproject/descriptor/LogDescriptor.java b/src/main/java/org/torproject/descriptor/LogDescriptor.java index ff02cae..6a6bf84 100644 --- a/src/main/java/org/torproject/descriptor/LogDescriptor.java +++ b/src/main/java/org/torproject/descriptor/LogDescriptor.java @@ -43,5 +43,17 @@ public interface LogDescriptor extends Descriptor { @Override public List<String> getUnrecognizedLines(); + /** + * Returns a list of all parseable log lines. + * Might require a lot of memory depending on log size. + */ + public List<? extends Line> logLines() throws DescriptorParseException; + + public interface Line { + + /** Returns a log line string. */ + public String toLogString(); + + } } diff --git a/src/main/java/org/torproject/descriptor/log/Method.java b/src/main/java/org/torproject/descriptor/Method.java similarity index 50% rename from src/main/java/org/torproject/descriptor/log/Method.java rename to src/main/java/org/torproject/descriptor/Method.java index c29d495..9135fe2 100644 --- a/src/main/java/org/torproject/descriptor/log/Method.java +++ b/src/main/java/org/torproject/descriptor/Method.java @@ -1,8 +1,9 @@ /* Copyright 2018 The Tor Project * See LICENSE for licensing information */ -package org.torproject.descriptor.log; +package org.torproject.descriptor; -public enum Method { +/** Enum for web server access log methods. */ +public enum Method { GET, HEAD, POST; } diff --git a/src/main/java/org/torproject/descriptor/WebServerAccessLog.java b/src/main/java/org/torproject/descriptor/WebServerAccessLog.java index b94bc30..b4f1940 100644 --- a/src/main/java/org/torproject/descriptor/WebServerAccessLog.java +++ b/src/main/java/org/torproject/descriptor/WebServerAccessLog.java @@ -5,6 +5,7 @@ package org.torproject.descriptor; import java.time.LocalDate; import java.util.List; +import java.util.Optional; /** * Contains a sanitized web server access log file from a {@code torproject.org} @@ -61,5 +62,32 @@ public interface WebServerAccessLog extends LogDescriptor { @Override public List<String> getUnrecognizedLines(); + public interface Line extends LogDescriptor.Line { + + /** Returns the IP address of the requesting host. */ + public String getIp(); + + /** Returns the HTTP method, e.g., GET. */ + public Method getMethod(); + + /** Returns the protocol and version, e.g., HTTP/1.1. */ + public String getProtocol(); + + /** Returns the requested resource. */ + public String getRequest(); + + /** Returns the size of the response in bytes, if available. */ + public Optional<Integer> getSize(); + + /** Returns the final status code, e.g., 200. */ + public int getResponse(); + + /** Returns the date when the request was received. */ + public LocalDate getDate(); + + /** True, if this is a valid web server access log line. */ + public boolean isValid(); + } + } diff --git a/src/main/java/org/torproject/descriptor/log/WebServerAccessLogImpl.java b/src/main/java/org/torproject/descriptor/log/WebServerAccessLogImpl.java index f02b1d7..7b56528 100644 --- a/src/main/java/org/torproject/descriptor/log/WebServerAccessLogImpl.java +++ b/src/main/java/org/torproject/descriptor/log/WebServerAccessLogImpl.java @@ -10,12 +10,17 @@ import org.torproject.descriptor.internal.FileType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; import java.io.File; +import java.io.InputStreamReader; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.Collection; +import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; /** * Implementation of web server access log descriptors. @@ -126,5 +131,20 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl return this.logDate; } + /** Returns a list of all valid log lines. */ + @Override + public List<WebServerAccessLog.Line> logLines() + throws DescriptorParseException { + try (BufferedReader br + = new BufferedReader(new InputStreamReader(new ByteArrayInputStream( + this.getRawDescriptorBytes())))) { + return br.lines().map(line + -> (WebServerAccessLog.Line) WebServerAccessLogLine.makeLine(line)) + .filter(line -> line.isValid()).collect(Collectors.toList()); + } catch (Exception ex) { + throw new DescriptorParseException("Cannot retrieve log lines.", ex); + } + } + } diff --git a/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java b/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java index c9d73cc..8a17230 100644 --- a/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java +++ b/src/main/java/org/torproject/descriptor/log/WebServerAccessLogLine.java @@ -3,6 +3,9 @@ package org.torproject.descriptor.log; +import org.torproject.descriptor.Method; +import org.torproject.descriptor.WebServerAccessLog; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -18,7 +21,7 @@ import java.util.Optional; import java.util.regex.Matcher; import java.util.regex.Pattern; -public class WebServerAccessLogLine { +public class WebServerAccessLogLine implements WebServerAccessLog.Line { private static final Logger log = LoggerFactory .getLogger(WebServerAccessLogLine.class); @@ -54,6 +57,7 @@ public class WebServerAccessLogLine { private String protocol; /** Returns a log line string. Possibly empty. */ + @Override public String toLogString() { if (!this.valid) { return ""; @@ -74,7 +78,7 @@ public class WebServerAccessLogLine { return this.date.format(DateTimeFormatter.ofPattern(DATE_PATTERN)); } - /** Returns a string containing the ip. */ + @Override public String getIp() { return this.ip; } @@ -84,22 +88,27 @@ public class WebServerAccessLogLine { this.ip = fromMap(ip, ipMap); } + @Override public Method getMethod() { return this.method; } + @Override public String getProtocol() { return this.protocol; } + @Override public String getRequest() { return this.request; } + @Override public Optional<Integer> getSize() { return this.size < 0 ? Optional.empty() : Optional.of(this.size); } + @Override public int getResponse() { return this.response; } @@ -109,10 +118,12 @@ public class WebServerAccessLogLine { this.request = fromMap(request, requestMap); } + @Override public LocalDate getDate() { return this.date; } + @Override public boolean isValid() { return this.valid; } diff --git a/src/test/java/org/torproject/descriptor/log/LogDescriptorTest.java b/src/test/java/org/torproject/descriptor/log/LogDescriptorTest.java index b12cfc0..a871791 100644 --- a/src/test/java/org/torproject/descriptor/log/LogDescriptorTest.java +++ b/src/test/java/org/torproject/descriptor/log/LogDescriptorTest.java @@ -1,4 +1,3 @@ - /* Copyright 2017--2018 The Tor Project * See LICENSE for licensing information */ @@ -51,6 +50,7 @@ public class LogDescriptorTest { protected String[] pan; protected Class<LogDescriptor> type; protected boolean isDecompressionTest; + protected int lineCount; /** All types of data that can be encountered during sync. */ @Parameters @@ -60,29 +60,30 @@ public class LogDescriptorTest { "metrics.torproject.org_meronense.torproject.org_access.log" + "_20170530.gz", "metrics.torproject.org", "20170530", "gz"}, - WebServerAccessLog.class}, + WebServerAccessLog.class, 24}, {Boolean.FALSE, 1878, new String[]{"meronense.torproject.org", "xy.host.org_meronense.torproject.org_access.log_20170530.log", "metrics.torproject.org", "20170530", "xz"}, - WebServerAccessLog.class}, + WebServerAccessLog.class, 24}, {Boolean.TRUE, 70730, new String[]{"archeotrichon.torproject.org", "archive.torproject.org_archeotrichon.torproject.org_access.log_" + "20151007.xz", "archive.torproject.org", "20151007", "xz"}, - WebServerAccessLog.class}, + WebServerAccessLog.class, 655}, {Boolean.TRUE, 0, new String[]{"dummy.host.net", "nix.server.org_dummy.host.net_access.log_20111111.bz2", "nix.server.org", "20111111", "bz2"}, - WebServerAccessLog.class}}); + WebServerAccessLog.class, 0}}); } /** This constructor receives the above defined data for each run. */ public LogDescriptorTest(boolean decompression, int size, String[] pan, - Class<LogDescriptor> type) { + Class<LogDescriptor> type, int lineCount) { this.pan = pan; this.size = size; this.type = type; this.isDecompressionTest = decompression; + this.lineCount = lineCount; } /** Prepares the temporary folder and writes files to it for this test. */ @@ -129,6 +130,8 @@ public class LogDescriptorTest { InternalLogDescriptor ld = (InternalLogDescriptor) descs.get(0); assertEquals("Wrong compression type string. " + dataUsed(), pan[4], ld.getCompressionType()); + List<? extends LogDescriptor.Line> lines = ld.logLines(); + assertEquals(this.lineCount, lines.size()); } private String dataUsed() {

1 0