commit b8fb876145cda3c14d335a3fc88b5e422a926150 Author: David Fifield david@bamsoftware.com Date: Tue Jan 22 23:38:26 2019 -0700
Integrate uTLS as an option for TLS camouflage.
This adapts a technique that Yawning used in obfs4proxy for meek_lite: https://gitlab.com/yawning/obfs4/commit/4d453dab2120082b00bf6e63ab4aaeeda6b8... https://lists.torproject.org/pipermail/tor-dev/2019-January/013633.html
It's activated by the new utls= SOCKS arg or --utls command line option. The argument is the name of a uTLS ClientHelloID; e.g., HelloChrome_Auto. We omit HelloCustom (not useful externally), HelloGolang (just don't use utls), and HelloRandomized (may negotiate HTTP/1.1 or HTTP/2 at different times, which is incompatible with the way the integration works).
When using HTTP/1, we copy default timeouts etc. from the Go net/http.DefaultTransport. Unfortunately I don't know a way to do the same for HTTP/2. Configuring an http.Transport and then calling http2.ConfigureTransport on it doesn't work; it leads to the same problem of an HTTP/1 client speaking to an HTTP/2 server.
We recognize utls=none and utls=HelloGolang as an alias for omitting utls=. This is for compatibility with obfs4proxy meek_lite. https://bugs.torproject.org/29077#comment:13 --- meek-client/meek-client.go | 30 +++++- meek-client/utls.go | 260 +++++++++++++++++++++++++++++++++++++++++++++ meek-client/utls_test.go | 232 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 518 insertions(+), 4 deletions(-)
diff --git a/meek-client/meek-client.go b/meek-client/meek-client.go index 73ca214..74ef2c5 100644 --- a/meek-client/meek-client.go +++ b/meek-client/meek-client.go @@ -79,9 +79,9 @@ const ( helperWriteTimeout = 2 * time.Second )
-// We use this RoundTripper to make all our requests when --helper is not -// in effect. We use the defaults, except we take control of the Proxy setting -// (notably, disabling the default ProxyFromEnvironment). +// We use this RoundTripper to make all our requests when neither --helper nor +// utls is in effect. We use the defaults, except we take control of the Proxy +// setting (notably, disabling the default ProxyFromEnvironment). var httpRoundTripper *http.Transport = http.DefaultTransport.(*http.Transport)
// We use this RoundTripper when --helper is in effect. @@ -96,6 +96,7 @@ var options struct { Front string ProxyURL *url.URL UseHelper bool + UTLSName string }
// RequestInfo encapsulates all the configuration used for a request–response @@ -307,9 +308,29 @@ func handler(conn *pt.SocksConn) error { info.URL.Host = front }
- info.RoundTripper = httpRoundTripper + // First check utls= SOCKS arg, then --utls option. + utlsName, utlsOK := conn.Req.Args.Get("utls") + if utlsOK { + } else if options.UTLSName != "" { + utlsName = options.UTLSName + utlsOK = true + } + + // First we check --helper: if it was specified, then we always use the + // helper, and utls is disallowed. Otherwise, we use utls if requested; + // or else fall back to native net/http. if options.UseHelper { + if utlsOK { + return fmt.Errorf("cannot use utls with --helper") + } info.RoundTripper = helperRoundTripper + } else if utlsOK { + info.RoundTripper, err = NewUTLSRoundTripper(utlsName, nil) + if err != nil { + return err + } + } else { + info.RoundTripper = httpRoundTripper }
return copyLoop(conn, &info) @@ -387,6 +408,7 @@ func main() { flag.StringVar(&logFilename, "log", "", "name of log file") flag.StringVar(&proxy, "proxy", "", "proxy URL") flag.StringVar(&options.URL, "url", "", "URL to request if no url= SOCKS arg") + flag.StringVar(&options.UTLSName, "utls", "", "uTLS Client Hello ID") flag.Parse()
ptInfo, err := pt.ClientSetup(nil) diff --git a/meek-client/utls.go b/meek-client/utls.go new file mode 100644 index 0000000..e8d0bc0 --- /dev/null +++ b/meek-client/utls.go @@ -0,0 +1,260 @@ +// Support code for TLS camouflage using uTLS. +// +// The goal is: provide an http.RoundTripper abstraction that retains the +// features of http.Transport (e.g., persistent connections and HTTP/2 support), +// while making TLS connections using uTLS in place of crypto/tls. The challenge +// is: while http.Transport provides a DialTLS hook, setting it to non-nil +// disables automatic HTTP/2 support in the client. Most of the uTLS +// fingerprints contain an ALPN extension containing "h2"; i.e., they declare +// support for HTTP/2. If the server also supports HTTP/2, then uTLS may +// negotiate an HTTP/2 connection without the http.Transport knowing it, which +// leads to an HTTP/1.1 client speaking to an HTTP/2 server, a protocol error. +// +// The code here uses an idea adapted from meek_lite in obfs4proxy: +// https://gitlab.com/yawning/obfs4/commit/4d453dab2120082b00bf6e63ab4aaeeda6b8... +// Instead of setting DialTLS on an http.Transport and exposing it directly, we +// expose a wrapper type, UTLSRoundTripper, that contains within it either an +// http.Transport or an http2.Transport. The first time a caller calls RoundTrip +// on the wrapper, we initiate a uTLS connection (bootstrapConn), then peek at +// the ALPN-negotiated protocol: if "h2", create an internal http2.Transport; +// otherwise, create an internal http.Transport. In either case, set DialTLS on +// the created Transport to a function that dials using uTLS. As a special case, +// the first time the DialTLS callback is called, it reuses bootstrapConn (the +// one made to peek at the ALPN), rather than make a new connection. +// +// Subsequent calls to RoundTripper on the wrapper just pass the requests though +// the previously created http.Transport or http2.Transport. We assume that in +// future RoundTrips, the ALPN-negotiated protocol will remain the same as it +// was in the initial RoundTrip. At this point it is the http.Transport or +// http2.Transport calling DialTLS, not us, so we can't dynamically swap the +// underlying transport based on the ALPN. +// +// https://bugs.torproject.org/29077 +// https://github.com/refraction-networking/utls/issues/16 +package main + +import ( + "crypto/tls" + "fmt" + "net" + "net/http" + "net/url" + "reflect" + "strings" + "sync" + + utls "github.com/refraction-networking/utls" + "golang.org/x/net/http2" +) + +// Copy the public fields (fields for which CanSet is true) from src to dst. +// src and dst must be pointers to the same type. We use this to make copies of +// httpRoundTripper. We cannot use struct assignment, because http.Transport +// contains private mutexes. The idea of using reflection to copy only the +// public fields comes from a post by Nick Craig-Wood: +// https://groups.google.com/d/msg/Golang-Nuts/SDiGYNVE8iY/89hRKTF4BAAJ +func copyPublicFields(dst, src interface{}) { + if reflect.TypeOf(dst) != reflect.TypeOf(src) { + panic("unequal types") + } + dstValue := reflect.ValueOf(dst).Elem() + srcValue := reflect.ValueOf(src).Elem() + for i := 0; i < dstValue.NumField(); i++ { + if dstValue.Field(i).CanSet() { + dstValue.Field(i).Set(srcValue.Field(i)) + } + } +} + +// Extract a host:port address from a URL, suitable for passing to net.Dial. +func addrForDial(url *url.URL) (string, error) { + host := url.Hostname() + // net/http would use golang.org/x/net/idna here, to convert a possible + // internationalized domain name to ASCII. + port := url.Port() + if port == "" { + // No port? Use the default for the scheme. + switch url.Scheme { + case "http": + port = "80" + case "https": + port = "443" + default: + return "", fmt.Errorf("unsupported URL scheme %q", url.Scheme) + } + } + return net.JoinHostPort(host, port), nil +} + +// Analogous to tls.Dial. Connect to the given address and initiate a TLS +// handshake using the given ClientHelloID, returning the resulting connection. +func dialUTLS(network, addr string, cfg *utls.Config, clientHelloID *utls.ClientHelloID) (*utls.UConn, error) { + if options.ProxyURL != nil { + return nil, fmt.Errorf("no proxy allowed with uTLS") + } + + conn, err := net.Dial(network, addr) + if err != nil { + return nil, err + } + uconn := utls.UClient(conn, cfg, *clientHelloID) + if cfg == nil || cfg.ServerName == "" { + serverName, _, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + uconn.SetSNI(serverName) + } + err = uconn.Handshake() + if err != nil { + return nil, err + } + return uconn, nil +} + +// A http.RoundTripper that uses uTLS (with a specified Client Hello ID) to make +// TLS connections. +// +// Can only be reused among servers which negotiate the same ALPN. +type UTLSRoundTripper struct { + sync.Mutex + + clientHelloID *utls.ClientHelloID + config *utls.Config + rt http.RoundTripper +} + +func (rt *UTLSRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + switch req.URL.Scheme { + case "http": + // If http, we don't invoke uTLS; just pass it to the global http.Transport. + return httpRoundTripper.RoundTrip(req) + case "https": + default: + return nil, fmt.Errorf("unsupported URL scheme %q", req.URL.Scheme) + } + + rt.Lock() + defer rt.Unlock() + + if rt.rt == nil { + // On the first call, make an http.Transport or http2.Transport + // as appropriate. + var err error + rt.rt, err = makeRoundTripper(req.URL, rt.clientHelloID, rt.config) + if err != nil { + return nil, err + } + } + // Forward the request to the internal http.Transport or http2.Transport. + return rt.rt.RoundTrip(req) +} + +func makeRoundTripper(url *url.URL, clientHelloID *utls.ClientHelloID, cfg *utls.Config) (http.RoundTripper, error) { + addr, err := addrForDial(url) + if err != nil { + return nil, err + } + + // Connect to the given address and initiate a TLS handshake using + // the given ClientHelloID. Return the resulting connection. + dial := func(network, addr string) (*utls.UConn, error) { + return dialUTLS(network, addr, cfg, clientHelloID) + } + + bootstrapConn, err := dial("tcp", addr) + if err != nil { + return nil, err + } + + // Peek at what protocol we negotiated. + protocol := bootstrapConn.ConnectionState().NegotiatedProtocol + + // Protects bootstrapConn. + var lock sync.Mutex + // This is the callback for future dials done by the internal + // http.Transport or http2.Transport. + dialTLS := func(network, addr string) (net.Conn, error) { + lock.Lock() + defer lock.Unlock() + + // On the first dial, reuse bootstrapConn. + if bootstrapConn != nil { + uconn := bootstrapConn + bootstrapConn = nil + return uconn, nil + } + + // Later dials make a new connection. + uconn, err := dial(network, addr) + if err != nil { + return nil, err + } + if uconn.ConnectionState().NegotiatedProtocol != protocol { + return nil, fmt.Errorf("unexpected switch from ALPN %q to %q", + protocol, uconn.ConnectionState().NegotiatedProtocol) + } + + return uconn, nil + } + + // Construct an http.Transport or http2.Transport depending on ALPN. + switch protocol { + case http2.NextProtoTLS: + // Unfortunately http2.Transport does not expose the same + // configuration options as http.Transport with regard to + // timeouts, etc., so we are at the mercy of the defaults. + // https://github.com/golang/go/issues/16581 + return &http2.Transport{ + DialTLS: func(network, addr string, _ *tls.Config) (net.Conn, error) { + // Ignore the *tls.Config parameter; use our + // static cfg instead. + return dialTLS(network, addr) + }, + }, nil + default: + // With http.Transport, copy important default fields from + // http.DefaultTransport, such as TLSHandshakeTimeout and + // IdleConnTimeout. + tr := &http.Transport{} + copyPublicFields(tr, httpRoundTripper) + tr.DialTLS = dialTLS + return tr, nil + } +} + +// When you update this map, also update the man page in doc/meek-client.1.txt. +var clientHelloIDMap = map[string]*utls.ClientHelloID{ + // No HelloCustom: not useful for external configuration. + // No HelloRandomized: doesn't negotiate consistent ALPN. + "none": nil, // special case: disable uTLS + "hellogolang": nil, // special case: disable uTLS + "hellorandomizedalpn": &utls.HelloRandomizedALPN, + "hellorandomizednoalpn": &utls.HelloRandomizedNoALPN, + "hellofirefox_auto": &utls.HelloFirefox_Auto, + "hellofirefox_55": &utls.HelloFirefox_55, + "hellofirefox_56": &utls.HelloFirefox_56, + "hellofirefox_63": &utls.HelloFirefox_63, + "hellochrome_auto": &utls.HelloChrome_Auto, + "hellochrome_58": &utls.HelloChrome_58, + "hellochrome_62": &utls.HelloChrome_62, + "hellochrome_70": &utls.HelloChrome_70, + "helloios_auto": &utls.HelloIOS_Auto, + "helloios_11_1": &utls.HelloIOS_11_1, +} + +func NewUTLSRoundTripper(name string, cfg *utls.Config) (http.RoundTripper, error) { + // Lookup is case-insensitive. + clientHelloID, ok := clientHelloIDMap[strings.ToLower(name)] + if !ok { + return nil, fmt.Errorf("no uTLS Client Hello ID named %q", name) + } + if clientHelloID == nil { + // Special case for "none" and HelloGolang. + return httpRoundTripper, nil + } + return &UTLSRoundTripper{ + clientHelloID: clientHelloID, + config: cfg, + }, nil +} diff --git a/meek-client/utls_test.go b/meek-client/utls_test.go new file mode 100644 index 0000000..2eb72af --- /dev/null +++ b/meek-client/utls_test.go @@ -0,0 +1,232 @@ +package main + +import ( + "bytes" + "io" + "net" + "net/http" + "net/url" + "testing" + + utls "github.com/refraction-networking/utls" +) + +func TestCopyPublicFieldsHTTPTransport(t *testing.T) { + src := http.DefaultTransport.(*http.Transport) + dst := &http.Transport{} + copyPublicFields(dst, src) + // Test various fields that we might care about a copy of http.Transport + // having. + if dst.DisableKeepAlives != src.DisableKeepAlives { + t.Errorf("mismatch on DisableKeepAlives") + } + if dst.DisableCompression != src.DisableCompression { + t.Errorf("mismatch on DisableCompression") + } + if dst.MaxIdleConns != src.MaxIdleConns { + t.Errorf("mismatch on MaxIdleConns") + } + if dst.MaxIdleConnsPerHost != src.MaxIdleConnsPerHost { + t.Errorf("mismatch on MaxIdleConnsPerHost") + } + if dst.MaxConnsPerHost != src.MaxConnsPerHost { + t.Errorf("mismatch on MaxConnsPerHost") + } + if dst.IdleConnTimeout != src.IdleConnTimeout { + t.Errorf("mismatch on IdleConnTimeout") + } + if dst.ResponseHeaderTimeout != src.ResponseHeaderTimeout { + t.Errorf("mismatch on ResponseHeaderTimeout") + } + if dst.ExpectContinueTimeout != src.ExpectContinueTimeout { + t.Errorf("mismatch on ExpectContinueTimeout") + } + if dst.MaxResponseHeaderBytes != src.MaxResponseHeaderBytes { + t.Errorf("mismatch on MaxResponseHeaderBytes") + } +} + +// Test that the name lookup of NewUTLSRoundTripper is case-insensitive. +func TestNewUTLSRoundTripperCase(t *testing.T) { + mixed, err := NewUTLSRoundTripper("HelloFirefox_Auto", nil, nil) + if err != nil { + t.Fatalf("error on %q: %v", "HelloFirefox_Auto", err) + } + upper, err := NewUTLSRoundTripper("HELLOFIREFOX_AUTO", nil, nil) + if err != nil { + t.Fatalf("error on %q: %v", "HELLOFIREFOX_AUTO", err) + } + lower, err := NewUTLSRoundTripper("hellofirefox_auto", nil, nil) + if err != nil { + t.Fatalf("error on %q: %v", "hellofirefox_auto", err) + } + if mixed.(*UTLSRoundTripper).clientHelloID != upper.(*UTLSRoundTripper).clientHelloID || + upper.(*UTLSRoundTripper).clientHelloID != lower.(*UTLSRoundTripper).clientHelloID { + t.Fatalf("mismatch %p %p %p", + mixed.(*UTLSRoundTripper).clientHelloID, + upper.(*UTLSRoundTripper).clientHelloID, + lower.(*UTLSRoundTripper).clientHelloID) + } +} + +// Return a byte slice which is the ClientHello sent when rt does a RoundTrip. +// Opens a temporary listener on an ephemeral port on localhost. The host you +// provide can be an IP address like "127.0.0.1" or a name like "localhost", but +// it has to resolve to localhost. +func clientHelloResultingFromRoundTrip(t *testing.T, host string, rt *UTLSRoundTripper) ([]byte, error) { + ch := make(chan []byte, 1) + + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, err + } + defer ln.Close() + + go func() { + defer func() { + close(ch) + }() + conn, err := ln.Accept() + if err != nil { + t.Error(err) + return + } + defer conn.Close() + buf := make([]byte, 1024) + n, err := conn.Read(buf) + if err != nil { + t.Error(err) + return + } + ch <- buf[:n] + }() + + _, port, err := net.SplitHostPort(ln.Addr().String()) + if err != nil { + return nil, err + } + u := &url.URL{ + Scheme: "https", + Host: net.JoinHostPort(host, port), + } + req, err := http.NewRequest("POST", u.String(), nil) + if err != nil { + return nil, err + } + // The RoundTrip fails because the goroutine "server" hangs up. So + // ignore an EOF error. + _, err = rt.RoundTrip(req) + if err != nil && err != io.EOF { + return nil, err + } + + return <-ch, nil +} + +// Test that a uTLS RoundTripper actually does something to the TLS Client +// Hello. We don't check all the ClientHelloIDs; this is just a guard against a +// catastrophic incompatibility or something else that makes uTLS stop working. +func TestUTLSClientHello(t *testing.T) { + // We use HelloIOS_11_1 because its lengthy ALPN means we will not + // confuse it with a native Go fingerprint, and lack of GREASE means we + // do not have to account for many variations. + rt, err := NewUTLSRoundTripper("HelloIOS_11_1", &utls.Config{InsecureSkipVerify: true, ServerName: "localhost"}, nil) + if err != nil { + panic(err) + } + + buf, err := clientHelloResultingFromRoundTrip(t, "127.0.0.1", rt.(*UTLSRoundTripper)) + // A poor man's regexp matching because the regexp package only works on + // UTF-8–encoded strings, not arbitrary byte slices. Every byte matches + // itself, except '.' which matches anything. NB '.' and '\x2e' are the + // same. + pattern := "" + + // Handshake, Client Hello, TLS 1.2, Client Random + "\x16\x03\x01\x01\x01\x01\x00\x00\xfd\x03\x03................................" + + // Session ID + "\x20................................" + + // Ciphersuites and compression methods + "\x00\x28\xc0\x2c\xc0\x2b\xc0\x24\xc0\x23\xc0\x0a\xc0\x09\xcc\xa9\xc0\x30\xc0\x2f\xc0\x28\xc0\x27\xc0\x14\xc0\x13\xcc\xa8\x00\x9d\x00\x9c\x00\x3d\x00\x3c\x00\x35\x00\x2f\x01\x00" + + // Extensions + "\x00\x8c\xff\x01\x00\x01\x00" + + "\x00\x00\x00\x0e\x00\x0c\x00\x00\x09localhost" + + "\x00\x17\x00\x00" + + "\x00\x0d\x00\x14\x00\x12\x04\x03\x08\x04\x04\x01\x05\x03\x08\x05\x05\x01\x08\x06\x06\x01\x02\x01" + + "\x00\x05\x00\x05\x01\x00\x00\x00\x00" + + "\x33\x74\x00\x00" + + "\x00\x12\x00\x00" + + "\x00\x10\x00\x30\x00\x2e\x02\x68\x32\x05\x68\x32\x2d\x31\x36\x05\x68\x32\x2d\x31\x35\x05\x68\x32\x2d\x31\x34\x08\x73\x70\x64\x79\x2f\x33\x2e\x31\x06\x73\x70\x64\x79\x2f\x33\x08\x68\x74\x74\x70\x2f\x31\x2e\x31" + + "\x00\x0b\x00\x02\x01\x00" + + "\x00\x0a\x00\x0a\x00\x08\x00\x1d\x00\x17\x00\x18\x00\x19" + if len(buf) != len(pattern) { + t.Errorf("fingerprint was not as expected: %+q", buf) + } + for i := 0; i < len(pattern); i++ { + a := buf[i] + b := pattern[i] + if b != '.' && a != b { + t.Fatalf("fingerprint mismatch a position %v: %+q", i, buf) + } + } +} + +func TestUTLSServerName(t *testing.T) { + const clientHelloIDName = "HelloFirefox_63" + + // No ServerName, dial IP address. Results in an invalid server_name + // extension with a 0-length host_name. Not sure if that's what it + // should do, but check if the behavior ever changes. + rt, err := NewUTLSRoundTripper(clientHelloIDName, &utls.Config{InsecureSkipVerify: true}, nil) + if err != nil { + panic(err) + } + buf, err := clientHelloResultingFromRoundTrip(t, "127.0.0.1", rt.(*UTLSRoundTripper)) + if err != nil { + panic(err) + } + if !bytes.Contains(buf, []byte("\x00\x00\x00\x05\x00\x03\x00\x00\x00")) { + t.Errorf("expected 0-length server_name extension with no ServerName and IP address dial") + } + + // No ServerName, dial hostname. server_name extension should come from + // the dial address. + rt, err = NewUTLSRoundTripper(clientHelloIDName, &utls.Config{InsecureSkipVerify: true}, nil) + if err != nil { + panic(err) + } + buf, err = clientHelloResultingFromRoundTrip(t, "localhost", rt.(*UTLSRoundTripper)) + if err != nil { + panic(err) + } + if !bytes.Contains(buf, []byte("\x00\x00\x00\x0e\x00\x0c\x00\x00\x09localhost")) { + t.Errorf("expected "localhost" server_name extension with no ServerName and hostname dial") + } + + // Given ServerName, dial IP address. server_name extension should from + // the ServerName. + rt, err = NewUTLSRoundTripper(clientHelloIDName, &utls.Config{InsecureSkipVerify: true, ServerName: "test.example"}, nil) + if err != nil { + panic(err) + } + buf, err = clientHelloResultingFromRoundTrip(t, "127.0.0.1", rt.(*UTLSRoundTripper)) + if err != nil { + panic(err) + } + if !bytes.Contains(buf, []byte("\x00\x00\x00\x11\x00\x0f\x00\x00\x0ctest.example")) { + t.Errorf("expected "test.example" server_name extension with given ServerName and IP address dial") + } + + // Given ServerName, dial hostname. server_name extension should from + // the ServerName. + rt, err = NewUTLSRoundTripper(clientHelloIDName, &utls.Config{InsecureSkipVerify: true, ServerName: "test.example"}, nil) + if err != nil { + panic(err) + } + buf, err = clientHelloResultingFromRoundTrip(t, "localhost", rt.(*UTLSRoundTripper)) + if err != nil { + panic(err) + } + if !bytes.Contains(buf, []byte("\x00\x00\x00\x11\x00\x0f\x00\x00\x0ctest.example")) { + t.Errorf("expected "test.example" server_name extension with given ServerName and hostname dial") + } +}