This is an automated email from the git hooks/post-receive script.
shelikhoo pushed a change to branch main in repository pluggable-transports/snowflake.
from 19e9e38 Merge remote-tracking branch 'gitlab/mr/78' new 006abde Add utls roundtripper new 4447860 Add repeated test for utls roundtripper new c1b0f76 Add reformat for utls roundtripper new c1c3596 Add name to utls client hello id new 9af0ad1 Add utls imitate setting to snowflake client new ccfdcab Add uTLS remove SNI to snowflake client new 1573502 Use uTLS aware broker channel constructor new f525490 Update utls test to match uTLS Round Tripper constructor new e3aeb5f Add line wrap to NewBrokerChannelWithUTlsSettings new 8d5998b Harmonize identifiers to uTLS new 3132f68 Add connection expire time for uTLS pendingConn new ab96044 Move uTLS configuration to socks5 arg new 6e29dc6 Add document for NewUTLSHTTPRoundTripper
The 13 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
Summary of changes: client/lib/rendezvous.go | 28 ++++- client/lib/snowflake.go | 11 +- client/snowflake.go | 11 ++ common/utls/client_hello_id.go | 38 +++++++ common/utls/roundtripper.go | 238 +++++++++++++++++++++++++++++++++++++++ common/utls/roundtripper_test.go | 156 +++++++++++++++++++++++++ go.mod | 1 + go.sum | 2 + 8 files changed, 479 insertions(+), 6 deletions(-) create mode 100644 common/utls/client_hello_id.go create mode 100644 common/utls/roundtripper.go create mode 100644 common/utls/roundtripper_test.go
This is an automated email from the git hooks/post-receive script.
shelikhoo pushed a commit to branch main in repository pluggable-transports/snowflake.
commit 006abdead41579022c36da337c23de45600966ab Author: Shelikhoo xiaokangwang@outlook.com AuthorDate: Wed Feb 9 15:36:54 2022 +0000
Add utls roundtripper --- common/utls/roundtripper.go | 191 +++++++++++++++++++++++++++++++++++++++ common/utls/roundtripper_test.go | 153 +++++++++++++++++++++++++++++++ go.mod | 1 + go.sum | 2 + 4 files changed, 347 insertions(+)
diff --git a/common/utls/roundtripper.go b/common/utls/roundtripper.go new file mode 100644 index 0000000..e2fc82b --- /dev/null +++ b/common/utls/roundtripper.go @@ -0,0 +1,191 @@ +package utls + +import ( + "context" + "crypto/tls" + "errors" + "net" + "net/http" + "sync" + + utls "github.com/refraction-networking/utls" + "golang.org/x/net/http2" +) + +func NewUTLSHTTPRoundTripper(clientHelloID utls.ClientHelloID, uTlsConfig *utls.Config, + backdropTransport http.RoundTripper, removeSNI bool) http.RoundTripper { + rtImpl := &uTLSHTTPRoundTripperImpl{ + clientHelloID: clientHelloID, + config: uTlsConfig, + connectWithH1: map[string]bool{}, + backdropTransport: backdropTransport, + pendingConn: map[pendingConnKey]net.Conn{}, + removeSNI: removeSNI, + } + rtImpl.init() + return rtImpl +} + +type uTLSHTTPRoundTripperImpl struct { + clientHelloID utls.ClientHelloID + config *utls.Config + + accessConnectWithH1 sync.Mutex + connectWithH1 map[string]bool + + httpsH1Transport http.RoundTripper + httpsH2Transport http.RoundTripper + backdropTransport http.RoundTripper + + accessDialingConnection sync.Mutex + pendingConn map[pendingConnKey]net.Conn + + removeSNI bool +} + +type pendingConnKey struct { + isH2 bool + dest string +} + +var errEAGAIN = errors.New("incorrect ALPN negotiated, try again with another ALPN") +var errEAGAINTooMany = errors.New("incorrect ALPN negotiated") + +func (r *uTLSHTTPRoundTripperImpl) RoundTrip(req *http.Request) (*http.Response, error) { + if req.URL.Scheme != "https" { + return r.backdropTransport.RoundTrip(req) + } + for retryCount := 0; retryCount < 5; retryCount++ { + if r.getShouldConnectWithH1(req.URL.Host) { + resp, err := r.httpsH1Transport.RoundTrip(req) + if errors.Is(err, errEAGAIN) { + continue + } + return resp, err + } + resp, err := r.httpsH2Transport.RoundTrip(req) + if errors.Is(err, errEAGAIN) { + continue + } + return resp, err + } + return nil, errEAGAINTooMany +} + +func (r *uTLSHTTPRoundTripperImpl) getShouldConnectWithH1(domainName string) bool { + r.accessConnectWithH1.Lock() + defer r.accessConnectWithH1.Unlock() + if value, set := r.connectWithH1[domainName]; set { + return value + } + return false +} + +func (r *uTLSHTTPRoundTripperImpl) setShouldConnectWithH1(domainName string) { + r.accessConnectWithH1.Lock() + defer r.accessConnectWithH1.Unlock() + r.connectWithH1[domainName] = true +} + +func (r *uTLSHTTPRoundTripperImpl) clearShouldConnectWithH1(domainName string) { + r.accessConnectWithH1.Lock() + defer r.accessConnectWithH1.Unlock() + r.connectWithH1[domainName] = false +} + +func getPendingConnectionID(dest string, alpnIsH2 bool) pendingConnKey { + return pendingConnKey{isH2: alpnIsH2, dest: dest} +} + +func (r *uTLSHTTPRoundTripperImpl) putConn(addr string, alpnIsH2 bool, conn net.Conn) { + connId := getPendingConnectionID(addr, alpnIsH2) + r.pendingConn[connId] = conn +} +func (r *uTLSHTTPRoundTripperImpl) getConn(addr string, alpnIsH2 bool) net.Conn { + connId := getPendingConnectionID(addr, alpnIsH2) + if conn, ok := r.pendingConn[connId]; ok { + return conn + } + return nil +} +func (r *uTLSHTTPRoundTripperImpl) dialOrGetTLSWithExpectedALPN(ctx context.Context, addr string, expectedH2 bool) (net.Conn, error) { + r.accessDialingConnection.Lock() + defer r.accessDialingConnection.Unlock() + + if r.getShouldConnectWithH1(addr) == expectedH2 { + return nil, errEAGAIN + } + + //Get a cached connection if possible to reduce preflight connection closed without sending data + if gconn := r.getConn(addr, expectedH2); gconn != nil { + return gconn, nil + } + + conn, err := r.dialTLS(ctx, addr) + if err != nil { + return nil, err + } + + protocol := conn.ConnectionState().NegotiatedProtocol + + protocolIsH2 := protocol == http2.NextProtoTLS + + if protocolIsH2 == expectedH2 { + return conn, err + } + + r.putConn(addr, protocolIsH2, conn) + + if protocolIsH2 { + r.clearShouldConnectWithH1(addr) + } else { + r.setShouldConnectWithH1(addr) + } + + return nil, errEAGAIN +} + +// based on https://repo.or.cz/dnstt.git/commitdiff/d92a791b6864901f9263f7d73d97cfd30ac5... +// by dcf1 +func (r *uTLSHTTPRoundTripperImpl) dialTLS(ctx context.Context, addr string) (*utls.UConn, error) { + config := r.config.Clone() + + host, _, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + config.ServerName = host + + dialer := &net.Dialer{} + conn, err := dialer.DialContext(ctx, "tcp", addr) + if err != nil { + return nil, err + } + uconn := utls.UClient(conn, config, r.clientHelloID) + if (net.ParseIP(config.ServerName) != nil) || r.removeSNI { + err := uconn.RemoveSNIExtension() + if err != nil { + uconn.Close() + return nil, err + } + } + + err = uconn.Handshake() + if err != nil { + return nil, err + } + return uconn, nil +} + +func (r *uTLSHTTPRoundTripperImpl) init() { + r.httpsH2Transport = &http2.Transport{ + DialTLS: func(network, addr string, cfg *tls.Config) (net.Conn, error) { + return r.dialOrGetTLSWithExpectedALPN(context.Background(), addr, true) + }, + } + r.httpsH1Transport = &http.Transport{ + DialTLSContext: func(ctx context.Context, network string, addr string) (net.Conn, error) { + return r.dialOrGetTLSWithExpectedALPN(ctx, addr, false) + }, + } +} diff --git a/common/utls/roundtripper_test.go b/common/utls/roundtripper_test.go new file mode 100644 index 0000000..b0209ff --- /dev/null +++ b/common/utls/roundtripper_test.go @@ -0,0 +1,153 @@ +package utls + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + utls "github.com/refraction-networking/utls" + "golang.org/x/net/http2" + "math/big" + "net/http" + "testing" + "time" +) + +import . "github.com/smartystreets/goconvey/convey" + +import stdcontext "context" + +func TestRoundTripper(t *testing.T) { + var selfSignedCert []byte + var selfSignedPrivateKey *rsa.PrivateKey + httpServerContext, cancel := stdcontext.WithCancel(stdcontext.Background()) + Convey("[Test]Set up http servers", t, func(c C) { + c.Convey("[Test]Generate Self-Signed Cert", func(c C) { + // Ported from https://gist.github.com/samuel/8b500ddd3f6118d052b5e6bc16bc4c09 + priv, err := rsa.GenerateKey(rand.Reader, 4096) + c.So(err, ShouldBeNil) + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + CommonName: "Testing Certificate", + }, + NotBefore: time.Now(), + NotAfter: time.Now().Add(time.Hour * 24 * 180), + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, priv.Public(), priv) + c.So(err, ShouldBeNil) + selfSignedPrivateKey = priv + selfSignedCert = derBytes + }) + c.Convey("[Test]Setup http2 server", func(c C) { + listener, err := tls.Listen("tcp", "127.0.0.1:23802", &tls.Config{ + NextProtos: []string{http2.NextProtoTLS}, + Certificates: []tls.Certificate{ + tls.Certificate{Certificate: [][]byte{selfSignedCert}, PrivateKey: selfSignedPrivateKey}, + }, + }) + c.So(err, ShouldBeNil) + s := http.Server{} + go s.Serve(listener) + go func() { + <-httpServerContext.Done() + s.Close() + }() + }) + c.Convey("[Test]Setup http1 server", func(c C) { + listener, err := tls.Listen("tcp", "127.0.0.1:23801", &tls.Config{ + NextProtos: []string{"http/1.1"}, + Certificates: []tls.Certificate{ + tls.Certificate{Certificate: [][]byte{selfSignedCert}, PrivateKey: selfSignedPrivateKey}, + }, + }) + c.So(err, ShouldBeNil) + s := http.Server{} + go s.Serve(listener) + go func() { + <-httpServerContext.Done() + s.Close() + }() + }) + }) + for _, v := range []struct { + id utls.ClientHelloID + name string + }{ + { + id: utls.HelloChrome_58, + name: "HelloChrome_58", + }, + { + id: utls.HelloChrome_62, + name: "HelloChrome_62", + }, + { + id: utls.HelloChrome_70, + name: "HelloChrome_70", + }, + { + id: utls.HelloChrome_72, + name: "HelloChrome_72", + }, + { + id: utls.HelloChrome_83, + name: "HelloChrome_83", + }, + { + id: utls.HelloFirefox_55, + name: "HelloFirefox_55", + }, + { + id: utls.HelloFirefox_55, + name: "HelloFirefox_55", + }, + { + id: utls.HelloFirefox_63, + name: "HelloFirefox_63", + }, + { + id: utls.HelloFirefox_65, + name: "HelloFirefox_65", + }, + { + id: utls.HelloIOS_11_1, + name: "HelloIOS_11_1", + }, + { + id: utls.HelloIOS_12_1, + name: "HelloIOS_12_1", + }, + } { + t.Run("Testing fingerprint for "+v.name, func(t *testing.T) { + rtter := NewUTLSHTTPRoundTripper(v.id, &utls.Config{ + InsecureSkipVerify: true, + }, http.DefaultTransport) + + Convey("HTTP 1.1 Test", t, func(c C) { + { + req, err := http.NewRequest("GET", "https://127.0.0.1:23801/", nil) + So(err, ShouldBeNil) + _, err = rtter.RoundTrip(req) + So(err, ShouldBeNil) + } + }) + + Convey("HTTP 2 Test", t, func(c C) { + { + req, err := http.NewRequest("GET", "https://127.0.0.1:23802/", nil) + So(err, ShouldBeNil) + _, err = rtter.RoundTrip(req) + So(err, ShouldBeNil) + } + }) + }) + } + + cancel() +} diff --git a/go.mod b/go.mod index 03541eb..705c05a 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/pion/webrtc/v3 v3.0.15 github.com/prometheus/client_golang v1.10.0 github.com/prometheus/client_model v0.2.0 + github.com/refraction-networking/utls v1.0.0 // indirect github.com/smartystreets/goconvey v1.6.4 github.com/stretchr/testify v1.7.0 // indirect github.com/xtaci/kcp-go/v5 v5.6.1 diff --git a/go.sum b/go.sum index ecf91a3..c2fa108 100644 --- a/go.sum +++ b/go.sum @@ -302,6 +302,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/refraction-networking/utls v1.0.0 h1:6XQHSjDmeBCF9sPq8p2zMVGq7Ud3rTD2q88Fw8Tz1tA= +github.com/refraction-networking/utls v1.0.0/go.mod h1:tz9gX959MEFfFN5whTIocCLUG57WiILqtdVxI8c6Wj0= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
This is an automated email from the git hooks/post-receive script.
shelikhoo pushed a commit to branch main in repository pluggable-transports/snowflake.
commit 44478606615c3ff848d9d9749a17fd89430aa6d9 Author: Shelikhoo xiaokangwang@outlook.com AuthorDate: Wed Feb 9 15:38:27 2022 +0000
Add repeated test for utls roundtripper --- common/utls/roundtripper_test.go | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-)
diff --git a/common/utls/roundtripper_test.go b/common/utls/roundtripper_test.go index b0209ff..90c09bd 100644 --- a/common/utls/roundtripper_test.go +++ b/common/utls/roundtripper_test.go @@ -129,23 +129,25 @@ func TestRoundTripper(t *testing.T) { InsecureSkipVerify: true, }, http.DefaultTransport)
- Convey("HTTP 1.1 Test", t, func(c C) { - { - req, err := http.NewRequest("GET", "https://127.0.0.1:23801/", nil) - So(err, ShouldBeNil) - _, err = rtter.RoundTrip(req) - So(err, ShouldBeNil) - } - }) + for count := 0; count <= 10; count++ { + Convey("HTTP 1.1 Test", t, func(c C) { + { + req, err := http.NewRequest("GET", "https://127.0.0.1:23801/", nil) + So(err, ShouldBeNil) + _, err = rtter.RoundTrip(req) + So(err, ShouldBeNil) + } + })
- Convey("HTTP 2 Test", t, func(c C) { - { - req, err := http.NewRequest("GET", "https://127.0.0.1:23802/", nil) - So(err, ShouldBeNil) - _, err = rtter.RoundTrip(req) - So(err, ShouldBeNil) - } - }) + Convey("HTTP 2 Test", t, func(c C) { + { + req, err := http.NewRequest("GET", "https://127.0.0.1:23802/", nil) + So(err, ShouldBeNil) + _, err = rtter.RoundTrip(req) + So(err, ShouldBeNil) + } + }) + } }) }
This is an automated email from the git hooks/post-receive script.
shelikhoo pushed a commit to branch main in repository pluggable-transports/snowflake.
commit c1b0f763efb57316469d689d7addf53566685f78 Author: Shelikhoo xiaokangwang@outlook.com AuthorDate: Wed Feb 9 15:43:53 2022 +0000
Add reformat for utls roundtripper --- common/utls/roundtripper_test.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/common/utls/roundtripper_test.go b/common/utls/roundtripper_test.go index 90c09bd..0962b4a 100644 --- a/common/utls/roundtripper_test.go +++ b/common/utls/roundtripper_test.go @@ -6,17 +6,18 @@ import ( "crypto/tls" "crypto/x509" "crypto/x509/pkix" - utls "github.com/refraction-networking/utls" - "golang.org/x/net/http2" "math/big" "net/http" "testing" "time" -)
-import . "github.com/smartystreets/goconvey/convey" + stdcontext "context"
-import stdcontext "context" + utls "github.com/refraction-networking/utls" + "golang.org/x/net/http2" + + . "github.com/smartystreets/goconvey/convey" +)
func TestRoundTripper(t *testing.T) { var selfSignedCert []byte
This is an automated email from the git hooks/post-receive script.
shelikhoo pushed a commit to branch main in repository pluggable-transports/snowflake.
commit c1c3596cf8bbc87b180e6d916da9515e27609969 Author: Max Bittman bittmanmax@gmail.com AuthorDate: Thu Feb 10 11:46:58 2022 +0000
Add name to utls client hello id --- common/utls/client_hello_id.go | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+)
diff --git a/common/utls/client_hello_id.go b/common/utls/client_hello_id.go new file mode 100644 index 0000000..e423cf3 --- /dev/null +++ b/common/utls/client_hello_id.go @@ -0,0 +1,38 @@ +package utls + +import ( + "errors" + utls "github.com/refraction-networking/utls" + "strings" +) + +// ported from https://github.com/max-b/snowflake/commit/9dded063cb74c6941a16ad90b9dd0e06e6... +var clientHelloIDMap = map[string]utls.ClientHelloID{ + // No HelloCustom: not useful for external configuration. + // No HelloRandomized: doesn't negotiate consistent ALPN. + "hellorandomizedalpn": utls.HelloRandomizedALPN, + "hellorandomizednoalpn": utls.HelloRandomizedNoALPN, + "hellofirefox_auto": utls.HelloFirefox_Auto, + "hellofirefox_55": utls.HelloFirefox_55, + "hellofirefox_56": utls.HelloFirefox_56, + "hellofirefox_63": utls.HelloFirefox_63, + "hellofirefox_65": utls.HelloFirefox_65, + "hellochrome_auto": utls.HelloChrome_Auto, + "hellochrome_58": utls.HelloChrome_58, + "hellochrome_62": utls.HelloChrome_62, + "hellochrome_70": utls.HelloChrome_70, + "hellochrome_72": utls.HelloChrome_72, + "helloios_auto": utls.HelloIOS_Auto, + "helloios_11_1": utls.HelloIOS_11_1, + "helloios_12_1": utls.HelloIOS_12_1, +} + +var errNameNotFound = errors.New("client hello name is unrecognized") + +func NameToUTlsID(name string) (utls.ClientHelloID, error) { + normalizedName := strings.ToLower(name) + if id, ok := clientHelloIDMap[normalizedName]; ok { + return id, nil + } + return utls.ClientHelloID{}, errNameNotFound +}
This is an automated email from the git hooks/post-receive script.
shelikhoo pushed a commit to branch main in repository pluggable-transports/snowflake.
commit 9af0ad119b8b0f129f015c5347fe5a3b03596ff0 Author: Shelikhoo xiaokangwang@outlook.com AuthorDate: Thu Feb 10 17:04:42 2022 +0000
Add utls imitate setting to snowflake client --- client/lib/rendezvous.go | 27 +++++++++++++++++++++++---- client/lib/snowflake.go | 3 +++ client/snowflake.go | 2 ++ 3 files changed, 28 insertions(+), 4 deletions(-)
diff --git a/client/lib/rendezvous.go b/client/lib/rendezvous.go index 98cd4d6..4c7c6f9 100644 --- a/client/lib/rendezvous.go +++ b/client/lib/rendezvous.go @@ -5,6 +5,8 @@ package snowflake_client
import ( "errors" + "fmt" + "log" "net/http" "sync" @@ -14,7 +16,9 @@ import ( "git.torproject.org/pluggable-transports/snowflake.git/v2/common/messages" "git.torproject.org/pluggable-transports/snowflake.git/v2/common/nat" "git.torproject.org/pluggable-transports/snowflake.git/v2/common/util" + utlsutil "git.torproject.org/pluggable-transports/snowflake.git/v2/common/utls" "github.com/pion/webrtc/v3" + utls "github.com/refraction-networking/utls" )
const ( @@ -51,10 +55,14 @@ func createBrokerTransport() http.RoundTripper { return transport }
-// NewBrokerChannel construct a new BrokerChannel, where: +func NewBrokerChannel(broker, ampCache, front string, keepLocalAddresses bool) (*BrokerChannel, error) { + return NewBrokerChannelWithUTlsClientID(broker, ampCache, front, keepLocalAddresses, "") +} + +// NewBrokerChannelWithUTlsClientID construct a new BrokerChannel, where: // |broker| is the full URL of the facilitating program which assigns proxies // to clients, and |front| is the option fronting domain. -func NewBrokerChannel(broker, ampCache, front string, keepLocalAddresses bool) (*BrokerChannel, error) { +func NewBrokerChannelWithUTlsClientID(broker, ampCache, front string, keepLocalAddresses bool, utlsClientID string) (*BrokerChannel, error) { log.Println("Rendezvous using Broker at:", broker) if ampCache != "" { log.Println("Through AMP cache at:", ampCache) @@ -63,12 +71,23 @@ func NewBrokerChannel(broker, ampCache, front string, keepLocalAddresses bool) ( log.Println("Domain fronting using:", front) }
+ brokerTransport := createBrokerTransport() + + if utlsClientID != "" { + utlsClientHelloID, err := utlsutil.NameToUTlsID(utlsClientID) + if err != nil { + return nil, fmt.Errorf("unable to create broker channel: %v", err) + } + config := &utls.Config{} + brokerTransport = utlsutil.NewUTLSHTTPRoundTripper(utlsClientHelloID, config, brokerTransport, false) + } + var rendezvous RendezvousMethod var err error if ampCache != "" { - rendezvous, err = newAMPCacheRendezvous(broker, ampCache, front, createBrokerTransport()) + rendezvous, err = newAMPCacheRendezvous(broker, ampCache, front, brokerTransport) } else { - rendezvous, err = newHTTPRendezvous(broker, front, createBrokerTransport()) + rendezvous, err = newHTTPRendezvous(broker, front, brokerTransport) } if err != nil { return nil, err diff --git a/client/lib/snowflake.go b/client/lib/snowflake.go index 594c62c..19442d8 100644 --- a/client/lib/snowflake.go +++ b/client/lib/snowflake.go @@ -97,6 +97,9 @@ type ClientConfig struct { // Max is the maximum number of snowflake proxy peers that the client should attempt to // connect to. Defaults to 1. Max int + // UTlsClientID is the type of user application that snowflake should imitate. + // If an empty value is provided, it will use Go's default TLS implementation + UTlsClientID string }
// NewSnowflakeClient creates a new Snowflake transport client that can spawn multiple diff --git a/client/snowflake.go b/client/snowflake.go index 5a00206..addedb9 100644 --- a/client/snowflake.go +++ b/client/snowflake.go @@ -126,6 +126,7 @@ func main() { frontDomain := flag.String("front", "", "front domain") ampCacheURL := flag.String("ampcache", "", "URL of AMP cache to use as a proxy for signaling") logFilename := flag.String("log", "", "name of log file") + utlsClientHelloID := flag.String("utls-imitate", "", "type of TLS client to imitate with utls") logToStateDir := flag.Bool("log-to-state-dir", false, "resolve the log file relative to tor's pt state dir") keepLocalAddresses := flag.Bool("keep-local-addresses", false, "keep local LAN address ICE candidates") unsafeLogging := flag.Bool("unsafe-logging", false, "prevent logs from being scrubbed") @@ -178,6 +179,7 @@ func main() { ICEAddresses: iceAddresses, KeepLocalAddresses: *keepLocalAddresses || *oldKeepLocalAddresses, Max: *max, + UTlsClientID: *utlsClientHelloID, }
// Begin goptlib client process.
This is an automated email from the git hooks/post-receive script.
shelikhoo pushed a commit to branch main in repository pluggable-transports/snowflake.
commit ccfdcab8feb7857a3089f2a88bc2e1e6c52d5865 Author: Shelikhoo xiaokangwang@outlook.com AuthorDate: Fri Feb 11 09:57:37 2022 +0000
Add uTLS remove SNI to snowflake client --- client/lib/rendezvous.go | 6 +++--- client/lib/snowflake.go | 3 +++ client/snowflake.go | 2 ++ 3 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/client/lib/rendezvous.go b/client/lib/rendezvous.go index 4c7c6f9..7c27dfc 100644 --- a/client/lib/rendezvous.go +++ b/client/lib/rendezvous.go @@ -56,13 +56,13 @@ func createBrokerTransport() http.RoundTripper { }
func NewBrokerChannel(broker, ampCache, front string, keepLocalAddresses bool) (*BrokerChannel, error) { - return NewBrokerChannelWithUTlsClientID(broker, ampCache, front, keepLocalAddresses, "") + return NewBrokerChannelWithUTlsClientID(broker, ampCache, front, keepLocalAddresses, "", false) }
// NewBrokerChannelWithUTlsClientID construct a new BrokerChannel, where: // |broker| is the full URL of the facilitating program which assigns proxies // to clients, and |front| is the option fronting domain. -func NewBrokerChannelWithUTlsClientID(broker, ampCache, front string, keepLocalAddresses bool, utlsClientID string) (*BrokerChannel, error) { +func NewBrokerChannelWithUTlsClientID(broker, ampCache, front string, keepLocalAddresses bool, utlsClientID string, removeSNI bool) (*BrokerChannel, error) { log.Println("Rendezvous using Broker at:", broker) if ampCache != "" { log.Println("Through AMP cache at:", ampCache) @@ -79,7 +79,7 @@ func NewBrokerChannelWithUTlsClientID(broker, ampCache, front string, keepLocalA return nil, fmt.Errorf("unable to create broker channel: %v", err) } config := &utls.Config{} - brokerTransport = utlsutil.NewUTLSHTTPRoundTripper(utlsClientHelloID, config, brokerTransport, false) + brokerTransport = utlsutil.NewUTLSHTTPRoundTripper(utlsClientHelloID, config, brokerTransport, removeSNI) }
var rendezvous RendezvousMethod diff --git a/client/lib/snowflake.go b/client/lib/snowflake.go index 19442d8..510567e 100644 --- a/client/lib/snowflake.go +++ b/client/lib/snowflake.go @@ -100,6 +100,9 @@ type ClientConfig struct { // UTlsClientID is the type of user application that snowflake should imitate. // If an empty value is provided, it will use Go's default TLS implementation UTlsClientID string + // UTlsRemoveSNI is the flag to control whether SNI should be removed from Client Hello + // when uTLS is used. + UTlsRemoveSNI bool }
// NewSnowflakeClient creates a new Snowflake transport client that can spawn multiple diff --git a/client/snowflake.go b/client/snowflake.go index addedb9..a693ca6 100644 --- a/client/snowflake.go +++ b/client/snowflake.go @@ -127,6 +127,7 @@ func main() { ampCacheURL := flag.String("ampcache", "", "URL of AMP cache to use as a proxy for signaling") logFilename := flag.String("log", "", "name of log file") utlsClientHelloID := flag.String("utls-imitate", "", "type of TLS client to imitate with utls") + utlsRemoveSNI := flag.Bool("utls-nosni", false, "remove SNI from client hello(ignored if uTLS is not used)") logToStateDir := flag.Bool("log-to-state-dir", false, "resolve the log file relative to tor's pt state dir") keepLocalAddresses := flag.Bool("keep-local-addresses", false, "keep local LAN address ICE candidates") unsafeLogging := flag.Bool("unsafe-logging", false, "prevent logs from being scrubbed") @@ -180,6 +181,7 @@ func main() { KeepLocalAddresses: *keepLocalAddresses || *oldKeepLocalAddresses, Max: *max, UTlsClientID: *utlsClientHelloID, + UTlsRemoveSNI: *utlsRemoveSNI, }
// Begin goptlib client process.
This is an automated email from the git hooks/post-receive script.
shelikhoo pushed a commit to branch main in repository pluggable-transports/snowflake.
commit 1573502e93b7149e8a4784e62bb1adc979312940 Author: Shelikhoo xiaokangwang@outlook.com AuthorDate: Fri Feb 11 10:03:45 2022 +0000
Use uTLS aware broker channel constructor --- client/lib/rendezvous.go | 6 +++--- client/lib/snowflake.go | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/client/lib/rendezvous.go b/client/lib/rendezvous.go index 7c27dfc..ee07600 100644 --- a/client/lib/rendezvous.go +++ b/client/lib/rendezvous.go @@ -56,13 +56,13 @@ func createBrokerTransport() http.RoundTripper { }
func NewBrokerChannel(broker, ampCache, front string, keepLocalAddresses bool) (*BrokerChannel, error) { - return NewBrokerChannelWithUTlsClientID(broker, ampCache, front, keepLocalAddresses, "", false) + return NewBrokerChannelWithUTlsSettings(broker, ampCache, front, keepLocalAddresses, "", false) }
-// NewBrokerChannelWithUTlsClientID construct a new BrokerChannel, where: +// NewBrokerChannelWithUTlsSettings construct a new BrokerChannel, where: // |broker| is the full URL of the facilitating program which assigns proxies // to clients, and |front| is the option fronting domain. -func NewBrokerChannelWithUTlsClientID(broker, ampCache, front string, keepLocalAddresses bool, utlsClientID string, removeSNI bool) (*BrokerChannel, error) { +func NewBrokerChannelWithUTlsSettings(broker, ampCache, front string, keepLocalAddresses bool, utlsClientID string, removeSNI bool) (*BrokerChannel, error) { log.Println("Rendezvous using Broker at:", broker) if ampCache != "" { log.Println("Through AMP cache at:", ampCache) diff --git a/client/lib/snowflake.go b/client/lib/snowflake.go index 510567e..e309b44 100644 --- a/client/lib/snowflake.go +++ b/client/lib/snowflake.go @@ -131,8 +131,9 @@ func NewSnowflakeClient(config ClientConfig) (*Transport, error) { }
// Rendezvous with broker using the given parameters. - broker, err := NewBrokerChannel( - config.BrokerURL, config.AmpCacheURL, config.FrontDomain, config.KeepLocalAddresses) + broker, err := NewBrokerChannelWithUTlsSettings( + config.BrokerURL, config.AmpCacheURL, config.FrontDomain, + config.KeepLocalAddresses, config.UTlsClientID, config.UTlsRemoveSNI) if err != nil { return nil, err }
This is an automated email from the git hooks/post-receive script.
shelikhoo pushed a commit to branch main in repository pluggable-transports/snowflake.
commit f5254900320aaa03b63c281351ebb145395d6357 Author: Shelikhoo xiaokangwang@outlook.com AuthorDate: Fri Feb 11 10:18:52 2022 +0000
Update utls test to match uTLS Round Tripper constructor --- common/utls/roundtripper_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/common/utls/roundtripper_test.go b/common/utls/roundtripper_test.go index 0962b4a..6a91385 100644 --- a/common/utls/roundtripper_test.go +++ b/common/utls/roundtripper_test.go @@ -128,7 +128,7 @@ func TestRoundTripper(t *testing.T) { t.Run("Testing fingerprint for "+v.name, func(t *testing.T) { rtter := NewUTLSHTTPRoundTripper(v.id, &utls.Config{ InsecureSkipVerify: true, - }, http.DefaultTransport) + }, http.DefaultTransport, false)
for count := 0; count <= 10; count++ { Convey("HTTP 1.1 Test", t, func(c C) {
This is an automated email from the git hooks/post-receive script.
shelikhoo pushed a commit to branch main in repository pluggable-transports/snowflake.
commit e3aeb5fe5b3cace5d482c2fa40e0b964711ab189 Author: Shelikhoo xiaokangwang@outlook.com AuthorDate: Fri Feb 11 10:23:15 2022 +0000
Add line wrap to NewBrokerChannelWithUTlsSettings --- client/lib/rendezvous.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/client/lib/rendezvous.go b/client/lib/rendezvous.go index ee07600..dcf613c 100644 --- a/client/lib/rendezvous.go +++ b/client/lib/rendezvous.go @@ -62,7 +62,8 @@ func NewBrokerChannel(broker, ampCache, front string, keepLocalAddresses bool) ( // NewBrokerChannelWithUTlsSettings construct a new BrokerChannel, where: // |broker| is the full URL of the facilitating program which assigns proxies // to clients, and |front| is the option fronting domain. -func NewBrokerChannelWithUTlsSettings(broker, ampCache, front string, keepLocalAddresses bool, utlsClientID string, removeSNI bool) (*BrokerChannel, error) { +func NewBrokerChannelWithUTlsSettings(broker, ampCache, front string, keepLocalAddresses bool, + utlsClientID string, removeSNI bool) (*BrokerChannel, error) { log.Println("Rendezvous using Broker at:", broker) if ampCache != "" { log.Println("Through AMP cache at:", ampCache)
This is an automated email from the git hooks/post-receive script.
shelikhoo pushed a commit to branch main in repository pluggable-transports/snowflake.
commit 8d5998b7441eb9e213b8d86052e94a27d7656495 Author: Shelikhoo xiaokangwang@outlook.com AuthorDate: Fri Feb 11 11:26:41 2022 +0000
Harmonize identifiers to uTLS --- client/lib/rendezvous.go | 12 ++++++------ client/lib/snowflake.go | 12 ++++++------ client/snowflake.go | 8 ++++---- common/utls/client_hello_id.go | 2 +- 4 files changed, 17 insertions(+), 17 deletions(-)
diff --git a/client/lib/rendezvous.go b/client/lib/rendezvous.go index dcf613c..1fc2a69 100644 --- a/client/lib/rendezvous.go +++ b/client/lib/rendezvous.go @@ -56,14 +56,14 @@ func createBrokerTransport() http.RoundTripper { }
func NewBrokerChannel(broker, ampCache, front string, keepLocalAddresses bool) (*BrokerChannel, error) { - return NewBrokerChannelWithUTlsSettings(broker, ampCache, front, keepLocalAddresses, "", false) + return NewBrokerChannelWithUTLSSettings(broker, ampCache, front, keepLocalAddresses, "", false) }
-// NewBrokerChannelWithUTlsSettings construct a new BrokerChannel, where: +// NewBrokerChannelWithUTLSSettings construct a new BrokerChannel, where: // |broker| is the full URL of the facilitating program which assigns proxies // to clients, and |front| is the option fronting domain. -func NewBrokerChannelWithUTlsSettings(broker, ampCache, front string, keepLocalAddresses bool, - utlsClientID string, removeSNI bool) (*BrokerChannel, error) { +func NewBrokerChannelWithUTLSSettings(broker, ampCache, front string, keepLocalAddresses bool, + uTLSClientID string, removeSNI bool) (*BrokerChannel, error) { log.Println("Rendezvous using Broker at:", broker) if ampCache != "" { log.Println("Through AMP cache at:", ampCache) @@ -74,8 +74,8 @@ func NewBrokerChannelWithUTlsSettings(broker, ampCache, front string, keepLocalA
brokerTransport := createBrokerTransport()
- if utlsClientID != "" { - utlsClientHelloID, err := utlsutil.NameToUTlsID(utlsClientID) + if uTLSClientID != "" { + utlsClientHelloID, err := utlsutil.NameToUTLSID(uTLSClientID) if err != nil { return nil, fmt.Errorf("unable to create broker channel: %v", err) } diff --git a/client/lib/snowflake.go b/client/lib/snowflake.go index e309b44..1c6c381 100644 --- a/client/lib/snowflake.go +++ b/client/lib/snowflake.go @@ -97,12 +97,12 @@ type ClientConfig struct { // Max is the maximum number of snowflake proxy peers that the client should attempt to // connect to. Defaults to 1. Max int - // UTlsClientID is the type of user application that snowflake should imitate. + // UTLSClientID is the type of user application that snowflake should imitate. // If an empty value is provided, it will use Go's default TLS implementation - UTlsClientID string - // UTlsRemoveSNI is the flag to control whether SNI should be removed from Client Hello + UTLSClientID string + // UTLSRemoveSNI is the flag to control whether SNI should be removed from Client Hello // when uTLS is used. - UTlsRemoveSNI bool + UTLSRemoveSNI bool }
// NewSnowflakeClient creates a new Snowflake transport client that can spawn multiple @@ -131,9 +131,9 @@ func NewSnowflakeClient(config ClientConfig) (*Transport, error) { }
// Rendezvous with broker using the given parameters. - broker, err := NewBrokerChannelWithUTlsSettings( + broker, err := NewBrokerChannelWithUTLSSettings( config.BrokerURL, config.AmpCacheURL, config.FrontDomain, - config.KeepLocalAddresses, config.UTlsClientID, config.UTlsRemoveSNI) + config.KeepLocalAddresses, config.UTLSClientID, config.UTLSRemoveSNI) if err != nil { return nil, err } diff --git a/client/snowflake.go b/client/snowflake.go index a693ca6..76a5cc4 100644 --- a/client/snowflake.go +++ b/client/snowflake.go @@ -126,8 +126,8 @@ func main() { frontDomain := flag.String("front", "", "front domain") ampCacheURL := flag.String("ampcache", "", "URL of AMP cache to use as a proxy for signaling") logFilename := flag.String("log", "", "name of log file") - utlsClientHelloID := flag.String("utls-imitate", "", "type of TLS client to imitate with utls") - utlsRemoveSNI := flag.Bool("utls-nosni", false, "remove SNI from client hello(ignored if uTLS is not used)") + uTLSClientHelloID := flag.String("utls-imitate", "", "type of TLS client to imitate with utls") + uTLSRemoveSNI := flag.Bool("utls-nosni", false, "remove SNI from client hello(ignored if uTLS is not used)") logToStateDir := flag.Bool("log-to-state-dir", false, "resolve the log file relative to tor's pt state dir") keepLocalAddresses := flag.Bool("keep-local-addresses", false, "keep local LAN address ICE candidates") unsafeLogging := flag.Bool("unsafe-logging", false, "prevent logs from being scrubbed") @@ -180,8 +180,8 @@ func main() { ICEAddresses: iceAddresses, KeepLocalAddresses: *keepLocalAddresses || *oldKeepLocalAddresses, Max: *max, - UTlsClientID: *utlsClientHelloID, - UTlsRemoveSNI: *utlsRemoveSNI, + UTLSClientID: *uTLSClientHelloID, + UTLSRemoveSNI: *uTLSRemoveSNI, }
// Begin goptlib client process. diff --git a/common/utls/client_hello_id.go b/common/utls/client_hello_id.go index e423cf3..8a13280 100644 --- a/common/utls/client_hello_id.go +++ b/common/utls/client_hello_id.go @@ -29,7 +29,7 @@ var clientHelloIDMap = map[string]utls.ClientHelloID{
var errNameNotFound = errors.New("client hello name is unrecognized")
-func NameToUTlsID(name string) (utls.ClientHelloID, error) { +func NameToUTLSID(name string) (utls.ClientHelloID, error) { normalizedName := strings.ToLower(name) if id, ok := clientHelloIDMap[normalizedName]; ok { return id, nil
This is an automated email from the git hooks/post-receive script.
shelikhoo pushed a commit to branch main in repository pluggable-transports/snowflake.
commit 3132f680122e27bb9cfb957fbb29c3cbe73935cf Author: Shelikhoo xiaokangwang@outlook.com AuthorDate: Wed Feb 16 11:11:37 2022 +0000
Add connection expire time for uTLS pendingConn --- common/utls/roundtripper.go | 47 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 4 deletions(-)
diff --git a/common/utls/roundtripper.go b/common/utls/roundtripper.go index e2fc82b..df31ff4 100644 --- a/common/utls/roundtripper.go +++ b/common/utls/roundtripper.go @@ -7,6 +7,7 @@ import ( "net" "net/http" "sync" + "time"
utls "github.com/refraction-networking/utls" "golang.org/x/net/http2" @@ -19,7 +20,7 @@ func NewUTLSHTTPRoundTripper(clientHelloID utls.ClientHelloID, uTlsConfig *utls. config: uTlsConfig, connectWithH1: map[string]bool{}, backdropTransport: backdropTransport, - pendingConn: map[pendingConnKey]net.Conn{}, + pendingConn: map[pendingConnKey]*unclaimedConnection{}, removeSNI: removeSNI, } rtImpl.init() @@ -38,7 +39,7 @@ type uTLSHTTPRoundTripperImpl struct { backdropTransport http.RoundTripper
accessDialingConnection sync.Mutex - pendingConn map[pendingConnKey]net.Conn + pendingConn map[pendingConnKey]*unclaimedConnection
removeSNI bool } @@ -50,6 +51,7 @@ type pendingConnKey struct {
var errEAGAIN = errors.New("incorrect ALPN negotiated, try again with another ALPN") var errEAGAINTooMany = errors.New("incorrect ALPN negotiated") +var errExpired = errors.New("connection have expired")
func (r *uTLSHTTPRoundTripperImpl) RoundTrip(req *http.Request) (*http.Response, error) { if req.URL.Scheme != "https" { @@ -99,12 +101,15 @@ func getPendingConnectionID(dest string, alpnIsH2 bool) pendingConnKey {
func (r *uTLSHTTPRoundTripperImpl) putConn(addr string, alpnIsH2 bool, conn net.Conn) { connId := getPendingConnectionID(addr, alpnIsH2) - r.pendingConn[connId] = conn + r.pendingConn[connId] = NewUnclaimedConnection(conn, time.Minute) } func (r *uTLSHTTPRoundTripperImpl) getConn(addr string, alpnIsH2 bool) net.Conn { connId := getPendingConnectionID(addr, alpnIsH2) if conn, ok := r.pendingConn[connId]; ok { - return conn + delete(r.pendingConn, connId) + if claimedConnection, err := conn.claimConnection(); err == nil { + return claimedConnection + } } return nil } @@ -189,3 +194,37 @@ func (r *uTLSHTTPRoundTripperImpl) init() { }, } } + +func NewUnclaimedConnection(conn net.Conn, expireTime time.Duration) *unclaimedConnection { + c := &unclaimedConnection{ + Conn: conn, + } + time.AfterFunc(expireTime, c.tick) + return c +} + +type unclaimedConnection struct { + net.Conn + claimed bool + access sync.Mutex +} + +func (c *unclaimedConnection) claimConnection() (net.Conn, error) { + c.access.Lock() + defer c.access.Unlock() + if !c.claimed { + c.claimed = true + return c.Conn, nil + } + return nil, errExpired +} + +func (c *unclaimedConnection) tick() { + c.access.Lock() + defer c.access.Unlock() + if !c.claimed { + c.claimed = true + c.Conn.Close() + c.Conn = nil + } +}
This is an automated email from the git hooks/post-receive script.
shelikhoo pushed a commit to branch main in repository pluggable-transports/snowflake.
commit ab9604476ee7673cf35a3aea33e225946a1426e0 Author: Shelikhoo xiaokangwang@outlook.com AuthorDate: Mon Mar 7 16:32:47 2022 +0000
Move uTLS configuration to socks5 arg --- client/snowflake.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-)
diff --git a/client/snowflake.go b/client/snowflake.go index 76a5cc4..5856750 100644 --- a/client/snowflake.go +++ b/client/snowflake.go @@ -84,6 +84,17 @@ func socksAcceptLoop(ln *pt.SocksListener, config sf.ClientConfig, shutdown chan if arg, ok := conn.Req.Args.Get("url"); ok { config.BrokerURL = arg } + if arg, ok := conn.Req.Args.Get("utls-nosni"); ok { + switch strings.ToLower(arg) { + case "true": + fallthrough + case "yes": + config.UTLSRemoveSNI = true + } + } + if arg, ok := conn.Req.Args.Get("utls-imitate"); ok { + config.UTLSClientID = arg + } transport, err := sf.NewSnowflakeClient(config) if err != nil { conn.Reject() @@ -126,8 +137,6 @@ func main() { frontDomain := flag.String("front", "", "front domain") ampCacheURL := flag.String("ampcache", "", "URL of AMP cache to use as a proxy for signaling") logFilename := flag.String("log", "", "name of log file") - uTLSClientHelloID := flag.String("utls-imitate", "", "type of TLS client to imitate with utls") - uTLSRemoveSNI := flag.Bool("utls-nosni", false, "remove SNI from client hello(ignored if uTLS is not used)") logToStateDir := flag.Bool("log-to-state-dir", false, "resolve the log file relative to tor's pt state dir") keepLocalAddresses := flag.Bool("keep-local-addresses", false, "keep local LAN address ICE candidates") unsafeLogging := flag.Bool("unsafe-logging", false, "prevent logs from being scrubbed") @@ -180,8 +189,6 @@ func main() { ICEAddresses: iceAddresses, KeepLocalAddresses: *keepLocalAddresses || *oldKeepLocalAddresses, Max: *max, - UTLSClientID: *uTLSClientHelloID, - UTLSRemoveSNI: *uTLSRemoveSNI, }
// Begin goptlib client process.
This is an automated email from the git hooks/post-receive script.
shelikhoo pushed a commit to branch main in repository pluggable-transports/snowflake.
commit 6e29dc676c44ea1b6fe5f13aec48aae91ff1cc3c Author: Shelikhoo xiaokangwang@outlook.com AuthorDate: Tue Mar 15 12:27:29 2022 +0000
Add document for NewUTLSHTTPRoundTripper --- common/utls/roundtripper.go | 8 ++++++++ 1 file changed, 8 insertions(+)
diff --git a/common/utls/roundtripper.go b/common/utls/roundtripper.go index df31ff4..53b997f 100644 --- a/common/utls/roundtripper.go +++ b/common/utls/roundtripper.go @@ -13,6 +13,14 @@ import ( "golang.org/x/net/http2" )
+// NewUTLSHTTPRoundTripper creates an instance of RoundTripper that dial to remote HTTPS endpoint with +// an alternative version of TLS implementation that attempts to imitate browsers' fingerprint. +// clientHelloID is the clientHello that uTLS attempts to imitate +// uTlsConfig is the TLS Configuration template +// backdropTransport is the transport that will be used for non-https traffic +// removeSNI indicates not to send Server Name Indication Extension +// returns a RoundTripper: its behaviour is documented at +// https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowf... func NewUTLSHTTPRoundTripper(clientHelloID utls.ClientHelloID, uTlsConfig *utls.Config, backdropTransport http.RoundTripper, removeSNI bool) http.RoundTripper { rtImpl := &uTLSHTTPRoundTripperImpl{
tor-commits@lists.torproject.org
-
gitolite role