commit 724098922e9a60b5c0654cbc923a90ed3c8386e6 Author: sergeyfrolov sergey.frolov@colorado.edu Date: Tue Jul 17 17:24:48 2018 -0400
Initial commit --- README.md | 97 ++++++++ client/client.go | 436 ++++++++++++++++++++++++++++++++++ server/inithack/hack.go | 41 ++++ server/server.go | 616 ++++++++++++++++++++++++++++++++++++++++++++++++ server/server_test.go | 113 +++++++++ 5 files changed, 1303 insertions(+)
diff --git a/README.md b/README.md new file mode 100644 index 0000000..93f4b7c --- /dev/null +++ b/README.md @@ -0,0 +1,97 @@ +# [httpsproxy](https://trac.torproject.org/projects/tor/ticket/26923) + +## Spin up a full Tor bridge +This instruction explains how to install and start following components: +* Caddy Web Server +* Pluggable Transport +* Tor daemon + +```bash +sudo apt install tor + +# build server from source code +git clone https://git.torproject.org/pluggable-transports/httpsproxy.git +cd httpsproxy/server +go get +go build +sudo cp server /var/lib/tor/httpsproxy + +# allow binding to ports 80 and 443 +sudo /sbin/setcap 'cap_net_bind_service=+ep' /var/lib/tor/httpsproxy +sudo sed -i -e 's/NoNewPrivileges=yes/NoNewPrivileges=no/g' /lib/systemd/system/tor@default.service +sudo sed -i -e 's/NoNewPrivileges=yes/NoNewPrivileges=no/g' /lib/systemd/system/tor@.service +sudo systemctl daemon-reload + +# don't forget to set correct ContactInfo +sudo cat <<EOT >> /etc/tor/torrc + RunAsDaemon 1 + BridgeRelay 1 + ExitRelay 0 + + PublishServerDescriptor 0 # 1 for public bridge + + ORPort 9001 + ExtORPort auto + + ServerTransportPlugin httpsproxy exec /var/lib/tor/httpsproxy -servername yourdomain.com -agree -email youremail@gmail.com + Address 1.2.3.4 # might be required per https://trac.torproject.org/projects/tor/ticket/12020 + + ContactInfo Dr Stephen Falken steph@gtnw.org + Nickname joshua +EOT + +sudo systemctl start tor + +# monitor logs: +sudo less +F /var/log/tor/log +sudo less +F /var/lib/tor/pt_state/caddy.log +``` + +### PT arguments +As mentioned in code, `flag` package is global and PT arguments are passed together with those of Caddy. + +``` + +Usage of ./server: + -runcaddy + Start Caddy web server on ports 443 and 80 (redirects to 443) together with the PT. + You can disable this option, set static 'ServerTransportListenAddr httpsproxy 127.0.0.1:ptPort' in torrc, + spin up frontend manually, and forward client's CONNECT request to 127.0.0.1:ptPort. (default true) + -servername string + Server Name used. Used as TLS SNI on the client side, and to start Caddy. + -agree + Agree to the CA's Subscriber Agreement + -email string + Default ACME CA account email address + -cert string + Path to TLS cert. Requires --key. If set, caddy will not get Lets Encrypt TLS certificate. + -key string + Path to TLS key. Requires --cert. If set, caddy will not get Lets Encrypt TLS certificate. + -logfile string + Log file for Pluggable Transport. (default: "$TOR_PT_STATE_LOCATION/caddy.log" -> /var/lib/tor/pt_state/caddy.log) + -url string + Set/override access url in form of https://username:password@1.2.3.4:443/. + If servername is set or cert argument has a certificate with correct domain name, + this arg is optional and will be inferred, username:password will be auto-generated and stored, if not provided. +``` + +## Configure client + +Ideally, this will be integrated with the Tor browser and distributed automatically, so clients would have to do nothing +In the meantime, here's how to test it with Tor Browser Bundle: + +1. Download [Tor Browser](https://www.torproject.org/projects/torbrowser.html.en) +2. Build httpsclient and configure torrc: +``` + git clone https://git.torproject.org/pluggable-transports/httpsproxy.git + cd httpsproxy/client + go get + go build + PATH_TO_CLIENT=`pwd` + PATH_TO_TORRC="/etc/tor/torrc" # if TBB is used, path will be different + echo "ClientTransportPlugin httpsproxy exec ${PATH_TO_CLIENT}/client" >> $PATH_TO_TORRC +``` +4. Launch Tor Browser, select "Tor is censored in my country" -> "Provide a bridge I know" +5. Copy bridge line like "httpsproxy 0.4.2.0:3 url=https://username:password@httpsproxy.com". + If you set up your own server, bridge line will be printed to caddy.log on server launch. + diff --git a/client/client.go b/client/client.go new file mode 100644 index 0000000..eea871d --- /dev/null +++ b/client/client.go @@ -0,0 +1,436 @@ +// Copyright 2018 Jigsaw Operations LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// HTTPS proxy based pluggable transport client. +package main + +import ( + "bufio" + "crypto/tls" + "encoding/base64" + "errors" + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "net" + "net/http" + "net/url" + "os" + "os/signal" + "strconv" + "sync" + "syscall" + + pt "git.torproject.org/pluggable-transports/goptlib.git" + "golang.org/x/net/http2" +) + +var ptInfo pt.ClientInfo + +// When a connection handler starts, +1 is written to this channel; when it +// ends, -1 is written. +var handlerChan = make(chan int) + +// TODO: stop goroutine leaking in copyLoops: if one side closes - close another after timeout + +// This function is copypasted from https://github.com/caddyserver/forwardproxy/blob/master/forwardproxy.go +// TODO: replace with padding-enabled function +// flushingIoCopy is analogous to buffering io.Copy(), but also attempts to flush on each iteration. +// If dst does not implement http.Flusher(e.g. net.TCPConn), it will do a simple io.CopyBuffer(). +// Reasoning: http2ResponseWriter will not flush on its own, so we have to do it manually. +func flushingIoCopy(dst io.Writer, src io.Reader, buf []byte) (written int64, err error) { + flusher, ok := dst.(http.Flusher) + if !ok { + return io.CopyBuffer(dst, src, buf) + } + for { + nr, er := src.Read(buf) + if nr > 0 { + nw, ew := dst.Write(buf[0:nr]) + flusher.Flush() + if nw > 0 { + written += int64(nw) + } + if ew != nil { + err = ew + break + } + if nr != nw { + err = io.ErrShortWrite + break + } + } + if er != nil { + if er != io.EOF { + err = er + } + break + } + } + return +} + +// simple copy loop without padding, works with http/1.1 +// TODO: we can't pad, but we probably can split +func copyLoop(local, remote net.Conn) { + var wg sync.WaitGroup + wg.Add(2) + + go func() { + io.Copy(remote, local) + wg.Done() + }() + go func() { + io.Copy(local, remote) + wg.Done() + }() + // TODO: try not to spawn extra goroutine + + wg.Wait() +} + +func h2copyLoop(w1 io.Writer, r1 io.Reader, w2 io.Writer, r2 io.Reader) { + var wg sync.WaitGroup + wg.Add(2) + + buf1 := make([]byte, 16384) + buf2 := make([]byte, 16384) + go func() { + flushingIoCopy(w1, r1, buf1) + wg.Done() + }() + go func() { + flushingIoCopy(w2, r2, buf2) + wg.Done() + }() + // TODO: try not to spawn extra goroutine + + wg.Wait() +} + +func parseTCPAddr(s string) (*net.TCPAddr, error) { + hostStr, portStr, err := net.SplitHostPort(s) + if err != nil { + fmt.Printf("net.SplitHostPort(%s) failed: %+v", s, err) + return nil, err + } + + port, err := strconv.Atoi(portStr) + if err != nil { + fmt.Printf("strconv.Atoi(%s) failed: %+v", portStr, err) + return nil, err + } + + ip := net.ParseIP(hostStr) + if ip == nil { + err = errors.New("net.ParseIP(" + s + ") returned nil") + fmt.Printf("%+v\n", err) + return nil, err + } + + return &net.TCPAddr{Port: port, IP: ip}, nil +} + +// handler will process a PT request, requests webproxy(that is given in URL arg) to connect to +// the Req.Target and relay traffic between client and webproxy +func handler(conn *pt.SocksConn) error { + handlerChan <- 1 + defer func() { + handlerChan <- -1 + }() + defer conn.Close() + + guardTCPAddr, err := parseTCPAddr(conn.Req.Target) + if err != nil { + conn.Reject() + return err + } + + webproxyUrlArg, ok := conn.Req.Args.Get("url") + if !ok { + err := errors.New("address of webproxy in form of `url=https://username:password@example.com%60 is required") + conn.Reject() + return err + } + + httpsClient, err := NewHTTPSClient(webproxyUrlArg) + if err != nil { + log.Printf("NewHTTPSClient(%s, nil) failed: %s\n", webproxyUrlArg, err) + conn.Reject() + return err + } + + err = httpsClient.Connect(conn.Req.Target) + if err != nil { + log.Printf("httpsClient.Connect(%s, nil) failed: %s\n", conn.Req.Target, err) + conn.Reject() + return err + } + + err = conn.Grant(guardTCPAddr) + if err != nil { + log.Printf("conn.Grant(%s) failed: %s\n", guardTCPAddr, err) + conn.Reject() + return err + } + + return httpsClient.CopyLoop(conn) +} + +func acceptLoop(ln *pt.SocksListener) error { + defer ln.Close() + for { + conn, err := ln.AcceptSocks() + if err != nil { + if e, ok := err.(net.Error); ok && e.Temporary() { + continue + } + return err + } + go handler(conn) + } +} + +func main() { + var err error + + logFile := flag.String("log", "", "Log file for debugging") + flag.Parse() + + ptInfo, err = pt.ClientSetup(nil) + if err != nil { + os.Exit(1) + } + + if ptInfo.ProxyURL != nil { + pt.ProxyError("proxy is not supported") + os.Exit(1) + } + + if *logFile != "" { + f, err := os.OpenFile(*logFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0660) + if err != nil { + pt.CmethodError("httpsproxy", + fmt.Sprintf("error opening file %s: %v", logFile, err)) + os.Exit(2) + } + defer f.Close() + log.SetOutput(f) + } + + listeners := make([]net.Listener, 0) + for _, methodName := range ptInfo.MethodNames { + switch methodName { + case "httpsproxy": + ln, err := pt.ListenSocks("tcp", "127.0.0.1:0") + if err != nil { + pt.CmethodError(methodName, err.Error()) + break + } + go acceptLoop(ln) + pt.Cmethod(methodName, ln.Version(), ln.Addr()) + log.Printf("Started %s %s at %s\n", methodName, ln.Version(), ln.Addr()) + listeners = append(listeners, ln) + default: + pt.CmethodError(methodName, "no such method") + } + } + pt.CmethodsDone() + + var numHandlers = 0 + var sig os.Signal + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGTERM) + + if os.Getenv("TOR_PT_EXIT_ON_STDIN_CLOSE") == "1" { + // This environment variable means we should treat EOF on stdin + // just like SIGTERM: https://bugs.torproject.org/15435. + go func() { + io.Copy(ioutil.Discard, os.Stdin) + sigChan <- syscall.SIGTERM + }() + } + + // keep track of handlers and wait for a signal + sig = nil + for sig == nil { + select { + case n := <-handlerChan: + numHandlers += n + case sig = <-sigChan: + } + } + + // signal received, shut down + for _, ln := range listeners { + ln.Close() + } + for numHandlers > 0 { + numHandlers += <-handlerChan + } +} + +type HTTPConnectClient struct { + Header http.Header + ProxyHost string + TlsConf tls.Config + + Conn *tls.Conn + + In io.Writer + Out io.Reader +} + +// NewHTTPSClient creates one-time use client to tunnel traffic via HTTPS proxy. +// If spkiFp is set, HTTPSClient will use it as SPKI fingerprint to confirm identity of the +// proxy, instead of relying on standard PKI CA roots +func NewHTTPSClient(proxyUrlStr string) (*HTTPConnectClient, error) { + proxyUrl, err := url.Parse(proxyUrlStr) + if err != nil { + return nil, err + } + + switch proxyUrl.Scheme { + case "http", "": + fallthrough + default: + return nil, errors.New("Scheme " + proxyUrl.Scheme + " is not supported") + case "https": + } + + if proxyUrl.Host == "" { + return nil, errors.New("misparsed `url=`, make sure to specify full url like https://username:password@hostname.com:443/") + } + + if proxyUrl.Port() == "" { + proxyUrl.Host = net.JoinHostPort(proxyUrl.Host, "443") + } + + tlsConf := tls.Config{ + NextProtos: []string{"h2", "http/1.1"}, + ServerName: proxyUrl.Hostname(), + } + + client := &HTTPConnectClient{ + Header: make(http.Header), + ProxyHost: proxyUrl.Host, + TlsConf: tlsConf, + } + + if proxyUrl.User.Username() != "" { + password, _ := proxyUrl.User.Password() + client.Header.Set("Proxy-Authorization", "Basic "+ + base64.StdEncoding.EncodeToString([]byte(proxyUrl.User.Username()+":"+password))) + } + return client, nil +} + +func (c *HTTPConnectClient) Connect(target string) error { + req := &http.Request{ + Method: "CONNECT", + URL: &url.URL{Host: target}, + Header: c.Header, + Host: target, + } + + tcpConn, err := net.Dial("tcp", c.ProxyHost) + if err != nil { + return err + } + + c.Conn = tls.Client(tcpConn, &c.TlsConf) + + err = c.Conn.Handshake() + if err != nil { + return err + } + + var resp *http.Response + switch c.Conn.ConnectionState().NegotiatedProtocol { + case "": + fallthrough + case "http/1.1": + req.Proto = "HTTP/1.1" + req.ProtoMajor = 1 + req.ProtoMinor = 1 + + err = req.Write(c.Conn) + if err != nil { + c.Conn.Close() + return err + } + + resp, err = http.ReadResponse(bufio.NewReader(c.Conn), req) + if err != nil { + c.Conn.Close() + return err + } + + c.In = c.Conn + c.Out = c.Conn + case "h2": + req.Proto = "HTTP/2.0" + req.ProtoMajor = 2 + req.ProtoMinor = 0 + pr, pw := io.Pipe() + req.Body = ioutil.NopCloser(pr) + + t := http2.Transport{} + h2client, err := t.NewClientConn(c.Conn) + if err != nil { + c.Conn.Close() + return err + } + + resp, err = h2client.RoundTrip(req) + if err != nil { + c.Conn.Close() + return err + } + + c.In = pw + c.Out = resp.Body + default: + c.Conn.Close() + return errors.New("negotiated unsupported application layer protocol: " + + c.Conn.ConnectionState().NegotiatedProtocol) + } + + if resp.StatusCode != http.StatusOK { + c.Conn.Close() + return errors.New("Proxy responded with non 200 code: " + resp.Status) + } + + return nil +} + +func (c *HTTPConnectClient) CopyLoop(conn net.Conn) error { + defer c.Conn.Close() + defer conn.Close() + + switch c.Conn.ConnectionState().NegotiatedProtocol { + case "": + fallthrough + case "http/1.1": + copyLoop(conn, c.Conn) + case "h2": + h2copyLoop(c.In, conn, conn, c.Out) + default: + return errors.New("negotiated unsupported application layer protocol: " + + c.Conn.ConnectionState().NegotiatedProtocol) + } + return nil +} diff --git a/server/inithack/hack.go b/server/inithack/hack.go new file mode 100644 index 0000000..b3801ca --- /dev/null +++ b/server/inithack/hack.go @@ -0,0 +1,41 @@ +// Copyright 2018 Jigsaw Operations LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package inithack + +import ( + "os" + "path" +) + +// We need to override CADDYPATH and make sure caddy writes things into TOR_PT_STATE_LOCATION +// as opposed to HOME (which will be something like "/root/" and hardening features rightfully +// prevent PT from writing there). Unfortunately, Caddy uses CADDYPATH in init() functions, which +// run before than anything in "server" package. +// +// For now, which just set CADDYPATH=TOR_PT_STATE_LOCATION here and import it before caddy. +// https://golang.org/ref/spec#Package_initialization does not guarantee a particular init order, +// which is why we should find an actual fix. TODO! +// +// Potential fixes: +// 1) refactor Caddy: seems like a big patch +// 2) Set CADDYHOME environment variable from Tor: torrc doesn't seem to allow setting arbitrary env vars +// 3) Change Tor behavior to set HOME to TOR_PT_STATE_LOCATION? +// 4) govendor Caddy, and change its source code to import this package, guaranteeing init order +// 5) run Caddy as a separate binary. +func init() { + if os.Getenv("CADDYPATH") == "" { + os.Setenv("CADDYPATH", path.Join(os.Getenv("TOR_PT_STATE_LOCATION"), ".caddy")) + } +} diff --git a/server/server.go b/server/server.go new file mode 100644 index 0000000..7f86319 --- /dev/null +++ b/server/server.go @@ -0,0 +1,616 @@ +// Copyright 2018 Jigsaw Operations LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "crypto/rand" + "crypto/sha256" + "crypto/x509" + "encoding/hex" + "encoding/pem" + "errors" + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "net" + "net/http" + "net/url" + "os" + "os/signal" + "path" + "runtime/debug" + "strings" + "sync" + "syscall" + + _ "github.com/Jigsaw-Code/volunteer/server/inithack" + + pt "git.torproject.org/pluggable-transports/goptlib.git" + "github.com/mholt/caddy" + + // imports below are to run init() and register the forwardproxy plugin, set default variables + _ "github.com/caddyserver/forwardproxy" + _ "github.com/mholt/caddy/caddy/caddymain" +) + +// TODO: stop goroutine leaking in copyLoops + +var ptInfo pt.ServerInfo + +// When a connection handler starts, +1 is written to this channel; when it +// ends, -1 is written. +var handlerChan = make(chan int) + +func copyLoop(a, b net.Conn) { + var wg sync.WaitGroup + wg.Add(2) + + go func() { + io.Copy(b, a) + wg.Done() + }() + go func() { + io.Copy(a, b) + wg.Done() + }() + wg.Wait() +} + +// Parses Forwarded and X-Forwarded-For headers, and returns client's IP:port. +// According to RFC, hostnames and addresses without port are valid, but Tor spec mandates IP:port, +// so those are currently return error. +// Returns "", nil if there are no headers indicating forwarding. +// Returns "", err if there are forwarding headers, but they are misformatted/don't contain IP:port +// Returns "IPAddr", nil on successful parse. +func parseForwardedTor(header http.Header) (string, error) { + var ipAddr string + proxyEvidence := false + xFFHeader := header.Get("X-Forwarded-For") + if xFFHeader != "" { + proxyEvidence = true + for _, ip := range strings.Split(xFFHeader, ",") { + ipAddr = strings.Trim(ip, " "") + break + } + } + forwardedHeader := header.Get("Forwarded") + if forwardedHeader != "" { + proxyEvidence = true + for _, fValue := range strings.Split(forwardedHeader, ";") { + s := strings.Split(fValue, "=") + if len(s) != 2 { + return "", errors.New("misformatted "Forwarded:" header") + } + if strings.ToLower(strings.Trim(s[0], " ")) == "for" { + ipAddr = strings.Trim(s[1], " "") + break + } + } + } + if ipAddr == "" { + if proxyEvidence == true { + return "", errors.New("Forwarded or X-Forwarded-For header is present, but could not be parsed") + } + return "", nil + } + + // According to https://github.com/torproject/torspec/blob/master/proposals/196-transport-co... + // there are 2 acceptable formats: + // 1.2.3.4:5678 + // [1:2::3:4]:5678 // (spec says [1:2::3:4]::5678 but that must be a typo) + h, p, err := net.SplitHostPort(ipAddr) + if err != nil { + return "", err + } + if net.ParseIP(h) == nil { + return "", errors.New(h + " is not a valid IP address") + } + return net.JoinHostPort(h, p), nil +} + +func handler(conn net.Conn) error { + defer conn.Close() + + handlerChan <- 1 + defer func() { + handlerChan <- -1 + }() + var err error + + req, err := http.ReadRequest(bufio.NewReader(conn)) + if err != nil { + return err + } + + clientIP, err := parseForwardedTor(req.Header) + if err != nil { + // just print the error to log. eventually, we may decide to reject connections, + // if Forwarded/X-Forwarded-For header is present, but misformatted/misparsed + log.Println(err) + } + if clientIP == "" { + // if err != nil, conn.RemoteAddr() is certainly not the right IP + // but testing showed that connection fails to establish if clientIP is empty + clientIP = conn.RemoteAddr().String() + } + + or, err := pt.DialOr(&ptInfo, clientIP, "httpsproxy") + if err != nil { + return err + } + defer or.Close() + + // TODO: consider adding support for HTTP/2, HAPROXY-style PROXY protocol, SOCKS, etc. + _, err = conn.Write([]byte("HTTP/1.1 200 OK\r\n\r\n")) + if err != nil { + return err + } + + copyLoop(conn, or) + + return nil +} + +func acceptLoop(ln net.Listener) error { + defer ln.Close() + for { + conn, err := ln.Accept() + if err != nil { + if e, ok := err.(net.Error); ok && e.Temporary() { + continue + } + return err + } + go handler(conn) + } +} + +var ( + torPtStateLocationEnvVar string // directory where PT is allowed to store things + + bridgeUrl url.URL // bridge URL to register with bridgeDB + + // cli args + runCaddy bool + serverName string + keyPemPath, certPemPath string + cliUrlPTstr string + logFile string +) + +func parseValidateCliArgs() error { + // flag package is global and arguments get inevitably mixed with those of Caddy + // It's a bit messy, but allows us to easily pass arguments to Caddy + // To cleanup, we would have to reimplement argument parsing (or use 3rd party flag package) + flag.BoolVar(&runCaddy, "runcaddy", true, "Start Caddy web server on ports 443 and 80 (redirects to 443) together with the PT."+ + " You can disable this option, set static 'ServerTransportListenAddr httpsproxy 127.0.0.1:ptPort' in torrc,"+ + " spin up frontend manually, and forward client's CONNECT request to 127.0.0.1:ptPort.") + flag.StringVar(&serverName, "servername", "", "Server Name used. Used as TLS SNI on the client side, and to start Caddy.") + + flag.StringVar(&keyPemPath, "key", "", "Path to TLS key. Requires --cert. If set, caddy will not get Lets Encrypt TLS certificate.") + flag.StringVar(&certPemPath, "cert", "", "Path to TLS cert. Requires --key. If set, caddy will not get Lets Encrypt TLS certificate.") + + flag.StringVar(&cliUrlPTstr, "url", "", "Set/override access url in form of https://username:password@1.2.3.4:443/.%22+ + " If servername is set or cert argument has a certificate with correct domain name,"+ + " this arg is optional and will be inferred, username:password will be auto-generated and stored, if not provided.") + + flag.StringVar(&logFile, "logfile", path.Join(torPtStateLocationEnvVar, "caddy.log"), + "Log file for Pluggable Transport.") + flag.Parse() + + if (keyPemPath == "" && certPemPath != "") || (keyPemPath != "" && certPemPath == "") { + return errors.New("--cert and --key options must be used together") + } + + if runCaddy == true && (serverName == "" && keyPemPath == "" && cliUrlPTstr == "") { + return errors.New("for automatic launch of Caddy web server(`runcaddy=true` by default)," + + "please specify either --servername, --url, or --cert and --key") + } + + var err error + cliUrlPT := &url.URL{} + if cliUrlPTstr != "" { + cliUrlPT, err = url.Parse(cliUrlPTstr) + if err != nil { + return err + } + } + + var storedCredentials *url.Userinfo + if cliUrlPT.User.Username() == "" && runCaddy == true { + // if operator hasn't specified the credentials in url and requests to start caddy, + // use credentials, stored to disk + storedCredentials, err = readCredentialsFromConfig() + if err != nil { + quitWithSmethodError(err.Error()) + } + err := saveCredentialsToConfig(storedCredentials) + if err != nil { + // if can't save credentials persistently, and they were NOT provided as cli, die + quitWithSmethodError( + fmt.Sprintf("failed to save auto-generated proxy credentials: %s."+ + "Fix the error or specify credentials in `url=` argument", err)) + } + } + + bridgeUrl, err = generatePTUrl(*cliUrlPT, storedCredentials, &serverName) + return err +} + +func quitWithSmethodError(errStr string) { + pt.SmethodError("httpsproxy", errStr) + os.Exit(2) +} + +var sigChan chan os.Signal + +func main() { + defer func() { + if r := recover(); r != nil { + log.Printf("panic: %s\nstack trace: %s\n", r, debug.Stack()) + pt.ProxyError(fmt.Sprintf("panic: %v. (check PT log for detailed trace)", r)) + } + }() + + torPtStateLocationEnvVar = os.Getenv("TOR_PT_STATE_LOCATION") + if torPtStateLocationEnvVar == "" { + quitWithSmethodError("Set torPtStateLocationEnvVar") + } + err := os.MkdirAll(torPtStateLocationEnvVar, 0700) + if err != nil { + quitWithSmethodError(fmt.Sprintf("Failed to open/create %s: %s", torPtStateLocationEnvVar, err)) + } + + if err := parseValidateCliArgs(); err != nil { + quitWithSmethodError("failed to parse PT arguments: " + err.Error()) + } + + if logFile != "" { + f, err := os.OpenFile(logFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0660) + if err != nil { + quitWithSmethodError(fmt.Sprintf("error opening file %s: %v", logFile, err)) + } + defer f.Close() + log.SetOutput(f) + os.Stdout = f + os.Stderr = f + } + + ptInfo, err = pt.ServerSetup(nil) + if err != nil { + quitWithSmethodError(err.Error()) + } + + var ptAddr net.Addr + if len(ptInfo.Bindaddrs) != 1 { + // TODO: is it even useful to have multiple bindaddrs and how would we use them? We don't + // want to accept direct connections to PT, as it doesn't use security protocols like TLS + quitWithSmethodError("only one bind address is supported") + } + bindaddr := ptInfo.Bindaddrs[0] + if bindaddr.MethodName != "httpsproxy" { + quitWithSmethodError("no such method") + } + + listener, err := net.ListenTCP("tcp", bindaddr.Addr) + if err != nil { + quitWithSmethodError(err.Error()) + } + ptAddr = listener.Addr() + colonIdx := strings.LastIndex(ptAddr.String(), ":") + if colonIdx == -1 || len(ptAddr.String()) == colonIdx+1 { + quitWithSmethodError("Bindaddr " + ptAddr.String() + " does not contain port") + } + ptAddrPort := ptAddr.String()[colonIdx+1:] + + go acceptLoop(listener) + + ptBridgeLineArgs := make(pt.Args) + if serverName != "" { + ptBridgeLineArgs["sni"] = []string{serverName} + } + + var numHandlers int = 0 + var sig os.Signal + + sigChan = make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGTERM) + + if runCaddy { + startUsptreamingCaddy("http://localhost:%22+ptAddrPort, ptBridgeLineArgs) + } + + ptBridgeLineArgs["proxy"] = []string{bridgeUrl.String()} + + // print bridge line + argsAsString := func(args *pt.Args) string { + str := "" + for k, v := range *args { + str += k + "=" + strings.Join(v, ",") + " " + } + return strings.Trim(str, " ") + } + log.Printf("Bridge line: %s %s [fingerprint] %s\n", + bindaddr.MethodName, listener.Addr(), argsAsString(&ptBridgeLineArgs)) + + // register bridge line + pt.SmethodArgs(bindaddr.MethodName, listener.Addr(), ptBridgeLineArgs) + pt.SmethodsDone() + + if os.Getenv("TOR_PT_EXIT_ON_STDIN_CLOSE") == "1" { + // This environment variable means we should treat EOF on stdin + // just like SIGTERM: https://bugs.torproject.org/15435. + go func() { + io.Copy(ioutil.Discard, os.Stdin) + sigChan <- syscall.SIGTERM + }() + } + + // keep track of handlers and wait for a signal + sig = nil + for sig == nil { + select { + case n := <-handlerChan: + numHandlers += n + case sig = <-sigChan: + log.Println("Got EOF on stdin, exiting") + } + } + + // signal received, shut down + listener.Close() + + for numHandlers > 0 { + numHandlers += <-handlerChan + } +} + +func generateRandomString(length int) string { + const alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + + randByte := make([]byte, 1) + + var randStr string + for i := 0; i < length; i++ { + _, err := rand.Read(randByte) + if err != nil { + panic(err) + } + randStr += string(alphabet[int(randByte[0])%len(alphabet)]) + } + return randStr +} + +// Reads credentials from ${TOR_PT_STATE_LOCATION}/config.txt, initializes blank values +func readCredentialsFromConfig() (*url.Userinfo, error) { + config, err := os.Open(path.Join(torPtStateLocationEnvVar, "config.txt")) + if err != nil { + config, err = os.Create(path.Join(torPtStateLocationEnvVar, "config.txt")) + if err != nil { + return nil, err + } + } + defer config.Close() + + var ptConfig map[string]string + ptConfig = make(map[string]string) + scanner := bufio.NewScanner(config) + for scanner.Scan() { + trimmedLine := strings.Trim(scanner.Text(), " ") + if trimmedLine == "" { + continue + } + line := strings.SplitN(trimmedLine, "=", 2) + if len(line) < 2 { + return nil, errors.New("Config line does not have '=': " + scanner.Text()) + } + ptConfig[strings.Trim(line[0], " ")] = strings.Trim(line[1], " ") + } + + if err := scanner.Err(); err != nil { + return nil, err + } + + if _, exists := ptConfig["username"]; !exists { + ptConfig["username"] = generateRandomString(6) + } + if _, exists := ptConfig["password"]; !exists { + ptConfig["password"] = generateRandomString(6) + } + + return url.UserPassword(ptConfig["username"], ptConfig["password"]), nil +} + +func saveCredentialsToConfig(creds *url.Userinfo) error { + configStr := fmt.Sprintf("%s=%s\n", "username", creds.Username()) + pw, _ := creds.Password() + configStr += fmt.Sprintf("%s=%s\n", "password", pw) + + return ioutil.WriteFile(path.Join(torPtStateLocationEnvVar, "config.txt"), []byte(configStr), 0700) +} + +// generates full https://user:pass@host:port URL using 'url=' argument(if given), +// then fills potential blanks with stored credentials and given serverName +func generatePTUrl(cliUrlPT url.URL, configCreds *url.Userinfo, serverName *string) (url.URL, error) { + ptUrl := cliUrlPT + switch ptUrl.Scheme { + case "": + ptUrl.Scheme = "https" + case "https": + default: + return ptUrl, errors.New("Unsupported scheme: " + ptUrl.Scheme) + } + + useCredsFromConfig := false + if ptUrl.User == nil { + useCredsFromConfig = true + } else { + if _, pwExists := ptUrl.User.Password(); ptUrl.User.Username() == "" && !pwExists { + useCredsFromConfig = true + } + } + if useCredsFromConfig { + ptUrl.User = configCreds + } + + port := ptUrl.Port() + if port == "" { + port = "443" + } + + hostname := ptUrl.Hostname() // first try hostname provided as cli arg, if any + if hostname == "" { + // then sni provided as cli arg + hostname = *serverName + } + if hostname == "" { + // lastly, try to get outbound IP by dialing https://diagnostic.opendns.com/myip + const errStr = "Could not automatically determine external ip using https://diagnostic.opendns.com/myip: %s. " + + "You can specify externally routable IP address in url=" + resp, err := http.Get("https://diagnostic.opendns.com/myip") + if err != nil { + return ptUrl, errors.New(fmt.Sprintf(errStr, err.Error())) + } + ipAddr, err := ioutil.ReadAll(resp.Body) + if err != nil { + return ptUrl, errors.New(fmt.Sprintf(errStr, err.Error())) + } + hostname = string(ipAddr) + if net.ParseIP(hostname) == nil { + return ptUrl, errors.New(fmt.Sprintf(errStr, "response: "+hostname)) + } + } + ptUrl.Host = net.JoinHostPort(hostname, port) + + return ptUrl, nil +} + +// If successful, returns domain name, parsed from cert (could be empty) and SPKI fingerprint. +// On error will os.Exit() +func validateAndParsePem(keyPath, certPath *string) (string, []byte) { + _, err := ioutil.ReadFile(*keyPath) + if err != nil { + quitWithSmethodError("Could not read" + *keyPath + ": " + err.Error()) + } + + certBytes, err := ioutil.ReadFile(*certPath) + if err != nil { + quitWithSmethodError("failed to read" + *certPath + ": " + err.Error()) + } + + var pemBlock *pem.Block + for { + // find last block + p, remainingCertBytes := pem.Decode([]byte(certBytes)) + if p == nil { + break + } + certBytes = remainingCertBytes + pemBlock = p + } + if pemBlock == nil { + quitWithSmethodError("failed to parse any blocks from " + *certPath) + } + + cert, err := x509.ParseCertificate(pemBlock.Bytes) + if err != nil { + quitWithSmethodError("failed to parse certificate from last block of" + + *certPath + ": " + err.Error()) + } + + cn := cert.Subject.CommonName + if strings.HasSuffix(cn, "*.") { + cn = cn[2:] + } + + h := sha256.New() + _, err = h.Write(cert.RawSubjectPublicKeyInfo) + if err != nil { + quitWithSmethodError("cert hashing error" + err.Error()) + } + spkiFP := h.Sum(nil) + + return cn, spkiFP +} + +// non-blocking +func startUsptreamingCaddy(upstream string, ptBridgeLineArgs pt.Args) { + if serverName == "" { + quitWithSmethodError("Set `-caddyname` argument in ServerTransportPlugin") + } + + caddyRoot := path.Join(torPtStateLocationEnvVar, "caddy_root") + err := os.MkdirAll(caddyRoot, 0700) + if err != nil { + quitWithSmethodError( + fmt.Sprintf("failed to read/create %s: %s\n", caddyRoot, err)) + } + if _, err := os.Stat(path.Join(caddyRoot, "index.html")); os.IsNotExist(err) { + log.Println("Please add/symlink web files (or at least index.html) to " + caddyRoot + + " to look like an actual website and stop serving 404 on /") + } + + extraDirectives := "" + if keyPemPath != "" && certPemPath != "" { + domainCN, spkiFp := validateAndParsePem(&keyPemPath, &certPemPath) + // We could potentially generate certs from Golang, but there's way too much stuff in x509 + // For fingerprintability reasons, might be better to advise use of openssl + serverName = domainCN + if _, alreadySetUsingCliArg := ptBridgeLineArgs.Get("sni"); domainCN != "" && net.ParseIP(domainCN) == nil && !alreadySetUsingCliArg { + ptBridgeLineArgs["sni"] = []string{domainCN} + } + + // TODO: if cert is already trusted: do not set proxyspki + ptBridgeLineArgs["proxyspki"] = []string{hex.EncodeToString(spkiFp)} + + extraDirectives += fmt.Sprintf("tls %s %s\n", certPemPath, keyPemPath) + } + + caddyHostname := serverName + if caddyHostname == "" { + caddyHostname = bridgeUrl.Hostname() + } + caddyPw, _ := bridgeUrl.User.Password() + caddyFile := fmt.Sprintf(`%s { + forwardproxy { + basicauth %s %s + probe_resistance + upstream %s + } + log / stdout "[{when}] "{method} {uri} {proto}" {status} {size}" + errors stdout + root %s + %s +} +`, caddyHostname, + bridgeUrl.User.Username(), caddyPw, + upstream, + caddyRoot, + extraDirectives) + + caddyInstance, err := caddy.Start(caddy.CaddyfileInput{ServerTypeName: "http", Contents: []byte(caddyFile)}) + if err != nil { + pt.ProxyError("failed to start caddy: " + err.Error()) + os.Exit(9) + } + go func() { + caddyInstance.Wait() // if caddy stopped -- exit + pt.ProxyError("Caddy has stopped. Exiting.") + sigChan <- syscall.SIGTERM + }() +} diff --git a/server/server_test.go b/server/server_test.go new file mode 100644 index 0000000..a60f4b4 --- /dev/null +++ b/server/server_test.go @@ -0,0 +1,113 @@ +// Copyright 2018 Jigsaw Operations LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "net/http" + "testing" +) + +func TestParseForwarded(t *testing.T) { + makeHeader := func(headers map[string]string) http.Header { + h := make(http.Header) + for k, v := range headers { + h.Add(k, v) + } + return h + } + + expectErr := func(headersMap map[string]string) { + header := makeHeader(headersMap) + h, err := parseForwardedTor(header) + if err == nil { + t.Fatalf("Expected: error, got: parsed %s\nheader was: %s\n", h, header) + } + } + + expectNoErr := func(headersMap map[string]string, expectedHostname string) { + header := makeHeader(headersMap) + h, err := parseForwardedTor(header) + if err != nil { + t.Fatalf("Expected: parsed %s, got: error %s\nheader was: %s\n", + expectedHostname, err, header) + } + + if h != expectedHostname { + t.Fatalf("Expected: %s, got: %s\nheader was: %s\n", + expectedHostname, h, header) + } + } + + // according to the rfc, many of those are valid, including 8.8.8.8 and bazinga:123, however + // tor spec requires that it is an IP address and has port + expectErr(map[string]string{ + "X-Forwarded-For": "bazinga", + }) + expectErr(map[string]string{ + "X-Forwarded-For": "bazinga:123", + }) + expectErr(map[string]string{ + "X-Forwarded-For": "8.8.8.8", + }) + expectErr(map[string]string{ + "Forwarded": "127.0.0.1", + }) + expectErr(map[string]string{ + "Forwarded": "127.0.0.1:22", + }) + expectErr(map[string]string{ + "Forwarded": "for=127.0.0.1", + }) + expectErr(map[string]string{ + "Forwarded": "for=you:123", + }) + expectErr(map[string]string{ + "Forwarded": "For=888.8.8.8:123", + }) + expectErr(map[string]string{ + "Forwarded": "for=[c:d:e:g:h:i]:5678", + }) + + expectNoErr(map[string]string{ + "Forwarded": "for=1.1.1.1:44444", + }, "1.1.1.1:44444") + expectNoErr(map[string]string{ + "x-ForwarDed-fOr": "8.8.8.8:123", + }, "8.8.8.8:123") + expectNoErr(map[string]string{ + "ForwarDed": "FoR=8.8.8.8:123", + }, "8.8.8.8:123") + expectNoErr(map[string]string{ + "ForwarDed": "FoR=[1:2::3:4]:5678", + }, "[1:2::3:4]:5678") + expectNoErr(map[string]string{ + "ForwarDed": "FoR=[fe80::1ff:fe23:4567:890a]:5678", + }, "[fe80::1ff:fe23:4567:890a]:5678") + expectNoErr(map[string]string{ + "ForwarDed": "FoR=[eeee:eeee:eeee:eeee:eeee:eeee:eeee:eeee]:5678", + }, "[eeee:eeee:eeee:eeee:eeee:eeee:eeee:eeee]:5678") + expectNoErr(map[string]string{ + "ForwarDed": "FoR=8.8.8.8:123;", + }, "8.8.8.8:123") + expectNoErr(map[string]string{ + "ForwarDed": "FoR=8.8.8.8:123; by=me", + }, "8.8.8.8:123") + expectNoErr(map[string]string{ + "ForwarDed": "proto=amazingProto; FoR=8.8.8.8:123; by=me", + }, "8.8.8.8:123") + expectNoErr(map[string]string{ + "ForwarDed": "proto=amazingProto;FoR = 8.8.8.8:123 ;by=me", + }, "8.8.8.8:123") +}