[tor-commits] [httpsproxy/master] Initial commit
dcf at torproject.org
dcf at torproject.org
Mon Aug 6 18:39:39 UTC 2018
commit 724098922e9a60b5c0654cbc923a90ed3c8386e6
Author: sergeyfrolov <sergey.frolov at colorado.edu>
Date: Tue Jul 17 17:24:48 2018 -0400
Initial commit
---
README.md | 97 ++++++++
client/client.go | 436 ++++++++++++++++++++++++++++++++++
server/inithack/hack.go | 41 ++++
server/server.go | 616 ++++++++++++++++++++++++++++++++++++++++++++++++
server/server_test.go | 113 +++++++++
5 files changed, 1303 insertions(+)
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..93f4b7c
--- /dev/null
+++ b/README.md
@@ -0,0 +1,97 @@
+# [httpsproxy](https://trac.torproject.org/projects/tor/ticket/26923)
+
+## Spin up a full Tor bridge
+This instruction explains how to install and start following components:
+* Caddy Web Server
+* Pluggable Transport
+* Tor daemon
+
+```bash
+sudo apt install tor
+
+# build server from source code
+git clone https://git.torproject.org/pluggable-transports/httpsproxy.git
+cd httpsproxy/server
+go get
+go build
+sudo cp server /var/lib/tor/httpsproxy
+
+# allow binding to ports 80 and 443
+sudo /sbin/setcap 'cap_net_bind_service=+ep' /var/lib/tor/httpsproxy
+sudo sed -i -e 's/NoNewPrivileges=yes/NoNewPrivileges=no/g' /lib/systemd/system/tor at default.service
+sudo sed -i -e 's/NoNewPrivileges=yes/NoNewPrivileges=no/g' /lib/systemd/system/tor at .service
+sudo systemctl daemon-reload
+
+# don't forget to set correct ContactInfo
+sudo cat <<EOT >> /etc/tor/torrc
+ RunAsDaemon 1
+ BridgeRelay 1
+ ExitRelay 0
+
+ PublishServerDescriptor 0 # 1 for public bridge
+
+ ORPort 9001
+ ExtORPort auto
+
+ ServerTransportPlugin httpsproxy exec /var/lib/tor/httpsproxy -servername yourdomain.com -agree -email youremail at gmail.com
+ Address 1.2.3.4 # might be required per https://trac.torproject.org/projects/tor/ticket/12020
+
+ ContactInfo Dr Stephen Falken steph at gtnw.org
+ Nickname joshua
+EOT
+
+sudo systemctl start tor
+
+# monitor logs:
+sudo less +F /var/log/tor/log
+sudo less +F /var/lib/tor/pt_state/caddy.log
+```
+
+### PT arguments
+As mentioned in code, `flag` package is global and PT arguments are passed together with those of Caddy.
+
+```
+
+Usage of ./server:
+ -runcaddy
+ Start Caddy web server on ports 443 and 80 (redirects to 443) together with the PT.
+ You can disable this option, set static 'ServerTransportListenAddr httpsproxy 127.0.0.1:ptPort' in torrc,
+ spin up frontend manually, and forward client's CONNECT request to 127.0.0.1:ptPort. (default true)
+ -servername string
+ Server Name used. Used as TLS SNI on the client side, and to start Caddy.
+ -agree
+ Agree to the CA's Subscriber Agreement
+ -email string
+ Default ACME CA account email address
+ -cert string
+ Path to TLS cert. Requires --key. If set, caddy will not get Lets Encrypt TLS certificate.
+ -key string
+ Path to TLS key. Requires --cert. If set, caddy will not get Lets Encrypt TLS certificate.
+ -logfile string
+ Log file for Pluggable Transport. (default: "$TOR_PT_STATE_LOCATION/caddy.log" -> /var/lib/tor/pt_state/caddy.log)
+ -url string
+ Set/override access url in form of https://username:password@1.2.3.4:443/.
+ If servername is set or cert argument has a certificate with correct domain name,
+ this arg is optional and will be inferred, username:password will be auto-generated and stored, if not provided.
+```
+
+## Configure client
+
+Ideally, this will be integrated with the Tor browser and distributed automatically, so clients would have to do nothing
+In the meantime, here's how to test it with Tor Browser Bundle:
+
+1. Download [Tor Browser](https://www.torproject.org/projects/torbrowser.html.en)
+2. Build httpsclient and configure torrc:
+```
+ git clone https://git.torproject.org/pluggable-transports/httpsproxy.git
+ cd httpsproxy/client
+ go get
+ go build
+ PATH_TO_CLIENT=`pwd`
+ PATH_TO_TORRC="/etc/tor/torrc" # if TBB is used, path will be different
+ echo "ClientTransportPlugin httpsproxy exec ${PATH_TO_CLIENT}/client" >> $PATH_TO_TORRC
+```
+4. Launch Tor Browser, select "Tor is censored in my country" -> "Provide a bridge I know"
+5. Copy bridge line like "httpsproxy 0.4.2.0:3 url=https://username:password@httpsproxy.com".
+ If you set up your own server, bridge line will be printed to caddy.log on server launch.
+
diff --git a/client/client.go b/client/client.go
new file mode 100644
index 0000000..eea871d
--- /dev/null
+++ b/client/client.go
@@ -0,0 +1,436 @@
+// Copyright 2018 Jigsaw Operations LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// HTTPS proxy based pluggable transport client.
+package main
+
+import (
+ "bufio"
+ "crypto/tls"
+ "encoding/base64"
+ "errors"
+ "flag"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "log"
+ "net"
+ "net/http"
+ "net/url"
+ "os"
+ "os/signal"
+ "strconv"
+ "sync"
+ "syscall"
+
+ pt "git.torproject.org/pluggable-transports/goptlib.git"
+ "golang.org/x/net/http2"
+)
+
+var ptInfo pt.ClientInfo
+
+// When a connection handler starts, +1 is written to this channel; when it
+// ends, -1 is written.
+var handlerChan = make(chan int)
+
+// TODO: stop goroutine leaking in copyLoops: if one side closes - close another after timeout
+
+// This function is copypasted from https://github.com/caddyserver/forwardproxy/blob/master/forwardproxy.go
+// TODO: replace with padding-enabled function
+// flushingIoCopy is analogous to buffering io.Copy(), but also attempts to flush on each iteration.
+// If dst does not implement http.Flusher(e.g. net.TCPConn), it will do a simple io.CopyBuffer().
+// Reasoning: http2ResponseWriter will not flush on its own, so we have to do it manually.
+func flushingIoCopy(dst io.Writer, src io.Reader, buf []byte) (written int64, err error) {
+ flusher, ok := dst.(http.Flusher)
+ if !ok {
+ return io.CopyBuffer(dst, src, buf)
+ }
+ for {
+ nr, er := src.Read(buf)
+ if nr > 0 {
+ nw, ew := dst.Write(buf[0:nr])
+ flusher.Flush()
+ if nw > 0 {
+ written += int64(nw)
+ }
+ if ew != nil {
+ err = ew
+ break
+ }
+ if nr != nw {
+ err = io.ErrShortWrite
+ break
+ }
+ }
+ if er != nil {
+ if er != io.EOF {
+ err = er
+ }
+ break
+ }
+ }
+ return
+}
+
+// simple copy loop without padding, works with http/1.1
+// TODO: we can't pad, but we probably can split
+func copyLoop(local, remote net.Conn) {
+ var wg sync.WaitGroup
+ wg.Add(2)
+
+ go func() {
+ io.Copy(remote, local)
+ wg.Done()
+ }()
+ go func() {
+ io.Copy(local, remote)
+ wg.Done()
+ }()
+ // TODO: try not to spawn extra goroutine
+
+ wg.Wait()
+}
+
+func h2copyLoop(w1 io.Writer, r1 io.Reader, w2 io.Writer, r2 io.Reader) {
+ var wg sync.WaitGroup
+ wg.Add(2)
+
+ buf1 := make([]byte, 16384)
+ buf2 := make([]byte, 16384)
+ go func() {
+ flushingIoCopy(w1, r1, buf1)
+ wg.Done()
+ }()
+ go func() {
+ flushingIoCopy(w2, r2, buf2)
+ wg.Done()
+ }()
+ // TODO: try not to spawn extra goroutine
+
+ wg.Wait()
+}
+
+func parseTCPAddr(s string) (*net.TCPAddr, error) {
+ hostStr, portStr, err := net.SplitHostPort(s)
+ if err != nil {
+ fmt.Printf("net.SplitHostPort(%s) failed: %+v", s, err)
+ return nil, err
+ }
+
+ port, err := strconv.Atoi(portStr)
+ if err != nil {
+ fmt.Printf("strconv.Atoi(%s) failed: %+v", portStr, err)
+ return nil, err
+ }
+
+ ip := net.ParseIP(hostStr)
+ if ip == nil {
+ err = errors.New("net.ParseIP(" + s + ") returned nil")
+ fmt.Printf("%+v\n", err)
+ return nil, err
+ }
+
+ return &net.TCPAddr{Port: port, IP: ip}, nil
+}
+
+// handler will process a PT request, requests webproxy(that is given in URL arg) to connect to
+// the Req.Target and relay traffic between client and webproxy
+func handler(conn *pt.SocksConn) error {
+ handlerChan <- 1
+ defer func() {
+ handlerChan <- -1
+ }()
+ defer conn.Close()
+
+ guardTCPAddr, err := parseTCPAddr(conn.Req.Target)
+ if err != nil {
+ conn.Reject()
+ return err
+ }
+
+ webproxyUrlArg, ok := conn.Req.Args.Get("url")
+ if !ok {
+ err := errors.New("address of webproxy in form of `url=https://username:password@example.com` is required")
+ conn.Reject()
+ return err
+ }
+
+ httpsClient, err := NewHTTPSClient(webproxyUrlArg)
+ if err != nil {
+ log.Printf("NewHTTPSClient(%s, nil) failed: %s\n", webproxyUrlArg, err)
+ conn.Reject()
+ return err
+ }
+
+ err = httpsClient.Connect(conn.Req.Target)
+ if err != nil {
+ log.Printf("httpsClient.Connect(%s, nil) failed: %s\n", conn.Req.Target, err)
+ conn.Reject()
+ return err
+ }
+
+ err = conn.Grant(guardTCPAddr)
+ if err != nil {
+ log.Printf("conn.Grant(%s) failed: %s\n", guardTCPAddr, err)
+ conn.Reject()
+ return err
+ }
+
+ return httpsClient.CopyLoop(conn)
+}
+
+func acceptLoop(ln *pt.SocksListener) error {
+ defer ln.Close()
+ for {
+ conn, err := ln.AcceptSocks()
+ if err != nil {
+ if e, ok := err.(net.Error); ok && e.Temporary() {
+ continue
+ }
+ return err
+ }
+ go handler(conn)
+ }
+}
+
+func main() {
+ var err error
+
+ logFile := flag.String("log", "", "Log file for debugging")
+ flag.Parse()
+
+ ptInfo, err = pt.ClientSetup(nil)
+ if err != nil {
+ os.Exit(1)
+ }
+
+ if ptInfo.ProxyURL != nil {
+ pt.ProxyError("proxy is not supported")
+ os.Exit(1)
+ }
+
+ if *logFile != "" {
+ f, err := os.OpenFile(*logFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0660)
+ if err != nil {
+ pt.CmethodError("httpsproxy",
+ fmt.Sprintf("error opening file %s: %v", logFile, err))
+ os.Exit(2)
+ }
+ defer f.Close()
+ log.SetOutput(f)
+ }
+
+ listeners := make([]net.Listener, 0)
+ for _, methodName := range ptInfo.MethodNames {
+ switch methodName {
+ case "httpsproxy":
+ ln, err := pt.ListenSocks("tcp", "127.0.0.1:0")
+ if err != nil {
+ pt.CmethodError(methodName, err.Error())
+ break
+ }
+ go acceptLoop(ln)
+ pt.Cmethod(methodName, ln.Version(), ln.Addr())
+ log.Printf("Started %s %s at %s\n", methodName, ln.Version(), ln.Addr())
+ listeners = append(listeners, ln)
+ default:
+ pt.CmethodError(methodName, "no such method")
+ }
+ }
+ pt.CmethodsDone()
+
+ var numHandlers = 0
+ var sig os.Signal
+ sigChan := make(chan os.Signal, 1)
+ signal.Notify(sigChan, syscall.SIGTERM)
+
+ if os.Getenv("TOR_PT_EXIT_ON_STDIN_CLOSE") == "1" {
+ // This environment variable means we should treat EOF on stdin
+ // just like SIGTERM: https://bugs.torproject.org/15435.
+ go func() {
+ io.Copy(ioutil.Discard, os.Stdin)
+ sigChan <- syscall.SIGTERM
+ }()
+ }
+
+ // keep track of handlers and wait for a signal
+ sig = nil
+ for sig == nil {
+ select {
+ case n := <-handlerChan:
+ numHandlers += n
+ case sig = <-sigChan:
+ }
+ }
+
+ // signal received, shut down
+ for _, ln := range listeners {
+ ln.Close()
+ }
+ for numHandlers > 0 {
+ numHandlers += <-handlerChan
+ }
+}
+
+type HTTPConnectClient struct {
+ Header http.Header
+ ProxyHost string
+ TlsConf tls.Config
+
+ Conn *tls.Conn
+
+ In io.Writer
+ Out io.Reader
+}
+
+// NewHTTPSClient creates one-time use client to tunnel traffic via HTTPS proxy.
+// If spkiFp is set, HTTPSClient will use it as SPKI fingerprint to confirm identity of the
+// proxy, instead of relying on standard PKI CA roots
+func NewHTTPSClient(proxyUrlStr string) (*HTTPConnectClient, error) {
+ proxyUrl, err := url.Parse(proxyUrlStr)
+ if err != nil {
+ return nil, err
+ }
+
+ switch proxyUrl.Scheme {
+ case "http", "":
+ fallthrough
+ default:
+ return nil, errors.New("Scheme " + proxyUrl.Scheme + " is not supported")
+ case "https":
+ }
+
+ if proxyUrl.Host == "" {
+ return nil, errors.New("misparsed `url=`, make sure to specify full url like https://username:password@hostname.com:443/")
+ }
+
+ if proxyUrl.Port() == "" {
+ proxyUrl.Host = net.JoinHostPort(proxyUrl.Host, "443")
+ }
+
+ tlsConf := tls.Config{
+ NextProtos: []string{"h2", "http/1.1"},
+ ServerName: proxyUrl.Hostname(),
+ }
+
+ client := &HTTPConnectClient{
+ Header: make(http.Header),
+ ProxyHost: proxyUrl.Host,
+ TlsConf: tlsConf,
+ }
+
+ if proxyUrl.User.Username() != "" {
+ password, _ := proxyUrl.User.Password()
+ client.Header.Set("Proxy-Authorization", "Basic "+
+ base64.StdEncoding.EncodeToString([]byte(proxyUrl.User.Username()+":"+password)))
+ }
+ return client, nil
+}
+
+func (c *HTTPConnectClient) Connect(target string) error {
+ req := &http.Request{
+ Method: "CONNECT",
+ URL: &url.URL{Host: target},
+ Header: c.Header,
+ Host: target,
+ }
+
+ tcpConn, err := net.Dial("tcp", c.ProxyHost)
+ if err != nil {
+ return err
+ }
+
+ c.Conn = tls.Client(tcpConn, &c.TlsConf)
+
+ err = c.Conn.Handshake()
+ if err != nil {
+ return err
+ }
+
+ var resp *http.Response
+ switch c.Conn.ConnectionState().NegotiatedProtocol {
+ case "":
+ fallthrough
+ case "http/1.1":
+ req.Proto = "HTTP/1.1"
+ req.ProtoMajor = 1
+ req.ProtoMinor = 1
+
+ err = req.Write(c.Conn)
+ if err != nil {
+ c.Conn.Close()
+ return err
+ }
+
+ resp, err = http.ReadResponse(bufio.NewReader(c.Conn), req)
+ if err != nil {
+ c.Conn.Close()
+ return err
+ }
+
+ c.In = c.Conn
+ c.Out = c.Conn
+ case "h2":
+ req.Proto = "HTTP/2.0"
+ req.ProtoMajor = 2
+ req.ProtoMinor = 0
+ pr, pw := io.Pipe()
+ req.Body = ioutil.NopCloser(pr)
+
+ t := http2.Transport{}
+ h2client, err := t.NewClientConn(c.Conn)
+ if err != nil {
+ c.Conn.Close()
+ return err
+ }
+
+ resp, err = h2client.RoundTrip(req)
+ if err != nil {
+ c.Conn.Close()
+ return err
+ }
+
+ c.In = pw
+ c.Out = resp.Body
+ default:
+ c.Conn.Close()
+ return errors.New("negotiated unsupported application layer protocol: " +
+ c.Conn.ConnectionState().NegotiatedProtocol)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ c.Conn.Close()
+ return errors.New("Proxy responded with non 200 code: " + resp.Status)
+ }
+
+ return nil
+}
+
+func (c *HTTPConnectClient) CopyLoop(conn net.Conn) error {
+ defer c.Conn.Close()
+ defer conn.Close()
+
+ switch c.Conn.ConnectionState().NegotiatedProtocol {
+ case "":
+ fallthrough
+ case "http/1.1":
+ copyLoop(conn, c.Conn)
+ case "h2":
+ h2copyLoop(c.In, conn, conn, c.Out)
+ default:
+ return errors.New("negotiated unsupported application layer protocol: " +
+ c.Conn.ConnectionState().NegotiatedProtocol)
+ }
+ return nil
+}
diff --git a/server/inithack/hack.go b/server/inithack/hack.go
new file mode 100644
index 0000000..b3801ca
--- /dev/null
+++ b/server/inithack/hack.go
@@ -0,0 +1,41 @@
+// Copyright 2018 Jigsaw Operations LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package inithack
+
+import (
+ "os"
+ "path"
+)
+
+// We need to override CADDYPATH and make sure caddy writes things into TOR_PT_STATE_LOCATION
+// as opposed to HOME (which will be something like "/root/" and hardening features rightfully
+// prevent PT from writing there). Unfortunately, Caddy uses CADDYPATH in init() functions, which
+// run before than anything in "server" package.
+//
+// For now, which just set CADDYPATH=TOR_PT_STATE_LOCATION here and import it before caddy.
+// https://golang.org/ref/spec#Package_initialization does not guarantee a particular init order,
+// which is why we should find an actual fix. TODO!
+//
+// Potential fixes:
+// 1) refactor Caddy: seems like a big patch
+// 2) Set CADDYHOME environment variable from Tor: torrc doesn't seem to allow setting arbitrary env vars
+// 3) Change Tor behavior to set HOME to TOR_PT_STATE_LOCATION?
+// 4) govendor Caddy, and change its source code to import this package, guaranteeing init order
+// 5) run Caddy as a separate binary.
+func init() {
+ if os.Getenv("CADDYPATH") == "" {
+ os.Setenv("CADDYPATH", path.Join(os.Getenv("TOR_PT_STATE_LOCATION"), ".caddy"))
+ }
+}
diff --git a/server/server.go b/server/server.go
new file mode 100644
index 0000000..7f86319
--- /dev/null
+++ b/server/server.go
@@ -0,0 +1,616 @@
+// Copyright 2018 Jigsaw Operations LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "bufio"
+ "crypto/rand"
+ "crypto/sha256"
+ "crypto/x509"
+ "encoding/hex"
+ "encoding/pem"
+ "errors"
+ "flag"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "log"
+ "net"
+ "net/http"
+ "net/url"
+ "os"
+ "os/signal"
+ "path"
+ "runtime/debug"
+ "strings"
+ "sync"
+ "syscall"
+
+ _ "github.com/Jigsaw-Code/volunteer/server/inithack"
+
+ pt "git.torproject.org/pluggable-transports/goptlib.git"
+ "github.com/mholt/caddy"
+
+ // imports below are to run init() and register the forwardproxy plugin, set default variables
+ _ "github.com/caddyserver/forwardproxy"
+ _ "github.com/mholt/caddy/caddy/caddymain"
+)
+
+// TODO: stop goroutine leaking in copyLoops
+
+var ptInfo pt.ServerInfo
+
+// When a connection handler starts, +1 is written to this channel; when it
+// ends, -1 is written.
+var handlerChan = make(chan int)
+
+func copyLoop(a, b net.Conn) {
+ var wg sync.WaitGroup
+ wg.Add(2)
+
+ go func() {
+ io.Copy(b, a)
+ wg.Done()
+ }()
+ go func() {
+ io.Copy(a, b)
+ wg.Done()
+ }()
+ wg.Wait()
+}
+
+// Parses Forwarded and X-Forwarded-For headers, and returns client's IP:port.
+// According to RFC, hostnames and addresses without port are valid, but Tor spec mandates IP:port,
+// so those are currently return error.
+// Returns "", nil if there are no headers indicating forwarding.
+// Returns "", err if there are forwarding headers, but they are misformatted/don't contain IP:port
+// Returns "IPAddr", nil on successful parse.
+func parseForwardedTor(header http.Header) (string, error) {
+ var ipAddr string
+ proxyEvidence := false
+ xFFHeader := header.Get("X-Forwarded-For")
+ if xFFHeader != "" {
+ proxyEvidence = true
+ for _, ip := range strings.Split(xFFHeader, ",") {
+ ipAddr = strings.Trim(ip, " \"")
+ break
+ }
+ }
+ forwardedHeader := header.Get("Forwarded")
+ if forwardedHeader != "" {
+ proxyEvidence = true
+ for _, fValue := range strings.Split(forwardedHeader, ";") {
+ s := strings.Split(fValue, "=")
+ if len(s) != 2 {
+ return "", errors.New("misformatted \"Forwarded:\" header")
+ }
+ if strings.ToLower(strings.Trim(s[0], " ")) == "for" {
+ ipAddr = strings.Trim(s[1], " \"")
+ break
+ }
+ }
+ }
+ if ipAddr == "" {
+ if proxyEvidence == true {
+ return "", errors.New("Forwarded or X-Forwarded-For header is present, but could not be parsed")
+ }
+ return "", nil
+ }
+
+ // According to https://github.com/torproject/torspec/blob/master/proposals/196-transport-control-ports.txt
+ // there are 2 acceptable formats:
+ // 1.2.3.4:5678
+ // [1:2::3:4]:5678 // (spec says [1:2::3:4]::5678 but that must be a typo)
+ h, p, err := net.SplitHostPort(ipAddr)
+ if err != nil {
+ return "", err
+ }
+ if net.ParseIP(h) == nil {
+ return "", errors.New(h + " is not a valid IP address")
+ }
+ return net.JoinHostPort(h, p), nil
+}
+
+func handler(conn net.Conn) error {
+ defer conn.Close()
+
+ handlerChan <- 1
+ defer func() {
+ handlerChan <- -1
+ }()
+ var err error
+
+ req, err := http.ReadRequest(bufio.NewReader(conn))
+ if err != nil {
+ return err
+ }
+
+ clientIP, err := parseForwardedTor(req.Header)
+ if err != nil {
+ // just print the error to log. eventually, we may decide to reject connections,
+ // if Forwarded/X-Forwarded-For header is present, but misformatted/misparsed
+ log.Println(err)
+ }
+ if clientIP == "" {
+ // if err != nil, conn.RemoteAddr() is certainly not the right IP
+ // but testing showed that connection fails to establish if clientIP is empty
+ clientIP = conn.RemoteAddr().String()
+ }
+
+ or, err := pt.DialOr(&ptInfo, clientIP, "httpsproxy")
+ if err != nil {
+ return err
+ }
+ defer or.Close()
+
+ // TODO: consider adding support for HTTP/2, HAPROXY-style PROXY protocol, SOCKS, etc.
+ _, err = conn.Write([]byte("HTTP/1.1 200 OK\r\n\r\n"))
+ if err != nil {
+ return err
+ }
+
+ copyLoop(conn, or)
+
+ return nil
+}
+
+func acceptLoop(ln net.Listener) error {
+ defer ln.Close()
+ for {
+ conn, err := ln.Accept()
+ if err != nil {
+ if e, ok := err.(net.Error); ok && e.Temporary() {
+ continue
+ }
+ return err
+ }
+ go handler(conn)
+ }
+}
+
+var (
+ torPtStateLocationEnvVar string // directory where PT is allowed to store things
+
+ bridgeUrl url.URL // bridge URL to register with bridgeDB
+
+ // cli args
+ runCaddy bool
+ serverName string
+ keyPemPath, certPemPath string
+ cliUrlPTstr string
+ logFile string
+)
+
+func parseValidateCliArgs() error {
+ // flag package is global and arguments get inevitably mixed with those of Caddy
+ // It's a bit messy, but allows us to easily pass arguments to Caddy
+ // To cleanup, we would have to reimplement argument parsing (or use 3rd party flag package)
+ flag.BoolVar(&runCaddy, "runcaddy", true, "Start Caddy web server on ports 443 and 80 (redirects to 443) together with the PT."+
+ " You can disable this option, set static 'ServerTransportListenAddr httpsproxy 127.0.0.1:ptPort' in torrc,"+
+ " spin up frontend manually, and forward client's CONNECT request to 127.0.0.1:ptPort.")
+ flag.StringVar(&serverName, "servername", "", "Server Name used. Used as TLS SNI on the client side, and to start Caddy.")
+
+ flag.StringVar(&keyPemPath, "key", "", "Path to TLS key. Requires --cert. If set, caddy will not get Lets Encrypt TLS certificate.")
+ flag.StringVar(&certPemPath, "cert", "", "Path to TLS cert. Requires --key. If set, caddy will not get Lets Encrypt TLS certificate.")
+
+ flag.StringVar(&cliUrlPTstr, "url", "", "Set/override access url in form of https://username:password@1.2.3.4:443/."+
+ " If servername is set or cert argument has a certificate with correct domain name,"+
+ " this arg is optional and will be inferred, username:password will be auto-generated and stored, if not provided.")
+
+ flag.StringVar(&logFile, "logfile", path.Join(torPtStateLocationEnvVar, "caddy.log"),
+ "Log file for Pluggable Transport.")
+ flag.Parse()
+
+ if (keyPemPath == "" && certPemPath != "") || (keyPemPath != "" && certPemPath == "") {
+ return errors.New("--cert and --key options must be used together")
+ }
+
+ if runCaddy == true && (serverName == "" && keyPemPath == "" && cliUrlPTstr == "") {
+ return errors.New("for automatic launch of Caddy web server(`runcaddy=true` by default)," +
+ "please specify either --servername, --url, or --cert and --key")
+ }
+
+ var err error
+ cliUrlPT := &url.URL{}
+ if cliUrlPTstr != "" {
+ cliUrlPT, err = url.Parse(cliUrlPTstr)
+ if err != nil {
+ return err
+ }
+ }
+
+ var storedCredentials *url.Userinfo
+ if cliUrlPT.User.Username() == "" && runCaddy == true {
+ // if operator hasn't specified the credentials in url and requests to start caddy,
+ // use credentials, stored to disk
+ storedCredentials, err = readCredentialsFromConfig()
+ if err != nil {
+ quitWithSmethodError(err.Error())
+ }
+ err := saveCredentialsToConfig(storedCredentials)
+ if err != nil {
+ // if can't save credentials persistently, and they were NOT provided as cli, die
+ quitWithSmethodError(
+ fmt.Sprintf("failed to save auto-generated proxy credentials: %s."+
+ "Fix the error or specify credentials in `url=` argument", err))
+ }
+ }
+
+ bridgeUrl, err = generatePTUrl(*cliUrlPT, storedCredentials, &serverName)
+ return err
+}
+
+func quitWithSmethodError(errStr string) {
+ pt.SmethodError("httpsproxy", errStr)
+ os.Exit(2)
+}
+
+var sigChan chan os.Signal
+
+func main() {
+ defer func() {
+ if r := recover(); r != nil {
+ log.Printf("panic: %s\nstack trace: %s\n", r, debug.Stack())
+ pt.ProxyError(fmt.Sprintf("panic: %v. (check PT log for detailed trace)", r))
+ }
+ }()
+
+ torPtStateLocationEnvVar = os.Getenv("TOR_PT_STATE_LOCATION")
+ if torPtStateLocationEnvVar == "" {
+ quitWithSmethodError("Set torPtStateLocationEnvVar")
+ }
+ err := os.MkdirAll(torPtStateLocationEnvVar, 0700)
+ if err != nil {
+ quitWithSmethodError(fmt.Sprintf("Failed to open/create %s: %s", torPtStateLocationEnvVar, err))
+ }
+
+ if err := parseValidateCliArgs(); err != nil {
+ quitWithSmethodError("failed to parse PT arguments: " + err.Error())
+ }
+
+ if logFile != "" {
+ f, err := os.OpenFile(logFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0660)
+ if err != nil {
+ quitWithSmethodError(fmt.Sprintf("error opening file %s: %v", logFile, err))
+ }
+ defer f.Close()
+ log.SetOutput(f)
+ os.Stdout = f
+ os.Stderr = f
+ }
+
+ ptInfo, err = pt.ServerSetup(nil)
+ if err != nil {
+ quitWithSmethodError(err.Error())
+ }
+
+ var ptAddr net.Addr
+ if len(ptInfo.Bindaddrs) != 1 {
+ // TODO: is it even useful to have multiple bindaddrs and how would we use them? We don't
+ // want to accept direct connections to PT, as it doesn't use security protocols like TLS
+ quitWithSmethodError("only one bind address is supported")
+ }
+ bindaddr := ptInfo.Bindaddrs[0]
+ if bindaddr.MethodName != "httpsproxy" {
+ quitWithSmethodError("no such method")
+ }
+
+ listener, err := net.ListenTCP("tcp", bindaddr.Addr)
+ if err != nil {
+ quitWithSmethodError(err.Error())
+ }
+ ptAddr = listener.Addr()
+ colonIdx := strings.LastIndex(ptAddr.String(), ":")
+ if colonIdx == -1 || len(ptAddr.String()) == colonIdx+1 {
+ quitWithSmethodError("Bindaddr " + ptAddr.String() + " does not contain port")
+ }
+ ptAddrPort := ptAddr.String()[colonIdx+1:]
+
+ go acceptLoop(listener)
+
+ ptBridgeLineArgs := make(pt.Args)
+ if serverName != "" {
+ ptBridgeLineArgs["sni"] = []string{serverName}
+ }
+
+ var numHandlers int = 0
+ var sig os.Signal
+
+ sigChan = make(chan os.Signal, 1)
+ signal.Notify(sigChan, syscall.SIGTERM)
+
+ if runCaddy {
+ startUsptreamingCaddy("http://localhost:"+ptAddrPort, ptBridgeLineArgs)
+ }
+
+ ptBridgeLineArgs["proxy"] = []string{bridgeUrl.String()}
+
+ // print bridge line
+ argsAsString := func(args *pt.Args) string {
+ str := ""
+ for k, v := range *args {
+ str += k + "=" + strings.Join(v, ",") + " "
+ }
+ return strings.Trim(str, " ")
+ }
+ log.Printf("Bridge line: %s %s [fingerprint] %s\n",
+ bindaddr.MethodName, listener.Addr(), argsAsString(&ptBridgeLineArgs))
+
+ // register bridge line
+ pt.SmethodArgs(bindaddr.MethodName, listener.Addr(), ptBridgeLineArgs)
+ pt.SmethodsDone()
+
+ if os.Getenv("TOR_PT_EXIT_ON_STDIN_CLOSE") == "1" {
+ // This environment variable means we should treat EOF on stdin
+ // just like SIGTERM: https://bugs.torproject.org/15435.
+ go func() {
+ io.Copy(ioutil.Discard, os.Stdin)
+ sigChan <- syscall.SIGTERM
+ }()
+ }
+
+ // keep track of handlers and wait for a signal
+ sig = nil
+ for sig == nil {
+ select {
+ case n := <-handlerChan:
+ numHandlers += n
+ case sig = <-sigChan:
+ log.Println("Got EOF on stdin, exiting")
+ }
+ }
+
+ // signal received, shut down
+ listener.Close()
+
+ for numHandlers > 0 {
+ numHandlers += <-handlerChan
+ }
+}
+
+func generateRandomString(length int) string {
+ const alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+
+ randByte := make([]byte, 1)
+
+ var randStr string
+ for i := 0; i < length; i++ {
+ _, err := rand.Read(randByte)
+ if err != nil {
+ panic(err)
+ }
+ randStr += string(alphabet[int(randByte[0])%len(alphabet)])
+ }
+ return randStr
+}
+
+// Reads credentials from ${TOR_PT_STATE_LOCATION}/config.txt, initializes blank values
+func readCredentialsFromConfig() (*url.Userinfo, error) {
+ config, err := os.Open(path.Join(torPtStateLocationEnvVar, "config.txt"))
+ if err != nil {
+ config, err = os.Create(path.Join(torPtStateLocationEnvVar, "config.txt"))
+ if err != nil {
+ return nil, err
+ }
+ }
+ defer config.Close()
+
+ var ptConfig map[string]string
+ ptConfig = make(map[string]string)
+ scanner := bufio.NewScanner(config)
+ for scanner.Scan() {
+ trimmedLine := strings.Trim(scanner.Text(), " ")
+ if trimmedLine == "" {
+ continue
+ }
+ line := strings.SplitN(trimmedLine, "=", 2)
+ if len(line) < 2 {
+ return nil, errors.New("Config line does not have '=': " + scanner.Text())
+ }
+ ptConfig[strings.Trim(line[0], " ")] = strings.Trim(line[1], " ")
+ }
+
+ if err := scanner.Err(); err != nil {
+ return nil, err
+ }
+
+ if _, exists := ptConfig["username"]; !exists {
+ ptConfig["username"] = generateRandomString(6)
+ }
+ if _, exists := ptConfig["password"]; !exists {
+ ptConfig["password"] = generateRandomString(6)
+ }
+
+ return url.UserPassword(ptConfig["username"], ptConfig["password"]), nil
+}
+
+func saveCredentialsToConfig(creds *url.Userinfo) error {
+ configStr := fmt.Sprintf("%s=%s\n", "username", creds.Username())
+ pw, _ := creds.Password()
+ configStr += fmt.Sprintf("%s=%s\n", "password", pw)
+
+ return ioutil.WriteFile(path.Join(torPtStateLocationEnvVar, "config.txt"), []byte(configStr), 0700)
+}
+
+// generates full https://user:pass@host:port URL using 'url=' argument(if given),
+// then fills potential blanks with stored credentials and given serverName
+func generatePTUrl(cliUrlPT url.URL, configCreds *url.Userinfo, serverName *string) (url.URL, error) {
+ ptUrl := cliUrlPT
+ switch ptUrl.Scheme {
+ case "":
+ ptUrl.Scheme = "https"
+ case "https":
+ default:
+ return ptUrl, errors.New("Unsupported scheme: " + ptUrl.Scheme)
+ }
+
+ useCredsFromConfig := false
+ if ptUrl.User == nil {
+ useCredsFromConfig = true
+ } else {
+ if _, pwExists := ptUrl.User.Password(); ptUrl.User.Username() == "" && !pwExists {
+ useCredsFromConfig = true
+ }
+ }
+ if useCredsFromConfig {
+ ptUrl.User = configCreds
+ }
+
+ port := ptUrl.Port()
+ if port == "" {
+ port = "443"
+ }
+
+ hostname := ptUrl.Hostname() // first try hostname provided as cli arg, if any
+ if hostname == "" {
+ // then sni provided as cli arg
+ hostname = *serverName
+ }
+ if hostname == "" {
+ // lastly, try to get outbound IP by dialing https://diagnostic.opendns.com/myip
+ const errStr = "Could not automatically determine external ip using https://diagnostic.opendns.com/myip: %s. " +
+ "You can specify externally routable IP address in url="
+ resp, err := http.Get("https://diagnostic.opendns.com/myip")
+ if err != nil {
+ return ptUrl, errors.New(fmt.Sprintf(errStr, err.Error()))
+ }
+ ipAddr, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ return ptUrl, errors.New(fmt.Sprintf(errStr, err.Error()))
+ }
+ hostname = string(ipAddr)
+ if net.ParseIP(hostname) == nil {
+ return ptUrl, errors.New(fmt.Sprintf(errStr, "response: "+hostname))
+ }
+ }
+ ptUrl.Host = net.JoinHostPort(hostname, port)
+
+ return ptUrl, nil
+}
+
+// If successful, returns domain name, parsed from cert (could be empty) and SPKI fingerprint.
+// On error will os.Exit()
+func validateAndParsePem(keyPath, certPath *string) (string, []byte) {
+ _, err := ioutil.ReadFile(*keyPath)
+ if err != nil {
+ quitWithSmethodError("Could not read" + *keyPath + ": " + err.Error())
+ }
+
+ certBytes, err := ioutil.ReadFile(*certPath)
+ if err != nil {
+ quitWithSmethodError("failed to read" + *certPath + ": " + err.Error())
+ }
+
+ var pemBlock *pem.Block
+ for {
+ // find last block
+ p, remainingCertBytes := pem.Decode([]byte(certBytes))
+ if p == nil {
+ break
+ }
+ certBytes = remainingCertBytes
+ pemBlock = p
+ }
+ if pemBlock == nil {
+ quitWithSmethodError("failed to parse any blocks from " + *certPath)
+ }
+
+ cert, err := x509.ParseCertificate(pemBlock.Bytes)
+ if err != nil {
+ quitWithSmethodError("failed to parse certificate from last block of" +
+ *certPath + ": " + err.Error())
+ }
+
+ cn := cert.Subject.CommonName
+ if strings.HasSuffix(cn, "*.") {
+ cn = cn[2:]
+ }
+
+ h := sha256.New()
+ _, err = h.Write(cert.RawSubjectPublicKeyInfo)
+ if err != nil {
+ quitWithSmethodError("cert hashing error" + err.Error())
+ }
+ spkiFP := h.Sum(nil)
+
+ return cn, spkiFP
+}
+
+// non-blocking
+func startUsptreamingCaddy(upstream string, ptBridgeLineArgs pt.Args) {
+ if serverName == "" {
+ quitWithSmethodError("Set `-caddyname` argument in ServerTransportPlugin")
+ }
+
+ caddyRoot := path.Join(torPtStateLocationEnvVar, "caddy_root")
+ err := os.MkdirAll(caddyRoot, 0700)
+ if err != nil {
+ quitWithSmethodError(
+ fmt.Sprintf("failed to read/create %s: %s\n", caddyRoot, err))
+ }
+ if _, err := os.Stat(path.Join(caddyRoot, "index.html")); os.IsNotExist(err) {
+ log.Println("Please add/symlink web files (or at least index.html) to " + caddyRoot +
+ " to look like an actual website and stop serving 404 on /")
+ }
+
+ extraDirectives := ""
+ if keyPemPath != "" && certPemPath != "" {
+ domainCN, spkiFp := validateAndParsePem(&keyPemPath, &certPemPath)
+ // We could potentially generate certs from Golang, but there's way too much stuff in x509
+ // For fingerprintability reasons, might be better to advise use of openssl
+ serverName = domainCN
+ if _, alreadySetUsingCliArg := ptBridgeLineArgs.Get("sni"); domainCN != "" && net.ParseIP(domainCN) == nil && !alreadySetUsingCliArg {
+ ptBridgeLineArgs["sni"] = []string{domainCN}
+ }
+
+ // TODO: if cert is already trusted: do not set proxyspki
+ ptBridgeLineArgs["proxyspki"] = []string{hex.EncodeToString(spkiFp)}
+
+ extraDirectives += fmt.Sprintf("tls %s %s\n", certPemPath, keyPemPath)
+ }
+
+ caddyHostname := serverName
+ if caddyHostname == "" {
+ caddyHostname = bridgeUrl.Hostname()
+ }
+ caddyPw, _ := bridgeUrl.User.Password()
+ caddyFile := fmt.Sprintf(`%s {
+ forwardproxy {
+ basicauth %s %s
+ probe_resistance
+ upstream %s
+ }
+ log / stdout "[{when}] \"{method} {uri} {proto}\" {status} {size}"
+ errors stdout
+ root %s
+ %s
+}
+`, caddyHostname,
+ bridgeUrl.User.Username(), caddyPw,
+ upstream,
+ caddyRoot,
+ extraDirectives)
+
+ caddyInstance, err := caddy.Start(caddy.CaddyfileInput{ServerTypeName: "http", Contents: []byte(caddyFile)})
+ if err != nil {
+ pt.ProxyError("failed to start caddy: " + err.Error())
+ os.Exit(9)
+ }
+ go func() {
+ caddyInstance.Wait() // if caddy stopped -- exit
+ pt.ProxyError("Caddy has stopped. Exiting.")
+ sigChan <- syscall.SIGTERM
+ }()
+}
diff --git a/server/server_test.go b/server/server_test.go
new file mode 100644
index 0000000..a60f4b4
--- /dev/null
+++ b/server/server_test.go
@@ -0,0 +1,113 @@
+// Copyright 2018 Jigsaw Operations LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "net/http"
+ "testing"
+)
+
+func TestParseForwarded(t *testing.T) {
+ makeHeader := func(headers map[string]string) http.Header {
+ h := make(http.Header)
+ for k, v := range headers {
+ h.Add(k, v)
+ }
+ return h
+ }
+
+ expectErr := func(headersMap map[string]string) {
+ header := makeHeader(headersMap)
+ h, err := parseForwardedTor(header)
+ if err == nil {
+ t.Fatalf("Expected: error, got: parsed %s\nheader was: %s\n", h, header)
+ }
+ }
+
+ expectNoErr := func(headersMap map[string]string, expectedHostname string) {
+ header := makeHeader(headersMap)
+ h, err := parseForwardedTor(header)
+ if err != nil {
+ t.Fatalf("Expected: parsed %s, got: error %s\nheader was: %s\n",
+ expectedHostname, err, header)
+ }
+
+ if h != expectedHostname {
+ t.Fatalf("Expected: %s, got: %s\nheader was: %s\n",
+ expectedHostname, h, header)
+ }
+ }
+
+ // according to the rfc, many of those are valid, including 8.8.8.8 and bazinga:123, however
+ // tor spec requires that it is an IP address and has port
+ expectErr(map[string]string{
+ "X-Forwarded-For": "bazinga",
+ })
+ expectErr(map[string]string{
+ "X-Forwarded-For": "bazinga:123",
+ })
+ expectErr(map[string]string{
+ "X-Forwarded-For": "8.8.8.8",
+ })
+ expectErr(map[string]string{
+ "Forwarded": "127.0.0.1",
+ })
+ expectErr(map[string]string{
+ "Forwarded": "127.0.0.1:22",
+ })
+ expectErr(map[string]string{
+ "Forwarded": "for=127.0.0.1",
+ })
+ expectErr(map[string]string{
+ "Forwarded": "for=you:123",
+ })
+ expectErr(map[string]string{
+ "Forwarded": "For=888.8.8.8:123",
+ })
+ expectErr(map[string]string{
+ "Forwarded": "for=[c:d:e:g:h:i]:5678",
+ })
+
+ expectNoErr(map[string]string{
+ "Forwarded": "for=1.1.1.1:44444",
+ }, "1.1.1.1:44444")
+ expectNoErr(map[string]string{
+ "x-ForwarDed-fOr": "8.8.8.8:123",
+ }, "8.8.8.8:123")
+ expectNoErr(map[string]string{
+ "ForwarDed": "FoR=8.8.8.8:123",
+ }, "8.8.8.8:123")
+ expectNoErr(map[string]string{
+ "ForwarDed": "FoR=[1:2::3:4]:5678",
+ }, "[1:2::3:4]:5678")
+ expectNoErr(map[string]string{
+ "ForwarDed": "FoR=[fe80::1ff:fe23:4567:890a]:5678",
+ }, "[fe80::1ff:fe23:4567:890a]:5678")
+ expectNoErr(map[string]string{
+ "ForwarDed": "FoR=[eeee:eeee:eeee:eeee:eeee:eeee:eeee:eeee]:5678",
+ }, "[eeee:eeee:eeee:eeee:eeee:eeee:eeee:eeee]:5678")
+ expectNoErr(map[string]string{
+ "ForwarDed": "FoR=8.8.8.8:123;",
+ }, "8.8.8.8:123")
+ expectNoErr(map[string]string{
+ "ForwarDed": "FoR=8.8.8.8:123; by=me",
+ }, "8.8.8.8:123")
+ expectNoErr(map[string]string{
+ "ForwarDed": "proto=amazingProto; FoR=8.8.8.8:123; by=me",
+ }, "8.8.8.8:123")
+ expectNoErr(map[string]string{
+ "ForwarDed": "proto=amazingProto;FoR = 8.8.8.8:123 ;by=me",
+ }, "8.8.8.8:123")
+}
More information about the tor-commits
mailing list