commit 6ea203b85aa8d98548ff6ef7cff8ce501ee401c3 Author: David Fifield david@bamsoftware.com Date: Wed Feb 13 19:30:30 2019 -0700

Minimal draft of WebExtension port of meek-http-helper.

Doesn't yet: * allow control of the Host header (domain fronting) * support a proxy --- webextension/README | 51 +++++ webextension/background.js | 159 +++++++++++++++ webextension/manifest.json | 22 ++ webextension/meek.http.helper.json | 9 + webextension/native/endian_amd64.go | 8 + webextension/native/main.go | 387 ++++++++++++++++++++++++++++++++++++ 6 files changed, 636 insertions(+)

diff --git a/webextension/README b/webextension/README new file mode 100644 index 0000000..a728842 --- /dev/null +++ b/webextension/README @@ -0,0 +1,51 @@ +Installation guide for the meek-http-helper WebExtension. + +The WebExtension is made of two parts: the extension and the native +application. The extension itself is JavaScript, runs in the browser, +and is responsible for making HTTP requests as instructed. The native +application runs as a subprocess of the browser; its job is to open a +localhost socket and act as an intermediary between the extension and +meek-client, because the extension cannot open a socket by itself. + +These instructions require Firefox 65. + +1. Compile the native application. + cd native && go build + +2. Edit meek.http.helper.json and set the "path" field to the path to + the native application. + "path": "/where/you/installed/native", + +3. Copy the edited meek.http.helper.json file to the OS-appropriate + location. + # macOS + mkdir -p ~/"Library/Application Support/Mozilla/NativeMessagingHosts/" + cp meek.http.helper.json ~/"Library/Application Support/Mozilla/NativeMessagingHosts/" + # other Unix + mkdir -p ~/.mozilla/native-messaging-hosts/ + cp meek.http.helper.json ~/.mozilla/native-messaging-hosts/ + The meek.http.helper.json file is called the "host manifest" or "app + manifest" and it tells the browser where to find the native part of + the WebExtension. More information: + https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/Nativ... + +4. Run Firefox in a terminal so you can see its stdout. In Firefox, go + to about:config and set + browser.dom.window.dump.enabled=true + This enables the extension to write to stdout. + +5. In Firefox, go to about:debugging and click "Load Temporary + Add-on...". Find manifest.json and click Open. + More information: + https://developer.mozilla.org/en-US/docs/Tools/about:debugging#Loading_a_tem... + In the terminal, you should see a line like this, with a random port + number in place of XXXX: + meek-http-helper: listen 127.0.0.1:XXXX + +Now the extension is running and ready to start making requests. You can +run "meek-client --helper", passing it the correct port number XXXX: + UseBridges 1 + ClientTransportPlugin meek exec ./meek-client --helper 127.0.0.1:XXXX --log meek-client.log + Bridge meek 0.0.2.0:1 url=https://meek.bamsoftware.com/ + +To debug, open the browser console with Ctrl+Shift+J. diff --git a/webextension/background.js b/webextension/background.js new file mode 100644 index 0000000..ba56e7f --- /dev/null +++ b/webextension/background.js @@ -0,0 +1,159 @@ +// This program is the browser part of the meek-http-helper WebExtension. Its +// purpose is to receive and execute commands from the native part. It +// understands two commands: "report-address" and "roundtrip". +// +// +// { +// "command": "report-address", +// "address": "127.0.0.1:XXXX" +// } +// The "report-address" command causes the extension to print to a line to +// stdout: +// meek-http-helper: listen 127.0.0.1:XXXX +// meek-client looks for this line to find out where the helper is listening. +// For this to work, you must set the pref browser.dom.window.dump.enabled. +// +// +// { +// "command": "roundtrip", +// "id": "...ID..." +// "request": { +// "method": "POST", +// "url": "https://allowed.example/", +// "header": { +// "Host": "forbidden.example", +// "X-Session-Id": ..., +// ... +// }, +// "proxy": { +// "type": "http", +// "host": "proxy.example", +// "port": 8080 +// }, +// "body": "...base64..." +// } +// } +// The "roundtrip" command causes the extension to make an HTTP request +// according to the given specification. It then sends a response back to the +// native part: +// { +// "id": "...ID...", +// "response": { +// "status": 200, +// "body": "...base64..." +// } +// } +// Or, if an error occurred: +// { +// "id": "...ID...", +// "response": { +// "error": "...error message..." +// } +// } +// The "id" field in the response will be the same as the one in the request, +// because that is what enables the native part to match up requests and +// responses. + +let port = browser.runtime.connectNative("meek.http.helper"); + +// Decode a base64-encoded string into an ArrayBuffer. +function base64_decode(enc_str) { + // First step is to decode the base64. atob returns a byte string; i.e., a + // string of 16-bit characters, each of whose character codes is restricted + // to the range 0x00–0xff. + let dec_str = atob(enc_str); + // Next, copy those character codes into an array of 8-bit elements. + let dec_array = new Uint8Array(dec_str.length); + for (let i = 0; i < dec_str.length; i++) { + dec_array[i] = dec_str.charCodeAt(i); + } + return dec_array.buffer; +} + +// Encode an ArrayBuffer into a base64-encoded string. +function base64_encode(dec_buf) { + let dec_array = new Uint8Array(dec_buf); + // Copy the elements of the array into a new byte string. + let dec_str = String.fromCharCode(...dec_array); + // base64-encode the byte string. + return btoa(dec_str); +} + +function roundtrip(id, request) { + // Process the incoming request spec and convert it into parameters to the + // fetch API. Also enforce some restrictions on what kinds of requests we + // are willing to make. + let url; + let init = {}; + try { + if (request.url == null) { + throw new Error("missing "url""); + } + if (!(request.url.startsWith("http://") || request.url.startsWith("https://"))) { + throw new Error("only http and https URLs are allowed"); + } + url = request.url; + + if (request.method !== "POST") { + throw new Error("only POST is allowed"); + } + init.method = request.method; + + if (request.header != null) { + init.headers = request.header; + } + + if (request.body != null && request.body !== "") { + init.body = base64_decode(request.body); + } + + // TODO: Host header + // TODO: strip Origin header? + // TODO: proxy + } catch (error) { + port.postMessage({id, response: {error: `request spec failed valiation: ${error.message}`}}); + return; + } + + // Now actually do the request and send the result back to the native + // process. + fetch(url, init) + .then(resp => resp.arrayBuffer().then(body => ({ + status: resp.status, + body: base64_encode(body), + }))) + // Convert any errors into an error response. + .catch(error => ({error: error.message})) + // Send the response (success or failure) back to the requester, tagged + // with its ID. + .then(response => port.postMessage({id, response})); +} + +port.onMessage.addListener((message) => { + switch (message.command) { + case "roundtrip": + roundtrip(message.id, message.request); + break; + case "report-address": + // Tell meek-client where our subprocess (the one that actually + // opens a socket) is listening. For the dump call to have any + // effect, the pref browser.dom.window.dump.enabled must be true. + // This output is supposed to be line-oriented, so ignore it if the + // address from the native part contains a newline. + if (message.address != null && message.address.indexOf("\n") == -1) { + dump(`meek-http-helper: listen ${message.address}\n`); + } + break; + default: + console.log(`${browser.runtime.id}: received unknown command: ${message.command}`); + } +}); + +port.onDisconnect.addListener((p) => { + // https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/r... + // "Note that in Google Chrome port.error is not supported: instead, use + // runtime.lastError to get the error message." + if (p.error) { + console.log(`${browser.runtime.id}: disconnected because of error: ${p.error.message}`); + } +}); diff --git a/webextension/manifest.json b/webextension/manifest.json new file mode 100644 index 0000000..2d44d87 --- /dev/null +++ b/webextension/manifest.json @@ -0,0 +1,22 @@ +{ + "manifest_version": 2, + "name": "meek HTTP helper", + "description": "Makes HTTP requests on behalf of meek-client.", + "version": "1.0", + + "browser_specific_settings": { + "gecko": { + "id": "meek-http-helper@bamsoftware.com" + } + }, + + "background": { + "scripts": ["background.js"] + }, + + "permissions": [ + "nativeMessaging", + "https://*/*", + "http://*/*" + ] +} diff --git a/webextension/meek.http.helper.json b/webextension/meek.http.helper.json new file mode 100644 index 0000000..269d5f8 --- /dev/null +++ b/webextension/meek.http.helper.json @@ -0,0 +1,9 @@ +{ + "name": "meek.http.helper", + "description": "Native half of meek-http-helper.", + "path": "/path/to/native", + "type": "stdio", + "allowed_extensions": [ + "meek-http-helper@bamsoftware.com" + ] +} diff --git a/webextension/native/endian_amd64.go b/webextension/native/endian_amd64.go new file mode 100644 index 0000000..6a1e44e --- /dev/null +++ b/webextension/native/endian_amd64.go @@ -0,0 +1,8 @@ +// The WebExtension browser–app protocol uses native-endian length prefixes :/ +// https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/Nativ... + +package main + +import "encoding/binary" + +var NativeEndian = binary.LittleEndian diff --git a/webextension/native/main.go b/webextension/native/main.go new file mode 100644 index 0000000..4c66993 --- /dev/null +++ b/webextension/native/main.go @@ -0,0 +1,387 @@ +// This program is the native part of the meek-http-helper WebExtension. Its +// purpose is to open a localhost TCP socket for communication with meek-client +// in its --helper mode (the WebExtension cannot open a socket on its own). This +// program is also in charge of multiplexing the many incoming socket +// connections over the single shared stdio stream to/from the WebExtension. + +package main + +import ( + "crypto/rand" + "encoding/binary" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "math" + "net" + "os" + "os/signal" + "sync" + "syscall" + "time" +) + +const ( + // How long we'll wait for meek-client to send a request spec or receive + // a response spec over the socket. This can be short because it's all + // localhost communication. + localReadTimeout = 2 * time.Second + localWriteTimeout = 2 * time.Second + + // How long we'll wait, after sending a request spec to the browser, for + // the browser to come back with a response. This is meant to be + // generous; its purpose is to allow reclaiming memory in case the + // browser somehow drops a request spec. + roundTripTimeout = 120 * time.Second + + // Self-defense against a malfunctioning meek-client. We'll refuse to + // read encoded responses that are longer than this. + maxRequestSpecLength = 1000000 + + // Self-defense against a malfunctioning browser. We'll refuse to + // receive WebExtension messages that are longer than this. + maxWebExtensionMessageLength = 1000000 +) + +// We receive multiple (possibly concurrent) connections over our listening +// socket, and we must multiplex all their requests/responses to/from the +// browser over the single shared stdio stream. When handleConn sends a +// webExtensionRoundTripRequest to the browser, creates a channel to receive the +// response, and stores the ID–channel mapping in requestResponseMap. When +// inFromBrowserLoop receives a webExtensionRoundTripResponse from the browser, +// it is tagged with the same ID as the corresponding request. inFromBrowserLoop +// looks up the matching channel and sends the response over it. +var requestResponseMap = make(map[string]chan<- *responseSpec) +var requestResponseMapLock sync.Mutex + +// A specification of an HTTP request, as received via the socket from +// "meek-client --helper". +type requestSpec struct { + Method string `json:"method,omitempty"` + URL string `json:"url,omitempty"` + Header map[string]string `json:"header,omitempty"` + Body []byte `json:"body,omitempty"` + Proxy *proxySpec `json:"proxy,omitempty"` +} + +type proxySpec struct { + Type string `json:"type"` + Host string `json:"host"` + Port string `json:"port"` +} + +// A specification of an HTTP request or an error, as sent via the socket to +// "meek-client --helper". +type responseSpec struct { + Error string `json:"error,omitempty"` + Status int `json:"status,omitempty"` + Body []byte `json:"body,omitempty"` +} + +// A "roundtrip" command sent out to the browser over the stdout stream. It +// encapsulates a requestSpec as received from the socket, plus +// command:"roundtrip" and an ID, which used to match up the eventual reply with +// this request. +// +// command:"roundtrip" is to disambiguate with the other command we may send, +// "report-address". +type webExtensionRoundTripRequest struct { + Command string `json:"command"` // "roundtrip" + ID string `json:"id"` + Request *requestSpec `json:"request"` +} + +// A message received from the the browser over the stdin stream. It +// encapsulates a responseSpec along with the ID of the webExtensionResponse +// that resulted in this response. +type webExtensionRoundTripResponse struct { + ID string `json:"id"` + Response *responseSpec `json:"response"` +} + +// Read a requestSpec (receive from "meek-client --helper"). +// +// The meek-client protocol is coincidentally similar to the WebExtension stdio +// protocol: a 4-byte length, followed by a JSON object of that length. The only +// difference is the byte order of the length: meek-client's is big-endian, +// while WebExtension's is native-endian. +func readRequestSpec(r io.Reader) (*requestSpec, error) { + var length uint32 + err := binary.Read(r, binary.BigEndian, &length) + if err != nil { + return nil, err + } + if length > maxRequestSpecLength { + return nil, fmt.Errorf("request spec is too long: %d (max %d)", length, maxRequestSpecLength) + } + + encodedSpec := make([]byte, length) + _, err = io.ReadFull(r, encodedSpec) + if err != nil { + return nil, err + } + + spec := new(requestSpec) + err = json.Unmarshal(encodedSpec, spec) + if err != nil { + return nil, err + } + + return spec, nil +} + +// Write a responseSpec (send to "meek-client --helper"). +func writeResponseSpec(w io.Writer, spec *responseSpec) error { + encodedSpec, err := json.Marshal(spec) + if err != nil { + panic(err) + } + + length := len(encodedSpec) + if length > math.MaxUint32 { + return fmt.Errorf("response spec is too long to represent: %d", length) + } + err = binary.Write(w, binary.BigEndian, uint32(length)) + if err != nil { + return err + } + + _, err = w.Write(encodedSpec) + if err != nil { + return err + } + + return nil +} + +// Receive a WebExtension message. +// https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/Nativ... +func recvWebExtensionMessage(r io.Reader) ([]byte, error) { + var length uint32 + err := binary.Read(r, NativeEndian, &length) + if err != nil { + return nil, err + } + if length > maxWebExtensionMessageLength { + return nil, fmt.Errorf("WebExtension message is too long: %d (max %d)", length, maxWebExtensionMessageLength) + } + message := make([]byte, length) + _, err = io.ReadFull(r, message) + if err != nil { + return nil, err + } + return message, nil +} + +// Send a WebExtension message. +// https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/Nativ... +func sendWebExtensionMessage(w io.Writer, message []byte) error { + length := len(message) + if length > math.MaxUint32 { + return fmt.Errorf("WebExtension message is too long to represent: %d", length) + } + err := binary.Write(w, NativeEndian, uint32(length)) + if err != nil { + return err + } + _, err = w.Write(message) + return err +} + +// Handle a socket connection, which is used for one request–response roundtrip +// through the browser. We read a responseSpec from the socket and wrap it in a +// webExtensionRoundTripRequest, tagging it with a random ID. We register the ID +// in requestResponseMap and forward the webExtensionRoundTripRequest to the +// browser. Then we wait for the browser to send back a +// webExtensionRoundTripResponse, which actually happens in inFromBrowserLoop. +// inFromBrowserLoop uses the ID to find this goroutine again. +func handleConn(conn net.Conn, outToBrowserChan chan<- []byte) error { + defer conn.Close() + + err := conn.SetReadDeadline(time.Now().Add(localReadTimeout)) + if err != nil { + return err + } + req, err := readRequestSpec(conn) + if err != nil { + return err + } + + // Generate an ID that will allow us to match a response to this request. + idRaw := make([]byte, 8) + _, err = rand.Read(idRaw) + if err != nil { + return err + } + id := hex.EncodeToString(idRaw) + + // This is the channel over which inFromBrowserLoop will send the + // response. Register it in requestResponseMap to enable + // inFromBrowserLoop to match the corresponding response to it. + responseSpecChan := make(chan *responseSpec) + requestResponseMapLock.Lock() + requestResponseMap[id] = responseSpecChan + requestResponseMapLock.Unlock() + + // Encode and send the message to the browser. + message, err := json.Marshal(&webExtensionRoundTripRequest{ + Command: "roundtrip", + ID: id, + Request: req, + }) + if err != nil { + panic(err) + } + outToBrowserChan <- message + + // Now wait for the browser to send the response back to us. + // inFromBrowserLoop will find the proper channel by looking up the ID + // in requestResponseMap. + timeout := time.NewTimer(roundTripTimeout) + select { + case resp := <-responseSpecChan: + timeout.Stop() + // Encode the response send it back out over the socket. + err = conn.SetWriteDeadline(time.Now().Add(localWriteTimeout)) + if err != nil { + return err + } + err = writeResponseSpec(conn, resp) + if err != nil { + return err + } + case <-timeout.C: + // But don't wait forever, so as to allow reclaiming memory in + // case of a malfunction elsewhere. + requestResponseMapLock.Lock() + delete(requestResponseMap, id) + requestResponseMapLock.Unlock() + } + + return nil +} + +// Receive socket connections and dispatch them to handleConn. +func acceptLoop(ln net.Listener, outToBrowserChan chan<- []byte) error { + for { + conn, err := ln.Accept() + if err != nil { + if err, ok := err.(net.Error); ok && err.Temporary() { + continue + } + return err + } + go func() { + err := handleConn(conn, outToBrowserChan) + if err != nil { + fmt.Fprintln(os.Stderr, "handling socket request:", err) + } + }() + } +} + +// Read messages from the browser over stdin, and send them (matching using the +// ID field) over the channel that corresponds to the original request. This is +// the only function allowed to read from stdin. +func inFromBrowserLoop() error { + for { + message, err := recvWebExtensionMessage(os.Stdin) + if err != nil { + return err + } + var resp webExtensionRoundTripResponse + err = json.Unmarshal(message, &resp) + if err != nil { + return err + } + + // Look up what channel (previously registered in + // requestResponseMap by handleConn) should receive the + // response. + requestResponseMapLock.Lock() + responseSpecChan, ok := requestResponseMap[resp.ID] + delete(requestResponseMap, resp.ID) + requestResponseMapLock.Unlock() + + if !ok { + // Either the browser made up an ID that we never sent + // it, or (more likely) it took too long and handleConn + // stopped waiting. Just drop the response on the floor. + continue + } + responseSpecChan <- resp.Response + // Each socket Conn is good for one request–response exchange only. + close(responseSpecChan) + } +} + +// Read messages from outToBrowserChan and send them to the browser over the +// stdout channel. This is the only function allowed to write to stdout. +func outToBrowserLoop(outToBrowserChan <-chan []byte) error { + for message := range outToBrowserChan { + err := sendWebExtensionMessage(os.Stdout, message) + if err != nil { + return err + } + } + return nil +} + +func main() { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + defer ln.Close() + + outToBrowserChan := make(chan []byte) + signalChan := make(chan os.Signal) + errChan := make(chan error) + + // Goroutine that handles new socket connections. + go func() { + errChan <- acceptLoop(ln, outToBrowserChan) + }() + + // Goroutine that writes WebExtension messages to stdout. + go func() { + errChan <- outToBrowserLoop(outToBrowserChan) + }() + + // Goroutine that reads WebExtension messages from stdin. + go func() { + err := inFromBrowserLoop() + if err == io.EOF { + // EOF is not an error. + err = nil + } + errChan <- err + }() + + // Tell the browser our listening socket address. + message, err := json.Marshal(struct { + Command string `json:"command"` + Address string `json:"address"` + }{ + Command: "report-address", + Address: ln.Addr().String(), + }) + if err != nil { + panic(err) + } + outToBrowserChan <- message + + // We quit when we receive a SIGTERM, or when our stdin is closed, or + // some irrecoverable error happens. + // https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/Nativ... + signal.Notify(signalChan, syscall.SIGTERM) + select { + case <-signalChan: + case err := <-errChan: + if err != nil { + fmt.Fprintln(os.Stderr, err) + } + } +}