July 2012 - tor-commits - lists.torproject.org

[stegotorus/master] Major bugfix in steg module initialization.
by zwol＠torproject.org 20 Jul '12

20 Jul '12

commit 8323c152091279b2b1e920aeea297e72dfac048a Author: Zack Weinberg <zackw(a)panix.com> Date: Wed Feb 1 21:04:38 2012 -0800 Major bugfix in steg module initialization. --- src/protocol/chop.cc | 19 ++++++++++++------- src/steg.cc | 4 ++-- src/steg.h | 2 +- src/steg/nosteg_rr.cc | 3 ++- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/protocol/chop.cc b/src/protocol/chop.cc index 3aa0efd..5acc010 100644 --- a/src/protocol/chop.cc +++ b/src/protocol/chop.cc @@ -799,7 +799,7 @@ chop_find_or_make_circuit(conn_t *conn, uint64_t circuit_id) ck = out.first->second; log_debug(conn, "found circuit to %s", ck->up_peer); } else { - ck = cfg->circuit_create(0); + ck = circuit_create(cfg, 0); if (!ck) { log_warn(conn, "failed to create new circuit"); return -1; @@ -969,6 +969,12 @@ chop_circuit_t::~chop_circuit_t() chop_reassembly_elt *p, *q, *queue; chop_circuit_table::iterator out; + log_debug(this, "syn%c%c fin%c%c eof%c ds=%lu", + sent_syn ? '+' : '-', received_syn ? '+' : '-', + sent_fin ? '+' : '-', received_fin ? '+' : '-', + upstream_eof ? '+' : '-', + (unsigned long)downstreams.size()); + for (unordered_set<conn_t *>::iterator i = this->downstreams.begin(); i != this->downstreams.end(); i++) { conn_t *conn = *i; @@ -1049,7 +1055,8 @@ chop_config_t::conn_create(size_t index) { chop_conn_t *conn = new chop_conn_t; conn->cfg = this; - conn->steg = steg_new(this->steg_targets.at(index)); + conn->steg = steg_new(this->steg_targets.at(index), + this->mode != LSN_SIMPLE_SERVER); if (!conn->steg) { free(conn); return 0; @@ -1275,15 +1282,13 @@ chop_conn_t::recv() int chop_conn_t::recv_eof() { - circuit_t *c = this->circuit; - /* EOF on a _connection_ does not mean EOF on a _circuit_. EOF on a _circuit_ occurs when chop_push_to_upstream processes a FIN. We should only drop the connection from the circuit if we're no longer sending in the opposite direction. Also, we should not drop the connection if its must-transmit timer is still pending. */ - if (c) { - chop_circuit_t *ckt = static_cast<chop_circuit_t *>(c); + if (this->circuit) { + chop_circuit_t *ckt = static_cast<chop_circuit_t *>(this->circuit); if (evbuffer_get_length(conn_get_inbound(this)) > 0) if (this->recv()) @@ -1292,7 +1297,7 @@ chop_conn_t::recv_eof() if ((ckt->sent_fin || this->no_more_transmissions) && (!this->must_transmit_timer || !evtimer_pending(this->must_transmit_timer, NULL))) - circuit_drop_downstream(c, this); + circuit_drop_downstream(ckt, this); } return 0; } diff --git a/src/steg.cc b/src/steg.cc index 768a6d8..8db5498 100644 --- a/src/steg.cc +++ b/src/steg.cc @@ -19,12 +19,12 @@ steg_is_supported(const char *name) /* Instantiate a steg module by name. */ steg_t * -steg_new(const char *name) +steg_new(const char *name, bool is_clientside) { const steg_module *const *s; for (s = supported_stegs; *s; s++) if (!strcmp(name, (**s).name)) - return (**s).new_(/*is_clientside=*/true); + return (**s).new_(is_clientside); return NULL; } diff --git a/src/steg.h b/src/steg.h index 03096f1..63f0fc9 100644 --- a/src/steg.h +++ b/src/steg.h @@ -61,7 +61,7 @@ struct steg_module extern const steg_module *const supported_stegs[]; int steg_is_supported(const char *name); -steg_t *steg_new(const char *name); +steg_t *steg_new(const char *name, bool is_clientside); /* Macros for use in defining steg modules. */ diff --git a/src/steg/nosteg_rr.cc b/src/steg/nosteg_rr.cc index 8fd70fe..3c8a5a0 100644 --- a/src/steg/nosteg_rr.cc +++ b/src/steg/nosteg_rr.cc @@ -94,7 +94,8 @@ nosteg_rr::receive(conn_t *conn, struct evbuffer *dest) { struct evbuffer *source = conn_get_inbound(conn); - log_debug(conn, "receiving %lu bytes", + log_debug(conn, "%s-side receiving %lu bytes", + is_clientside ? "client" : "server", (unsigned long)evbuffer_get_length(source)); if (evbuffer_add_buffer(dest, source)) {

1 0

[stegotorus/master] FreeBSD portability fixes
by zwol＠torproject.org 20 Jul '12

20 Jul '12

commit 572cfd2fc2071dd3ecd3c3ecdb41d77285b468fc Author: Zack Weinberg <zackw(a)cmu.edu> Date: Mon Feb 13 18:48:57 2012 +0000 FreeBSD portability fixes --- src/main.cc | 2 +- src/socks.cc | 1 + src/test/unittest_socks.cc | 1 + src/util.cc | 4 ++-- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main.cc b/src/main.cc index 8b6ca37..d7f3d87 100644 --- a/src/main.cc +++ b/src/main.cc @@ -105,8 +105,8 @@ static void lethal_signal(int signum, siginfo_t *si, void *) { char faultmsg[80]; - int n; #ifdef HAVE_EXECINFO_H + int n; void *backtracebuf[256]; #endif diff --git a/src/socks.cc b/src/socks.cc index dfcb745..05da3f4 100644 --- a/src/socks.cc +++ b/src/socks.cc @@ -8,6 +8,7 @@ #include "socks.h" #include <errno.h> +#include <netinet/in.h> #include <event2/buffer.h> diff --git a/src/test/unittest_socks.cc b/src/test/unittest_socks.cc index b46aedc..84c256b 100644 --- a/src/test/unittest_socks.cc +++ b/src/test/unittest_socks.cc @@ -9,6 +9,7 @@ #include "socks.h" #include <event2/buffer.h> +#include <netinet/in.h> /* All the tests below use this test environment. Some of them do not need both evbuffers, but we give them two anyway. */ diff --git a/src/util.cc b/src/util.cc index 9987871..9cca1fd 100644 --- a/src/util.cc +++ b/src/util.cc @@ -10,9 +10,9 @@ #include <unistd.h> #include <event2/dns.h> -#ifndef _WIN32 + +#include <netinet/in.h> #include <arpa/inet.h> -#endif #ifdef AF_LOCAL #include <sys/un.h> #endif

1 0

[stegotorus/master] Import pytorctl as a submodule.
by zwol＠torproject.org 20 Jul '12

20 Jul '12

commit 92757f00be2423045654602a89c60316299482c3 Author: Zack Weinberg <zackw(a)panix.com> Date: Mon Feb 13 20:14:28 2012 -0800 Import pytorctl as a submodule. --- .gitignore | 2 ++ .gitmodules | 3 +++ scripts/torctl | 1 + 3 files changed, 6 insertions(+), 0 deletions(-) diff --git a/.gitignore b/.gitignore index cfaa827..8de7c1e 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,5 @@ /build/ /build-*/ +/bench/ +/bench-*/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..a5c8001 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "scripts/torctl"] + path = scripts/torctl + url = git://git.torproject.org/pytorctl.git diff --git a/scripts/torctl b/scripts/torctl new file mode 160000 index 0000000..3de9176 --- /dev/null +++ b/scripts/torctl @@ -0,0 +1 @@ +Subproject commit 3de91767e208d1642a8cf89dc1b645f637abaf8e

1 0

[stegotorus/master] Rough draft benchmarking tools.
by zwol＠torproject.org 20 Jul '12

20 Jul '12

commit 704c053fbae93353976e3f7abf585b6283edb3a9 Author: Zack Weinberg <zackw(a)panix.com> Date: Sun Feb 12 20:36:56 2012 -0800 Rough draft benchmarking tools. --- scripts/benchmark-plot.R | 7 + scripts/benchmark.py | 410 +++++++++++++++++++++++++++++++++++ scripts/bm-fixedrate-cgi.c | 158 ++++++++++++++ scripts/bm-genfiles.py | 162 ++++++++++++++ scripts/bm-mcurl.c | 196 +++++++++++++++++ scripts/tool_urlglob.c | 516 ++++++++++++++++++++++++++++++++++++++++++++ scripts/tool_urlglob.h | 69 ++++++ 7 files changed, 1518 insertions(+), 0 deletions(-) diff --git a/scripts/benchmark-plot.R b/scripts/benchmark-plot.R new file mode 100644 index 0000000..a58ae0a --- /dev/null +++ b/scripts/benchmark-plot.R @@ -0,0 +1,7 @@ +#! /usr/bin/Rscript + +suppressPackageStartupMessages({ + library(ggplot2) +}) + +lf.direct <- read.csv("bench-lf-direct.tab", header=TRUE) diff --git a/scripts/benchmark.py b/scripts/benchmark.py new file mode 100755 index 0000000..c6487b6 --- /dev/null +++ b/scripts/benchmark.py @@ -0,0 +1,410 @@ +#! /usr/bin/python + +# Stegotorus benchmarking script. +# Several different computers are involved: +# +# - the "client" is the machine you run this script on; the workload +# generator will run there, as will the StegoTorus and Tor clients. +# +# - the "proxy" is a machine that you can ssh to with no password. +# It will run the StegoTorus and Tor bridge servers. +# +# - the "target" is the HTTP server that will be contacted in various ways. +# +# bm-genfiles.py must have been run on this server to create file +# trees named 'fixed' and 'pareto' which appear as direct children +# of the root URL. bm-fixedrate-cgi.c must have been compiled for +# that server and appear as /bm-fixedrate.cgi. +# +# Software you need on the client machine: +# +# bwm-ng: http://www.gropp.org/?id=projects&sub=bwm-ng +# curl: http://curl.haxx.se/ +# httperf: http://www.hpl.hp.com/research/linux/httperf/ +# tsocks: http://tsocks.sourceforge.net/about.php +# tor: https://torproject.org/ +# stegotorus: you already have it :) +# +# Software you need on the proxy machine: +# +# nylon: http://monkey.org/~marius/pages/?page=nylon +# tor, stegotorus +# +# You configure this script by setting variables below. + +# Client host + +CLIENT_IP = "99.113.33.155" +CLIENT_IFACE = "eth0" + +# Proxy host + +PROXY = "sandbox03.sv.cmu.edu" +PROXY_IP = "209.129.244.30" # some things won't do DNS for this +PROXY_PORT = "1080" +PROXY_SSH_CMD = ("ssh", PROXY) + +# Target + +TARGET = "storustest.nfshost.com" + +# Fudge factors. For some reason, bm-fixedrate generates data a +# linear factor slower than it was meant to; this is the quick fix. + +FUDGE_FIXEDRATE = 2.5 + +# Programs we need to run. Change these if any binary is not in the +# default path or hasn't got the default name. +# C_ - for the client. P_ - for the proxy. +# You can NOT specify arguments here - if you need to do any +# setup, write a wrapper script. + +C_bwm = "bwm-ng" +C_curl = "curl" +C_httperf = "httperf" +C_storus = "stegotorus-wrapper" +C_tor = "/usr/sbin/tor" +C_tsocks = "/usr/lib/libtsocks.so" + +P_nylon = "nylon" +P_storus = "stegotorus-wrapper" +P_tor = "tor" +P_python = "/usr/local/bin/python" # this must be an absolute path, + # it goes on a shebang line + +# ACTUAL PROGRAM STARTS HERE + +from types import MethodType +import os +import os.path +import pickle +import subprocess +import sys +import time + +def monitor(report, label, period): + """Monitor network utilization (bytes/sec up and down) for a + period of PERIOD seconds, writing the report to REPORT, labeling + each line with LABEL.""" + + bwm = subprocess.Popen((C_bwm, "-o", "csv", "-c", str(period), "-t", "1000", + "-u", "bytes", "-T", "rate", "-I", CLIENT_IFACE), + stdout=subprocess.PIPE, + universal_newlines=True) + try: + n = 1 + for line in bwm.stdout: + (stamp, iface, upbytes, dnbytes, rest) = line.split(';', 4) + if iface == 'total': continue + + # convert to most compact possible form + upbytes = str(float(upbytes)) + dnbytes = str(float(dnbytes)) + + report.write("%s,%d,%s,%s\n" % (label,n,upbytes,dnbytes)) + n += 1 + except: + bwm.terminate() + raise + finally: + bwm.wait() + +class ProxyProcess(object): + """A process running on the proxy host. It has a command line and + an optional config file. It is not expected to produce any output + (if it does, it will get dumped to this script's stdout/stderr) or + require any input (input is redirected from /dev/null). It is + expected to run until it is killed.""" + + @staticmethod + def prepare_remote(): + remote_driver=r"""#! %s +import pickle +import signal +import subprocess +import sys +import traceback + +wrote_rpid = False + +# Remote driver for proxy processes. +try: + data = pickle.load(sys.stdin) + sys.stdin.close() + if data['cfgname']: + f = open(data['cfgname'], "w") + f.write(data['cfgdata']) + f.close() + proc = subprocess.Popen(data['args'], stdin=open("/dev/null", "r"), + stdout=2) # redirect child stdout to our stderr + sys.stdout.write(str(proc.pid) + "\n") + wrote_rpid = True + sys.stdout.close() + proc.wait() + + # the process being killed by SIGTERM is normal + if proc.returncode != 0 and proc.returncode != -signal.SIGTERM: + raise subprocess.CalledProcessError(proc.returncode, data['args'][0]) +except: + traceback.print_exc() + if not wrote_rpid: sys.stdout.write("X\n") + sys.exit(1) + +sys.exit(0) +""" % P_python + remote_setup=r"""newdriver=`mktemp ./driver.py.XXXXXX` || exit 1 +cat > "$newdriver" +if cmp -s "$newdriver" driver.py +then rm -f "$newdriver" +else set -e; mv -f "$newdriver" driver.py; chmod +x driver.py +fi +""" + prep_worker = subprocess.Popen(PROXY_SSH_CMD + (remote_setup,), + stdin=subprocess.PIPE, + stdout=2) + prep_worker.communicate(remote_driver) + if prep_worker.returncode != 0: + raise subprocess.CalledProcessError(prep_worker.returncode, + 'remote_setup script') + + def __init__(self, args, cfgname=None, cfgdata=None): + if ((cfgname is None or cfgdata is None) and + (cfgname is not None or cfgdata is not None)): + raise TypeError("either both or neither of cfgname and cfgdata" + " must be specified") + + self._rpid = "X" + + ProxyProcess.prepare_remote() + self._proc = subprocess.Popen(PROXY_SSH_CMD + ("./driver.py",), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + close_fds=True) + pickle.dump({ 'args' : args, + 'cfgname' : cfgname, + 'cfgdata' : cfgdata }, + self._proc.stdin, 2) + self._proc.stdin.close() + self._rpid = self._proc.stdout.readline().strip() + if self._rpid == "X" or self._rpid == "": + self._rpid = "X" + self._proc.wait() + raise RuntimeError("failed to execute '%s' on proxy host" + % " ".join(args)) + + def terminate(self): + if self._rpid == "X": return + subprocess.check_call(PROXY_SSH_CMD + ("kill", self._rpid)) + + def kill(self): + if self._rpid == "X": return + subprocess.check_call(PROXY_SSH_CMD + ("kill", "-9", self._rpid)) + + # forward everything else to _proc; logic copied verbatim from + # http://code.activestate.com/recipes/519639- + # true-lieberman-style-delegation-in-python/ + def __getattr__(self, aname): + target = self._proc + f = getattr(target, aname) + if isinstance(f, MethodType): + return MethodType(f.im_func, self, target.__class__) + else: + return f + +# Individual proxy-side test runners. +def p_nylon(): + return ProxyProcess((P_nylon, "-f", "-c", "nylon.conf"), + "nylon.conf", + """\ +[General] +No-Simultaneous-Conn=10 +Log=0 +Verbose=0 +PIDfile=nylon.pid + +[Server] +Port=%s +Allow-IP=%s/32 +""" % (PROXY_PORT, CLIENT_IP)) + +def p_tor_direct(): + return ProxyProcess((P_tor, "--quiet", "-f", "tor-direct.conf"), + "tor-direct.conf", + """\ +ORPort %s +SocksPort 0 +BridgeRelay 1 +PublishServerDescriptor 0 +ExitPolicy reject *:* +DataDirectory . +Log err stderr +# unfortunately there doesn't seem to be any way to tell Tor to accept +# OR connections from specific IP addresses only. +""" % PROXY_PORT) + +class ClientProcess(subprocess.Popen): + """A process running on the local machine. This is probably doing + the meat of the work of some benchmark. Basically a shim around + subprocess.Popen to fix constructor arguments.""" + + def __init__(self, argv, envp=None): + if envp is not None: + env = os.environ.copy() + env.update(envp) + subprocess.Popen.__init__(self, argv, + stdin=open("/dev/null", "r"), + stdout=open("/dev/null", "w"), + stderr=subprocess.STDOUT, env=env) + else: + subprocess.Popen.__init__(self, argv, + stdin=open("/dev/null", "r"), + stdout=2) + +def c_tor_direct(): + fp = open("tor-direct-client.conf", "w") + fp.write("""\ +ORPort 0 +SocksPort %s +DataDirectory . +Log err stderr +Bridge %s:%s +UseBridges 1 +SafeSocks 0 +""" % (PROXY_PORT, PROXY_IP, PROXY_PORT)) + fp.close() + return ClientProcess((C_tor, "--quiet", "-f", "tor-direct-client.conf")) + +def c_curl(url, proxyhost): + return ClientProcess((C_curl, "-s", "--socks5-hostname", + proxyhost + ":" + PROXY_PORT, + url, "-o", "/dev/null")) + +def c_httperf(prefix, rate, proxyhost): + fp = open("tsocks.conf", "w") + fp.write("""\ +server = %s +local = %s/255.255.255.255 +server_port = %s +server_type = 5 +""" % (proxyhost, proxyhost, PROXY_PORT)) + fp.close() + return ClientProcess((C_httperf, "--hog", + "--server=" + TARGET, + "--uri=" + prefix, + "--period=" + str(rate), + "--num-calls=5", "--num-conns=2000", + "--wset=10000,1"), + { 'LD_PRELOAD' : C_tsocks, + 'TSOCKS_CONF_FILE' : + os.path.join(os.getcwd(), "tsocks.conf") }) + +# Benchmarks. + +def bench_fixedrate_direct(report): + client = None + proxy = None + try: + proxy = p_nylon() + + for cap in range(10, 810, 10): + sys.stderr.write("fixedrate,direct,%d\n" % (cap * 1000)) + try: + client = c_curl('http://' + TARGET + '/bm-fixedrate.cgi/' + + str(int(cap * 1000 * FUDGE_FIXEDRATE)), + PROXY) + monitor(report, "fixedrate,direct,%d" % (cap * 1000), 60) + finally: + if client is not None: + client.terminate() + client.wait() + client = None + finally: + if proxy is not None: + proxy.terminate() + proxy.wait() + +def bench_fixedrate_tor(report): + client = None + proxy = None + proxyl = None + try: + proxy = p_tor_direct() + proxyl = c_tor_direct() + time.sleep(5) # tor startup is slow + + for cap in range(10,810,10): + sys.stderr.write("fixedrate,tor,%d\n" % (cap * 1000)) + try: + client = c_curl('http://' + TARGET + '/bm-fixedrate.cgi/' + + str(int(cap * 1000 * FUDGE_FIXEDRATE)), + '127.0.0.1') + monitor(report, "fixedrate,tor,%d" % (cap * 1000), 60) + finally: + if client is not None: + client.terminate() + client.wait() + client = None + finally: + if proxy is not None: + proxy.terminate() + proxy.wait() + if proxyl is not None: + proxyl.terminate() + proxyl.wait() + +def bench_files_direct(report, prefix): + client = None + proxy = None + try: + proxy = p_nylon() + + for cps in range(1,81): + sys.stderr.write("files.%s,direct,%d\n" % (prefix, cps)) + try: + client = c_httperf(prefix, 1./cps, PROXY_IP) + monitor(report, "files.%s,direct,%d" % (prefix, cps), 60) + finally: + if client is not None: + client.terminate() + client.wait() + client = None + finally: + if proxy is not None: + proxy.terminate() + proxy.wait() + +def bench_files_tor(report, prefix): + client = None + proxy = None + proxyl = None + try: + proxy = p_tor_direct() + proxyl = c_tor_direct() + time.sleep(5) # tor startup is slow + + for cps in range(1,81): + sys.stderr.write("files.%s,tor,%d\n" % (prefix, cps)) + try: + client = c_httperf(prefix, 1./cps, '127.0.0.1') + monitor(report, "files.%s,tor,%d" % (prefix, cps), 60) + finally: + if client is not None: + client.terminate() + client.wait() + client = None + finally: + if proxy is not None: + proxy.terminate() + proxy.wait() + if proxyl is not None: + proxyl.terminate() + proxyl.wait() + +if __name__ == '__main__': + sys.stdout.write("benchmark,relay,cap,obs,up,down\n") + bench_fixedrate_direct(sys.stdout) + bench_fixedrate_tor(sys.stdout) + bench_files_direct(sys.stdout, "fixed") + bench_files_tor(sys.stdout, "fixed") + bench_files_direct(sys.stdout, "pareto") + bench_files_tor(sys.stdout, "pareto") diff --git a/scripts/bm-fixedrate-cgi.c b/scripts/bm-fixedrate-cgi.c new file mode 100644 index 0000000..2b48f98 --- /dev/null +++ b/scripts/bm-fixedrate-cgi.c @@ -0,0 +1,158 @@ +#define _XOPEN_SOURCE 600 +#define _POSIX_C_SOURCE 200112 + +#include <stdbool.h> +#include <stddef.h> + +#include <errno.h> +#include <math.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <unistd.h> + +/* 1400 bytes is a safe figure for per-packet transmissible payload. */ +#define BLOCKSZ 1400 + + +#if __GNUC__ >= 3 +#define NORETURN void __attribute__((noreturn)) +#else +#define NORETURN void +#endif + +extern char **environ; + +static NORETURN +error_400(const char *msg) +{ + char **p; + printf("Status: 400 Bad Request\nContent-Type: text/plain\n\n" + "400 Bad Request (%s)\nCGI environment dump follows:\n\n", msg); + for (p = environ; *p; p++) + puts(*p); + exit(0); +} + +static NORETURN +error_500(const char *syscall) +{ + printf("Status: 500 Internal Server Error\nContent-Type:text/plain\n\n" + "500 Internal Server Error: %s: %s\n", + syscall, strerror(errno)); + exit(0); +} + +static void +generate(unsigned long rate, bool dryrun) +{ + double interval; + timer_t timerid; + struct sigevent sev; + struct itimerspec its; + sigset_t mask; + int sig; + char *data; + size_t bufsz = BLOCKSZ; + + /* You send data at R bytes per second in 1400-byte blocks by + calling write() every 1/(R/1400) second. However, despite our + use of the high-resolution interval timers, we cannot count on + being scheduled more often than every 1/CLOCKS_PER_SEC seconds, + so if we need to send data faster than that, bump up the block + size instead. */ + interval = 1./(rate/(double)BLOCKSZ); + + if (interval < 1./CLOCKS_PER_SEC) { + interval = 1./CLOCKS_PER_SEC; + bufsz = rate / CLOCKS_PER_SEC; + } + + its.it_value.tv_sec = lrint(floor(interval)); + its.it_value.tv_nsec = lrint((interval - its.it_value.tv_sec) * 1e9); + its.it_interval.tv_sec = its.it_value.tv_sec; + its.it_interval.tv_nsec = its.it_value.tv_nsec; + + if (dryrun) { + printf("Content-Type: text/plain\n\n" + "Goal %lu bytes per second:\n" + "would send %lu bytes every %f seconds\n" + " \" \" \" \" \" %lu sec + %lu nsec\n", + rate, bufsz, interval, + (unsigned long)its.it_value.tv_sec, + (unsigned long)its.it_value.tv_nsec); + return; + } + + data = malloc(bufsz); + if (!data) + error_500("malloc"); + memset(data, 0, bufsz); + + fflush(stdout); + setvbuf(stdout, 0, _IONBF, 0); + fputs("Content-Type: application/octet-stream\n" + "Cache-Control: no-store,no-cache\n\n", stdout); + + sigemptyset(&mask); + sigaddset(&mask, SIGRTMIN); + if (sigprocmask(SIG_SETMASK, &mask, 0)) + error_500("sigprocmask"); + + memset(&sev, 0, sizeof sev); + sev.sigev_notify = SIGEV_SIGNAL; + sev.sigev_signo = SIGRTMIN; + sev.sigev_value.sival_ptr = &timerid; + if (timer_create(CLOCK_MONOTONIC, &sev, &timerid)) + error_500("timer_create"); + + if (timer_settime(timerid, 0, &its, 0)) + error_500("timer_settime"); + + do { + size_t r, n = bufsz; + char *p = data; + do { + r = fwrite(p, 1, n, stdout); + if (r == 0) + exit(1); + n -= r; + p += r; + } while (n > 0); + } while (sigwait(&mask, &sig) == 0); +} + +int +main(void) +{ + unsigned long rate; + char *endp; + bool dryrun; + char *request_method = getenv("REQUEST_METHOD"); + char *query_string = getenv("QUERY_STRING"); + char *path_info = getenv("PATH_INFO"); + + if (!request_method || strcmp(request_method, "GET")) + error_400("method not supported"); + if (query_string && strcmp(query_string, "")) + error_400("no query parameters accepted"); + + if (!path_info || path_info[0] != '/') + error_400("malformed or missing PATH_INFO"); + + rate = strtoul(path_info+1, &endp, 10); + if (endp == path_info+1) + error_400("missing rate (specify bytes per second)"); + + if (endp[0] == '\0') + dryrun = false; + else if (endp[0] == ';' && endp[1] == 'd' && endp[2] == '\0') + dryrun = true; + else + error_400("unrecognized extra arguments"); + + generate(rate, dryrun); + return 0; +} diff --git a/scripts/bm-genfiles.py b/scripts/bm-genfiles.py new file mode 100755 index 0000000..dcd1030 --- /dev/null +++ b/scripts/bm-genfiles.py @@ -0,0 +1,162 @@ +#! /usr/bin/python + +"""Generate files for network performance testing. + +The default behavior is to generate 10,000 files all of which are +exactly 3584 bytes long, because that is approximately how big +Flickr's 75x75px JPEG thumbnails are. You can request a different +size, or you can request that the file sizes instead follow a bounded +Pareto distribution with tunable alpha. + +The files have names compatible with httperf's --wset mode. Since +it insists on .html as a file suffix, the files are syntactically +valid HTML. Their contents are word salad. + +There is one mandatory command line argument: the path to the root +of the tree of files to generate. It is created if it doesn't +already exist. If it already exists, its contents will be erased! +(so don't use '.')""" + +from __future__ import division + +import argparse +import errno +import math +import os +import os.path +import random +import shutil +import sys +import textwrap + +def ensure_empty_dir(dpath): + todelete = [] + try: + todelete = os.listdir(dpath) + except OSError, e: + # Don't delete a _file_ that's in the way. + # Don't try to create parent directories that are missing. + if e.errno != errno.ENOENT: + raise + os.mkdir(dpath) + return + for f in todelete: + p = os.path.join(dpath, f) + try: + os.remove(p) + except OSError, e: + if e.errno != errno.EISDIR and e.errno != errno.EPERM: + raise + shutil.rmtree(p) + +def ensure_parent_directories(path): + try: + os.makedirs(os.path.dirname(path)) + except OSError, e: + if e.errno != errno.EEXIST: + raise + +def word_salad(f, words, seed, maxlen): + rng = random.Random(seed) + salad = [] + slen = 0 + while slen < maxlen - 1: + nl = rng.randint(1, min((maxlen - 1) - slen, len(words))) - 1 + w = rng.choice(words[nl]) + salad.append(w) + slen += len(w) + 1 + salad = textwrap.fill(" ".join(salad), 78) + while len(salad) < maxlen-1: + salad += '.' + salad += '\n' + f.write(salad) + +def load_words(): + words = [ [] for _ in xrange(15) ] + for w in open('/usr/share/dict/words'): + w = w.strip() + if w.endswith("'s"): continue + if len(w) > 15 or len(w) < 2: continue + words[len(w)-1].append(w) + # special case words[0] as dictfiles often have every single single letter + words[0].extend(('a','I')) + return words + +FILE_PREFIX = '<!doctype html>\n<title>{0}</title>\n<p>\n' +FILE_SUFFIX = '</p>\n' + +def create_one(parent, ctr, digits, words, filesize, seed, resume, progress): + label = format(ctr, '0'+str(digits)+'d') + fname = os.path.join(parent, *label) + '.html' + ensure_parent_directories(fname) + + if os.path.exists(fname): + if not resume: raise RuntimeError('{0} already exists'.format(fname)) + return + + prefix = FILE_PREFIX.format(label) + suffix = FILE_SUFFIX + limit = filesize - (len(prefix) + len(suffix)) + if limit <= 0: + raise TypeError("{0} bytes is too small to generate (minimum {1})" + .format(filesize, len(prefix)+len(suffix))) + + if progress: + sys.stderr.write(fname + '\n') + + f = open(fname, "w") + f.write(prefix) + word_salad(f, words, ctr+seed, limit) + f.write(suffix) + +def bounded_pareto(rng, alpha, L, H): + while True: + U = rng.random() + if U < 1: break + Ha = H**alpha + La = L**alpha + return int(round((-(U*Ha - U*La - Ha)/(Ha * La)) ** (-1/alpha))) + +if __name__ == '__main__': + + default_filesize = 3584 + default_filecount = 10000 # 0/0/0/0.html through 9/9/9/9.html + + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('directory', + help='directory to populate with files') + parser.add_argument('-c', '--count', type=int, default=default_filecount, + help='number of files to generate') + sg = parser.add_mutually_exclusive_group() + sg.add_argument('-s', '--size', type=int, default=default_filesize, + help='all files will be exactly SIZE bytes long') + sg.add_argument('-p', '--pareto', type=float, + metavar='ALPHA', + help='file sizes will follow a bounded Pareto distribution' + ' with parameter ALPHA') + parser.add_argument('-m', '--minsize', type=int, default=512, + help='minimum file size (only useful with -p)') + parser.add_argument('-M', '--maxsize', type=int, default=2*1024*1024, + help='maximum file size (only useful with -p)') + parser.add_argument('-S', '--seed', type=int, default=719, + help='seed for random number generator') + parser.add_argument('--resume', action='store_true', + help='resume an interrupted run where it left off') + parser.add_argument('--progress', action='store_true', + help='report progress') + + args = parser.parse_args() + digits = len(str(args.count - 1)) + rng = random.Random(args.seed) + + words = load_words() + if not args.resume: + ensure_empty_dir(args.directory) + + size = args.size + for i in xrange(args.count): + if args.pareto is not None: + size = bounded_pareto(rng, args.pareto, args.minsize, args.maxsize) + create_one(args.directory, i, digits, words, size, args.seed, + args.resume, args.progress) diff --git a/scripts/bm-mcurl.c b/scripts/bm-mcurl.c new file mode 100644 index 0000000..ac24f3a --- /dev/null +++ b/scripts/bm-mcurl.c @@ -0,0 +1,196 @@ +/* Use libcurl to retrieve many URLs, according to a wildcard pattern, + starting new connections at a constant rate until we hit a limit. + + Command line arguments -- all are required, but 'proxy' may be an + empty string if you want direct connections: + + bm-mcurl [-v] rate limit proxy url-pattern [url-pattern ...] + + There is no output; it is assumed that you are monitoring traffic + externally. Passing -v turns on CURLOPT_VERBOSE debugging spew. + */ + +#define _XOPEN_SOURCE 600 + +#include <stdbool.h> +#include <stddef.h> + +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <unistd.h> + +#include <curl/curl.h> +#include "tool_urlglob.h" + +#define NORETURN __attribute__((noreturn)) + +static bool verbose = false; + +static size_t +discard_data(char *ptr, size_t size, size_t nmemb, void *userdata) +{ + return size * nmemb; +} + +static size_t +read_abort(void *ptr, size_t size, size_t nmemb, void *userdata) +{ + /* we don't do anything that should require this to be called, + so if it does get called, something is wrong */ + return CURL_READFUNC_ABORT; +} + +static CURL * +setup_curl_easy_handle(char *proxy) +{ + CURL *h = curl_easy_init(); + if (!h) abort(); + +#define SET_OR_CRASH(h, opt, param) \ + do { if (curl_easy_setopt(h, opt, param)) abort(); } while (0) + + SET_OR_CRASH(h, CURLOPT_VERBOSE, (unsigned long)verbose); + SET_OR_CRASH(h, CURLOPT_NOPROGRESS, 1L); + SET_OR_CRASH(h, CURLOPT_FAILONERROR, 1L); + SET_OR_CRASH(h, CURLOPT_USERAGENT, "bm-mcurl/0.1"); + SET_OR_CRASH(h, CURLOPT_ACCEPT_ENCODING, ""); + SET_OR_CRASH(h, CURLOPT_AUTOREFERER, 1L); + SET_OR_CRASH(h, CURLOPT_FOLLOWLOCATION, 1L); + SET_OR_CRASH(h, CURLOPT_MAXREDIRS, 30L); + + SET_OR_CRASH(h, CURLOPT_WRITEFUNCTION, discard_data); + SET_OR_CRASH(h, CURLOPT_WRITEDATA, NULL); + SET_OR_CRASH(h, CURLOPT_READFUNCTION, read_abort); + SET_OR_CRASH(h, CURLOPT_READDATA, NULL); + + if (proxy && proxy[0]) { + SET_OR_CRASH(h, CURLOPT_PROXY, proxy); + SET_OR_CRASH(h, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5_HOSTNAME); + } +#undef SET_OR_CRASH +} + +static bool +process_events_once(CURLM *multi, unsigned long timeout_max) +{ + struct timeval tv; + int rc; /* select() return code */ + + fd_set fdread; + fd_set fdwrite; + fd_set fdexcept; + int maxfd = -1; + + unsigned long timeout = 1000000; /* one second - ultimate default */ + long curl_tout_ms = -1; + + /* get fd sets for all pending transfers */ + FD_ZERO(&fdread); + FD_ZERO(&fdwrite); + FD_ZERO(&fdexcept); + curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcept, &maxfd); + + /* timeout */ + if (timeout_max > 0 && timeout_max < timeout) + timeout = timeout_max; + + curl_multi_timeout(multi_handle, &curl_tout_ms); + + if (curl_tout_ms >= 0) { + unsigned long curl_tout_us = ((unsigned long)curl_tout_ms) * 1000; + if (timeout > curl_tout_us) + timeout = curl_tout_us; + } + + tv.tv_sec = timeout / 1000000; + if(tv.tv_sec >= 1) + tv.tv_sec = 1; + else + tv.tv_usec = timeout % 1000000; + + do { + rc = select(maxfd+1, &fdread, &fdwrite, &fdexcept, &tv); + } while (rc == -1 && errno == EINTR); + + if (rc > 0) { + int still_running; + curl_multi_perform(multi_handle, &still_running); + return !!still_running; + } else + abort(); +} + +/* Note: this function must not return until we are ready to start + another connection. */ +static void +queue_one(CURLM *multi, unsigned long rate, unsigned long limit, + char *proxy, char *url) +{ + +} + +static void +run(unsigned long rate, unsigned long limit, char *proxy, char **urls) +{ + CURLM *multi; + curl_global_init(); + multi = curl_multi_init(); + if (!multi) abort(); + + for (char **upat = urls; *upat; url++) { + URLGlob *uglob; + int *n; + if (glob_url(&uglob, *upat, &n, stderr)) + continue; + do { + char *url; + if (glob_next_url(&url, uglob)) abort(); + queue_one(multi, rate, limit, proxy, url); /* takes ownership */ + } while (--n); + glob_cleanup(uglob); + } + + /* spin the event loop until all outstanding transfers complete */ + while (process_events_once(multi, 0)); + + curl_multi_cleanup(multi); +} + +static NORETURN +usage(const char *av0, const char *complaint) +{ + fprintf(stderr, + "%s\nusage: %s [-v] rate limit proxy url [url...]\n", + complaint, av0); + exit(2); +} + +int +main(int argc, char **argv) +{ + unsigned long rate; + unsigned long limit; + char *endp; + + if (argv[1] && (!strcmp(argv[1], "-v") || !strcmp(argv[1], "--verbose"))) { + verbose = true; + argc--; + argv[1] = argv[0]; + argv++; + } + + if (argc < 5) + usage("not enough arguments"); + + rate = strtoul(argv[1], &endp, 10); + if (endp == argv[1] || *endp) + usage("rate must be a positive integer (connections per second)"); + + limit = strtoul(argv[2], &endp, 10); + if (endp == argv[2] || *endp) + usage("limit must be a positive integer (max outstanding requests)"); + + run(rate, limit, argv[3], argv+4); + return 0; +} diff --git a/scripts/tool_urlglob.c b/scripts/tool_urlglob.c new file mode 100644 index 0000000..d714971 --- /dev/null +++ b/scripts/tool_urlglob.c @@ -0,0 +1,516 @@ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2011, Daniel Stenberg, <daniel(a)haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +#include <stdbool.h> +#include <stdlib.h> +#include <string.h> +#include <curl/curl.h> + +#include "tool_urlglob.h" + +typedef enum { + GLOB_OK, + GLOB_NO_MEM, + GLOB_ERROR +} GlobCode; + +/* + * glob_word() + * + * Input a full globbed string, set the forth argument to the amount of + * strings we get out of this. Return GlobCode. + */ +static GlobCode glob_word(URLGlob *, /* object anchor */ + char *, /* globbed string */ + size_t, /* position */ + int *); /* returned number of strings */ + +static GlobCode glob_set(URLGlob *glob, char *pattern, + size_t pos, int *amount) +{ + /* processes a set expression with the point behind the opening '{' + ','-separated elements are collected until the next closing '}' + */ + URLPattern *pat; + GlobCode res; + bool done = false; + char* buf = glob->glob_buffer; + + pat = &glob->pattern[glob->size / 2]; + /* patterns 0,1,2,... correspond to size=1,3,5,... */ + pat->type = UPTSet; + pat->content.Set.size = 0; + pat->content.Set.ptr_s = 0; + pat->content.Set.elements = NULL; + + ++glob->size; + + while(!done) { + switch (*pattern) { + case '\0': /* URL ended while set was still open */ + snprintf(glob->errormsg, sizeof(glob->errormsg), + "unmatched brace at pos %zu\n", pos); + return GLOB_ERROR; + + case '{': + case '[': /* no nested expressions at this time */ + snprintf(glob->errormsg, sizeof(glob->errormsg), + "nested braces not supported at pos %zu\n", pos); + return GLOB_ERROR; + + case ',': + case '}': /* set element completed */ + *buf = '\0'; + if(pat->content.Set.elements) { + char **new_arr = realloc(pat->content.Set.elements, + (pat->content.Set.size + 1) * sizeof(char*)); + if(!new_arr) { + short elem; + for(elem = 0; elem < pat->content.Set.size; elem++) + Curl_safefree(pat->content.Set.elements[elem]); + Curl_safefree(pat->content.Set.elements); + pat->content.Set.ptr_s = 0; + pat->content.Set.size = 0; + } + pat->content.Set.elements = new_arr; + } + else + pat->content.Set.elements = malloc(sizeof(char*)); + if(!pat->content.Set.elements) { + snprintf(glob->errormsg, sizeof(glob->errormsg), "out of memory\n"); + return GLOB_NO_MEM; + } + pat->content.Set.elements[pat->content.Set.size] = + strdup(glob->glob_buffer); + if(!pat->content.Set.elements[pat->content.Set.size]) { + short elem; + for(elem = 0; elem < pat->content.Set.size; elem++) + Curl_safefree(pat->content.Set.elements[elem]); + Curl_safefree(pat->content.Set.elements); + pat->content.Set.ptr_s = 0; + pat->content.Set.size = 0; + snprintf(glob->errormsg, sizeof(glob->errormsg), "out of memory\n"); + return GLOB_NO_MEM; + } + ++pat->content.Set.size; + + if(*pattern == '}') { + /* entire set pattern completed */ + int wordamount; + + /* always check for a literal (may be "") between patterns */ + res = glob_word(glob, ++pattern, ++pos, &wordamount); + if(res) { + short elem; + for(elem = 0; elem < pat->content.Set.size; elem++) + Curl_safefree(pat->content.Set.elements[elem]); + Curl_safefree(pat->content.Set.elements); + pat->content.Set.ptr_s = 0; + pat->content.Set.size = 0; + return res; + } + + *amount = pat->content.Set.size * wordamount; + + done = true; + continue; + } + + buf = glob->glob_buffer; + ++pattern; + ++pos; + break; + + case ']': /* illegal closing bracket */ + snprintf(glob->errormsg, sizeof(glob->errormsg), + "illegal pattern at pos %zu\n", pos); + return GLOB_ERROR; + + case '\\': /* escaped character, skip '\' */ + if(pattern[1]) { + ++pattern; + ++pos; + } + /* intentional fallthrough */ + default: + *buf++ = *pattern++; /* copy character to set element */ + ++pos; + } + } + return GLOB_OK; +} + +static GlobCode glob_range(URLGlob *glob, char *pattern, + size_t pos, int *amount) +{ + /* processes a range expression with the point behind the opening '[' + - char range: e.g. "a-z]", "B-Q]" + - num range: e.g. "0-9]", "17-2000]" + - num range with leading zeros: e.g. "001-999]" + expression is checked for well-formedness and collected until the next ']' + */ + URLPattern *pat; + char *c; + char sep; + char sep2; + int step; + int rc; + GlobCode res; + int wordamount = 1; + + pat = &glob->pattern[glob->size / 2]; + /* patterns 0,1,2,... correspond to size=1,3,5,... */ + ++glob->size; + + if(ISALPHA(*pattern)) { + /* character range detected */ + char min_c; + char max_c; + + pat->type = UPTCharRange; + + rc = sscanf(pattern, "%c-%c%c%d%c", &min_c, &max_c, &sep, &step, &sep2); + + if((rc < 3) || (min_c >= max_c) || ((max_c - min_c) > ('z' - 'a'))) { + /* the pattern is not well-formed */ + snprintf(glob->errormsg, sizeof(glob->errormsg), + "error: bad range specification after pos %zu\n", pos); + return GLOB_ERROR; + } + + /* check the (first) separating character */ + if((sep != ']') && (sep != ':')) { + snprintf(glob->errormsg, sizeof(glob->errormsg), + "error: unsupported character (%c) after range at pos %zu\n", + sep, pos); + return GLOB_ERROR; + } + + /* if there was a ":[num]" thing, use that as step or else use 1 */ + pat->content.CharRange.step = + ((sep == ':') && (rc == 5) && (sep2 == ']')) ? step : 1; + + pat->content.CharRange.ptr_c = pat->content.CharRange.min_c = min_c; + pat->content.CharRange.max_c = max_c; + } + else if(ISDIGIT(*pattern)) { + /* numeric range detected */ + int min_n; + int max_n; + + pat->type = UPTNumRange; + pat->content.NumRange.padlength = 0; + + rc = sscanf(pattern, "%d-%d%c%d%c", &min_n, &max_n, &sep, &step, &sep2); + + if((rc < 2) || (min_n > max_n)) { + /* the pattern is not well-formed */ + snprintf(glob->errormsg, sizeof(glob->errormsg), + "error: bad range specification after pos %zu\n", pos); + return GLOB_ERROR; + } + pat->content.NumRange.ptr_n = pat->content.NumRange.min_n = min_n; + pat->content.NumRange.max_n = max_n; + + /* if there was a ":[num]" thing, use that as step or else use 1 */ + pat->content.NumRange.step = + ((sep == ':') && (rc == 5) && (sep2 == ']')) ? step : 1; + + if(*pattern == '0') { + /* leading zero specified */ + c = pattern; + while(ISDIGIT(*c)) { + c++; + ++pat->content.NumRange.padlength; /* padding length is set for all + instances of this pattern */ + } + } + } + else { + snprintf(glob->errormsg, sizeof(glob->errormsg), + "illegal character in range specification at pos %zu\n", pos); + return GLOB_ERROR; + } + + c = (char*)strchr(pattern, ']'); /* continue after next ']' */ + if(c) + c++; + else { + snprintf(glob->errormsg, sizeof(glob->errormsg), "missing ']'"); + return GLOB_ERROR; /* missing ']' */ + } + + /* always check for a literal (may be "") between patterns */ + + res = glob_word(glob, c, pos + (c - pattern), &wordamount); + if(res == GLOB_ERROR) { + wordamount = 1; + res = GLOB_OK; + } + + if(!res) { + if(pat->type == UPTCharRange) + *amount = wordamount * (pat->content.CharRange.max_c - + pat->content.CharRange.min_c + 1); + else + *amount = wordamount * (pat->content.NumRange.max_n - + pat->content.NumRange.min_n + 1); + } + + return res; /* GLOB_OK or GLOB_NO_MEM */ +} + +static GlobCode glob_word(URLGlob *glob, char *pattern, + size_t pos, int *amount) +{ + /* processes a literal string component of a URL + special characters '{' and '[' branch to set/range processing functions + */ + char* buf = glob->glob_buffer; + size_t litindex; + GlobCode res = GLOB_OK; + + *amount = 1; /* default is one single string */ + + while(*pattern != '\0' && *pattern != '{' && *pattern != '[') { + if(*pattern == '}' || *pattern == ']') { + snprintf(glob->errormsg, sizeof(glob->errormsg), + "unmatched close brace/bracket at pos %zu\n", pos); + return GLOB_ERROR; + } + + /* only allow \ to escape known "special letters" */ + if(*pattern == '\\' && + (*(pattern+1) == '{' || *(pattern+1) == '[' || + *(pattern+1) == '}' || *(pattern+1) == ']') ) { + + /* escape character, skip '\' */ + ++pattern; + ++pos; + } + *buf++ = *pattern++; /* copy character to literal */ + ++pos; + } + *buf = '\0'; + litindex = glob->size / 2; + /* literals 0,1,2,... correspond to size=0,2,4,... */ + glob->literal[litindex] = strdup(glob->glob_buffer); + if(!glob->literal[litindex]) { + snprintf(glob->errormsg, sizeof(glob->errormsg), "out of memory\n"); + return GLOB_NO_MEM; + } + ++glob->size; + + switch (*pattern) { + case '\0': + /* singular URL processed */ + break; + + case '{': + /* process set pattern */ + res = glob_set(glob, ++pattern, ++pos, amount); + break; + + case '[': + /* process range pattern */ + res = glob_range(glob, ++pattern, ++pos, amount); + break; + } + + if(res) + Curl_safefree(glob->literal[litindex]); + + return res; +} + +int glob_url(URLGlob** glob, char* url, int *urlnum, FILE *error) +{ + /* + * We can deal with any-size, just make a buffer with the same length + * as the specified URL! + */ + URLGlob *glob_expand; + int amount; + char *glob_buffer; + GlobCode res; + + *glob = NULL; + + glob_buffer = malloc(strlen(url) + 1); + if(!glob_buffer) + return CURLE_OUT_OF_MEMORY; + + glob_expand = calloc(1, sizeof(URLGlob)); + if(!glob_expand) { + Curl_safefree(glob_buffer); + return CURLE_OUT_OF_MEMORY; + } + glob_expand->size = 0; + glob_expand->urllen = strlen(url); + glob_expand->glob_buffer = glob_buffer; + glob_expand->beenhere = 0; + + res = glob_word(glob_expand, url, 1, &amount); + if(!res) + *urlnum = amount; + else { + if(error && glob_expand->errormsg[0]) { + /* send error description to the error-stream */ + fprintf(error, "curl: (%d) [globbing] %s", + (res == GLOB_NO_MEM) ? CURLE_OUT_OF_MEMORY : CURLE_URL_MALFORMAT, + glob_expand->errormsg); + } + /* it failed, we cleanup */ + Curl_safefree(glob_buffer); + Curl_safefree(glob_expand); + *urlnum = 1; + return (res == GLOB_NO_MEM) ? CURLE_OUT_OF_MEMORY : CURLE_URL_MALFORMAT; + } + + *glob = glob_expand; + return CURLE_OK; +} + +void glob_cleanup(URLGlob* glob) +{ + size_t i; + int elem; + + for(i = glob->size - 1; i < glob->size; --i) { + if(!(i & 1)) { /* even indexes contain literals */ + Curl_safefree(glob->literal[i/2]); + } + else { /* odd indexes contain sets or ranges */ + if((glob->pattern[i/2].type == UPTSet) && + (glob->pattern[i/2].content.Set.elements)) { + for(elem = glob->pattern[i/2].content.Set.size - 1; + elem >= 0; + --elem) { + Curl_safefree(glob->pattern[i/2].content.Set.elements[elem]); + } + Curl_safefree(glob->pattern[i/2].content.Set.elements); + } + } + } + Curl_safefree(glob->glob_buffer); + Curl_safefree(glob); +} + +int glob_next_url(char **globbed, URLGlob *glob) +{ + URLPattern *pat; + char *lit; + size_t i; + size_t j; + size_t len; + size_t buflen = glob->urllen + 1; + char *buf = glob->glob_buffer; + + *globbed = NULL; + + if(!glob->beenhere) + glob->beenhere = 1; + else { + bool carry = true; + + /* implement a counter over the index ranges of all patterns, + starting with the rightmost pattern */ + for(i = glob->size / 2 - 1; carry && (i < glob->size); --i) { + carry = false; + pat = &glob->pattern[i]; + switch (pat->type) { + case UPTSet: + if((pat->content.Set.elements) && + (++pat->content.Set.ptr_s == pat->content.Set.size)) { + pat->content.Set.ptr_s = 0; + carry = true; + } + break; + case UPTCharRange: + pat->content.CharRange.ptr_c = (char)(pat->content.CharRange.step + + (int)((unsigned char)pat->content.CharRange.ptr_c)); + if(pat->content.CharRange.ptr_c > pat->content.CharRange.max_c) { + pat->content.CharRange.ptr_c = pat->content.CharRange.min_c; + carry = true; + } + break; + case UPTNumRange: + pat->content.NumRange.ptr_n += pat->content.NumRange.step; + if(pat->content.NumRange.ptr_n > pat->content.NumRange.max_n) { + pat->content.NumRange.ptr_n = pat->content.NumRange.min_n; + carry = true; + } + break; + default: + printf("internal error: invalid pattern type (%d)\n", (int)pat->type); + return CURLE_FAILED_INIT; + } + } + if(carry) { /* first pattern ptr has run into overflow, done! */ + /* TODO: verify if this should actally return CURLE_OK. */ + return CURLE_OK; /* CURLE_OK to match previous behavior */ + } + } + + for(j = 0; j < glob->size; ++j) { + if(!(j&1)) { /* every other term (j even) is a literal */ + lit = glob->literal[j/2]; + len = snprintf(buf, buflen, "%s", lit); + buf += len; + buflen -= len; + } + else { /* the rest (i odd) are patterns */ + pat = &glob->pattern[j/2]; + switch(pat->type) { + case UPTSet: + if(pat->content.Set.elements) { + len = strlen(pat->content.Set.elements[pat->content.Set.ptr_s]); + snprintf(buf, buflen, "%s", + pat->content.Set.elements[pat->content.Set.ptr_s]); + buf += len; + buflen -= len; + } + break; + case UPTCharRange: + *buf++ = pat->content.CharRange.ptr_c; + break; + case UPTNumRange: + len = snprintf(buf, buflen, "%0*d", + pat->content.NumRange.padlength, + pat->content.NumRange.ptr_n); + buf += len; + buflen -= len; + break; + default: + printf("internal error: invalid pattern type (%d)\n", (int)pat->type); + return CURLE_FAILED_INIT; + } + } + } + *buf = '\0'; + + *globbed = strdup(glob->glob_buffer); + if(!*globbed) + return CURLE_OUT_OF_MEMORY; + + return CURLE_OK; +} diff --git a/scripts/tool_urlglob.h b/scripts/tool_urlglob.h new file mode 100644 index 0000000..562b08e --- /dev/null +++ b/scripts/tool_urlglob.h @@ -0,0 +1,69 @@ +#ifndef HEADER_CURL_TOOL_URLGLOB_H +#define HEADER_CURL_TOOL_URLGLOB_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2011, Daniel Stenberg, <daniel(a)haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +typedef enum { + UPTSet = 1, + UPTCharRange, + UPTNumRange +} URLPatternType; + +typedef struct { + URLPatternType type; + union { + struct { + char **elements; + short size; + short ptr_s; + } Set; + struct { + char min_c; + char max_c; + char ptr_c; + int step; + } CharRange; + struct { + int min_n; + int max_n; + short padlength; + int ptr_n; + int step; + } NumRange ; + } content; +} URLPattern; + +typedef struct { + char *literal[10]; + URLPattern pattern[9]; + size_t size; + size_t urllen; + char *glob_buffer; + char beenhere; + char errormsg[80]; /* error message buffer */ +} URLGlob; + +int glob_url(URLGlob**, char*, int *, FILE *); +int glob_next_url(char **, URLGlob *); +void glob_cleanup(URLGlob* glob); + +#endif /* HEADER_CURL_TOOL_URLGLOB_H */

1 0

[stegotorus/master] Finish bm-mcurl
by zwol＠torproject.org 20 Jul '12

20 Jul '12

commit 3ca55ce2498c8b946fcdcdd52539f2dd90c08ee1 Author: Zack Weinberg <zackw(a)cmu.edu> Date: Mon Feb 13 16:32:08 2012 -0800 Finish bm-mcurl --- scripts/bm-mcurl.c | 266 ++++++++++++++++++++++++++++++++++-------------- scripts/tool_urlglob.c | 46 ++++++--- 2 files changed, 219 insertions(+), 93 deletions(-) diff --git a/scripts/bm-mcurl.c b/scripts/bm-mcurl.c index ac24f3a..890b9e4 100644 --- a/scripts/bm-mcurl.c +++ b/scripts/bm-mcurl.c @@ -15,6 +15,8 @@ #include <stdbool.h> #include <stddef.h> +#include <errno.h> +#include <math.h> #include <stdlib.h> #include <string.h> #include <time.h> @@ -23,18 +25,73 @@ #include <curl/curl.h> #include "tool_urlglob.h" -#define NORETURN __attribute__((noreturn)) +#define NORETURN void __attribute__((noreturn)) +#define UNUSED __attribute__((unused)) static bool verbose = false; +static inline double +timevaldiff(const struct timeval *start, const struct timeval *finish) +{ + double s = finish->tv_sec - start->tv_sec; + s += ((double)(finish->tv_usec - start->tv_usec)) / 1.0e6; + return s; +} + +struct url_iter +{ + char **upats; + URLGlob *uglob; + int nglob; +}; + +static inline struct url_iter +url_prep(char **upats) +{ + struct url_iter it; + it.upats = upats; + it.uglob = NULL; + it.nglob = -1; + return it; +} + +static char * +url_next(struct url_iter *it) +{ + char *url; + + if (!it->uglob) { + for (;;) { + if (!*it->upats) + return 0; + if (!glob_url(&it->uglob, *it->upats, &it->nglob, stderr)) { + if (verbose) + fprintf(stderr, "# %s\n", *it->upats); + break; + } + it->upats++; + } + } + + if (glob_next_url(&url, it->uglob)) + abort(); + if (--it->nglob == 0) { + glob_cleanup(it->uglob); + it->uglob = 0; + it->upats++; + } + return url; +} + static size_t -discard_data(char *ptr, size_t size, size_t nmemb, void *userdata) +discard_data(char *ptr UNUSED, size_t size, size_t nmemb, void *userdata UNUSED) { return size * nmemb; } static size_t -read_abort(void *ptr, size_t size, size_t nmemb, void *userdata) +read_abort(void *ptr UNUSED, size_t size UNUSED, size_t nmemb UNUSED, + void *userdata UNUSED) { /* we don't do anything that should require this to be called, so if it does get called, something is wrong */ @@ -50,11 +107,11 @@ setup_curl_easy_handle(char *proxy) #define SET_OR_CRASH(h, opt, param) \ do { if (curl_easy_setopt(h, opt, param)) abort(); } while (0) - SET_OR_CRASH(h, CURLOPT_VERBOSE, (unsigned long)verbose); + /*SET_OR_CRASH(h, CURLOPT_VERBOSE, (unsigned long)verbose);*/ SET_OR_CRASH(h, CURLOPT_NOPROGRESS, 1L); SET_OR_CRASH(h, CURLOPT_FAILONERROR, 1L); SET_OR_CRASH(h, CURLOPT_USERAGENT, "bm-mcurl/0.1"); - SET_OR_CRASH(h, CURLOPT_ACCEPT_ENCODING, ""); + SET_OR_CRASH(h, CURLOPT_ENCODING, ""); SET_OR_CRASH(h, CURLOPT_AUTOREFERER, 1L); SET_OR_CRASH(h, CURLOPT_FOLLOWLOCATION, 1L); SET_OR_CRASH(h, CURLOPT_MAXREDIRS, 30L); @@ -69,99 +126,148 @@ setup_curl_easy_handle(char *proxy) SET_OR_CRASH(h, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5_HOSTNAME); } #undef SET_OR_CRASH + + return h; } -static bool -process_events_once(CURLM *multi, unsigned long timeout_max) +static void +process_urls(struct url_iter *it, CURLM *multi, CURL **handles, + unsigned long limit, double interval) { - struct timeval tv; - int rc; /* select() return code */ - + struct timeval last, now, timeout; fd_set fdread; fd_set fdwrite; fd_set fdexcept; int maxfd = -1; - - unsigned long timeout = 1000000; /* one second - ultimate default */ - long curl_tout_ms = -1; - - /* get fd sets for all pending transfers */ - FD_ZERO(&fdread); - FD_ZERO(&fdwrite); - FD_ZERO(&fdexcept); - curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcept, &maxfd); - - /* timeout */ - if (timeout_max > 0 && timeout_max < timeout) - timeout = timeout_max; - - curl_multi_timeout(multi_handle, &curl_tout_ms); - - if (curl_tout_ms >= 0) { - unsigned long curl_tout_us = ((unsigned long)curl_tout_ms) * 1000; - if (timeout > curl_tout_us) - timeout = curl_tout_us; + int rc; + int still_running; + int dummy; + unsigned long maxh = 0; + unsigned long i; + CURLMsg *msg; + CURL *h; + char *url; + double d_timeout; + long curl_timeout; + bool no_more_urls = false; + + last.tv_sec = 0; + last.tv_usec = 0; + + for (;;) { + /* possibly queue another URL for download */ + if (!no_more_urls) { + gettimeofday(&now, 0); + if (timevaldiff(&last, &now) >= interval && maxh < limit) { + last = now; + url = url_next(it); + if (url) { + if (curl_easy_setopt(handles[maxh], CURLOPT_URL, url) != CURLM_OK) + abort(); + if (curl_multi_add_handle(multi, handles[maxh]) != CURLM_OK) + abort(); + maxh++; + free(url); /* curl takes a copy */ + } else + no_more_urls = true; + } + } + + /* call curl_multi_perform as many times as it wants */ + again: + switch (curl_multi_perform(multi, &still_running)) { + case CURLM_OK: break; + case CURLM_CALL_MULTI_PERFORM: goto again; + default: + abort(); + } + if (no_more_urls && still_running == 0) + break; + + /* clean up finished downloads */ + while ((msg = curl_multi_info_read(multi, &dummy))) { + if (msg->msg != CURLMSG_DONE) + abort(); /* no other messages are defined as of Feb 2012 */ + h = msg->easy_handle; + if (verbose) { + double rqtime = 0.0; + char *url = "<?>"; + curl_easy_getinfo(h, CURLINFO_EFFECTIVE_URL, &url); + curl_easy_getinfo(h, CURLINFO_TOTAL_TIME, &rqtime); + fprintf(stderr, "%f %s\n", rqtime, url); + } + if (curl_multi_remove_handle(multi, h) != CURLM_OK) + abort(); + for (i = 0; i < maxh; i++) { + if (handles[i] == h) + goto found; + } + abort(); + found: + /* shuffle 'h' to the beginning of the set of handles not + currently in use */ + handles[i] = handles[--maxh]; + handles[maxh] = h; + } + + /* wait for external event or timeout */ + FD_ZERO(&fdread); + FD_ZERO(&fdwrite); + FD_ZERO(&fdexcept); + curl_multi_fdset(multi, &fdread, &fdwrite, &fdexcept, &maxfd); + + curl_multi_timeout(multi, &curl_timeout); + if (curl_timeout >= 0) + d_timeout = ((double)curl_timeout) / 1000.0; + else + d_timeout = 1; + if (d_timeout > interval) + d_timeout = interval; + + timeout.tv_sec = floor(d_timeout); + timeout.tv_usec = lrint((d_timeout - timeout.tv_sec) * 1e6); + + do + rc = select(maxfd+1, &fdread, &fdwrite, &fdexcept, &timeout); + while (rc == -1 && errno == EINTR); + if (rc == -1) + abort(); } - - tv.tv_sec = timeout / 1000000; - if(tv.tv_sec >= 1) - tv.tv_sec = 1; - else - tv.tv_usec = timeout % 1000000; - - do { - rc = select(maxfd+1, &fdread, &fdwrite, &fdexcept, &tv); - } while (rc == -1 && errno == EINTR); - - if (rc > 0) { - int still_running; - curl_multi_perform(multi_handle, &still_running); - return !!still_running; - } else - abort(); } -/* Note: this function must not return until we are ready to start - another connection. */ static void -queue_one(CURLM *multi, unsigned long rate, unsigned long limit, - char *proxy, char *url) -{ - -} - -static void -run(unsigned long rate, unsigned long limit, char *proxy, char **urls) +run(double interval, unsigned long limit, char *proxy, char **upats) { + struct url_iter it; CURLM *multi; - curl_global_init(); + CURL **handles; + unsigned long n; + + curl_global_init(CURL_GLOBAL_ALL); multi = curl_multi_init(); if (!multi) abort(); - for (char **upat = urls; *upat; url++) { - URLGlob *uglob; - int *n; - if (glob_url(&uglob, *upat, &n, stderr)) - continue; - do { - char *url; - if (glob_next_url(&url, uglob)) abort(); - queue_one(multi, rate, limit, proxy, url); /* takes ownership */ - } while (--n); - glob_cleanup(uglob); + handles = calloc(limit, sizeof(CURL *)); + for (n = 0; n < limit; n++) { + handles[n] = setup_curl_easy_handle(proxy); } - /* spin the event loop until all outstanding transfers complete */ - while (process_events_once(multi, 0)); + it = url_prep(upats); + process_urls(&it, multi, handles, limit, interval); + for (n = 0; n < limit; n++) { + curl_easy_cleanup(handles[n]); + } + free(handles); curl_multi_cleanup(multi); + curl_global_cleanup(); } static NORETURN usage(const char *av0, const char *complaint) { fprintf(stderr, - "%s\nusage: %s [-v] rate limit proxy url [url...]\n", + "%s\nusage: %s [-v] cps limit proxy url [url...]\n", complaint, av0); exit(2); } @@ -169,7 +275,7 @@ usage(const char *av0, const char *complaint) int main(int argc, char **argv) { - unsigned long rate; + unsigned long cps; unsigned long limit; char *endp; @@ -181,16 +287,20 @@ main(int argc, char **argv) } if (argc < 5) - usage("not enough arguments"); + usage(argv[0], "not enough arguments"); - rate = strtoul(argv[1], &endp, 10); + cps = strtoul(argv[1], &endp, 10); if (endp == argv[1] || *endp) - usage("rate must be a positive integer (connections per second)"); + usage(argv[0], "cps must be a positive integer (connections per second)"); limit = strtoul(argv[2], &endp, 10); if (endp == argv[2] || *endp) - usage("limit must be a positive integer (max outstanding requests)"); + usage(argv[0], + "limit must be a positive integer (max outstanding requests)"); + + if (limit == 0) + usage(argv[0], "minimum number of outstanding requests is 1"); - run(rate, limit, argv[3], argv+4); + run(1./(double)cps, limit, argv[3], argv+4); return 0; } diff --git a/scripts/tool_urlglob.c b/scripts/tool_urlglob.c index d714971..b160188 100644 --- a/scripts/tool_urlglob.c +++ b/scripts/tool_urlglob.c @@ -20,6 +20,8 @@ * ***************************************************************************/ +#define _XOPEN_SOURCE 600 /* strdup */ + #include <stdbool.h> #include <stdlib.h> #include <string.h> @@ -33,6 +35,20 @@ typedef enum { GLOB_ERROR } GlobCode; +/* assumes ASCII */ +static inline bool +ISALPHA(unsigned char c) +{ + return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); +} + +/* C90 guarantees '0' through '9' are consecutive */ +static inline bool +ISDIGIT(unsigned char c) +{ + return c >= '0' && c <= '9'; +} + /* * glob_word() * @@ -86,8 +102,8 @@ static GlobCode glob_set(URLGlob *glob, char *pattern, if(!new_arr) { short elem; for(elem = 0; elem < pat->content.Set.size; elem++) - Curl_safefree(pat->content.Set.elements[elem]); - Curl_safefree(pat->content.Set.elements); + free(pat->content.Set.elements[elem]); + free(pat->content.Set.elements); pat->content.Set.ptr_s = 0; pat->content.Set.size = 0; } @@ -104,8 +120,8 @@ static GlobCode glob_set(URLGlob *glob, char *pattern, if(!pat->content.Set.elements[pat->content.Set.size]) { short elem; for(elem = 0; elem < pat->content.Set.size; elem++) - Curl_safefree(pat->content.Set.elements[elem]); - Curl_safefree(pat->content.Set.elements); + free(pat->content.Set.elements[elem]); + free(pat->content.Set.elements); pat->content.Set.ptr_s = 0; pat->content.Set.size = 0; snprintf(glob->errormsg, sizeof(glob->errormsg), "out of memory\n"); @@ -122,8 +138,8 @@ static GlobCode glob_set(URLGlob *glob, char *pattern, if(res) { short elem; for(elem = 0; elem < pat->content.Set.size; elem++) - Curl_safefree(pat->content.Set.elements[elem]); - Curl_safefree(pat->content.Set.elements); + free(pat->content.Set.elements[elem]); + free(pat->content.Set.elements); pat->content.Set.ptr_s = 0; pat->content.Set.size = 0; return res; @@ -337,7 +353,7 @@ static GlobCode glob_word(URLGlob *glob, char *pattern, } if(res) - Curl_safefree(glob->literal[litindex]); + free(glob->literal[litindex]); return res; } @@ -361,7 +377,7 @@ int glob_url(URLGlob** glob, char* url, int *urlnum, FILE *error) glob_expand = calloc(1, sizeof(URLGlob)); if(!glob_expand) { - Curl_safefree(glob_buffer); + free(glob_buffer); return CURLE_OUT_OF_MEMORY; } glob_expand->size = 0; @@ -380,8 +396,8 @@ int glob_url(URLGlob** glob, char* url, int *urlnum, FILE *error) glob_expand->errormsg); } /* it failed, we cleanup */ - Curl_safefree(glob_buffer); - Curl_safefree(glob_expand); + free(glob_buffer); + free(glob_expand); *urlnum = 1; return (res == GLOB_NO_MEM) ? CURLE_OUT_OF_MEMORY : CURLE_URL_MALFORMAT; } @@ -397,7 +413,7 @@ void glob_cleanup(URLGlob* glob) for(i = glob->size - 1; i < glob->size; --i) { if(!(i & 1)) { /* even indexes contain literals */ - Curl_safefree(glob->literal[i/2]); + free(glob->literal[i/2]); } else { /* odd indexes contain sets or ranges */ if((glob->pattern[i/2].type == UPTSet) && @@ -405,14 +421,14 @@ void glob_cleanup(URLGlob* glob) for(elem = glob->pattern[i/2].content.Set.size - 1; elem >= 0; --elem) { - Curl_safefree(glob->pattern[i/2].content.Set.elements[elem]); + free(glob->pattern[i/2].content.Set.elements[elem]); } - Curl_safefree(glob->pattern[i/2].content.Set.elements); + free(glob->pattern[i/2].content.Set.elements); } } } - Curl_safefree(glob->glob_buffer); - Curl_safefree(glob); + free(glob->glob_buffer); + free(glob); } int glob_next_url(char **globbed, URLGlob *glob)

1 0

[stegotorus/master] Merge branch 'master' of ssh://sandbox03.sv.cmu.edu/usr/home/zack/stegotorus
by zwol＠torproject.org 20 Jul '12

20 Jul '12

commit 33dd34526ccf5c5b87875838857251f7bd4f8963 Merge: 704c053 572cfd2 Author: Zack Weinberg <zackw(a)cmu.edu> Date: Mon Feb 13 10:55:06 2012 -0800 Merge branch 'master' of ssh://sandbox03.sv.cmu.edu/usr/home/zack/stegotorus src/main.cc | 2 +- src/socks.cc | 1 + src/test/unittest_socks.cc | 1 + src/util.cc | 4 ++-- 4 files changed, 5 insertions(+), 3 deletions(-)

1 0

[stegotorus/master] Another subtle bug involving races near circuit teardown.
by zwol＠torproject.org 20 Jul '12

20 Jul '12

commit 4ae14e9e21d9b2848d089ba3ee150582e20744b9 Author: Zack Weinberg <zackw(a)panix.com> Date: Wed Feb 1 21:19:18 2012 -0800 Another subtle bug involving races near circuit teardown. --- src/network.cc | 3 ++- src/protocol/chop.cc | 32 +++++++++++++++++++------------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/network.cc b/src/network.cc index a11cd7d..a55e539 100644 --- a/src/network.cc +++ b/src/network.cc @@ -423,7 +423,8 @@ upstream_flush_cb(struct bufferevent *bev, void *arg) ckt->connected ? "" : " (not connected)", ckt->flushing ? "" : " (not flushing)"); - if (remain == 0 && ckt->flushing && ckt->connected) { + if (remain == 0 && ckt->flushing && ckt->connected + && (!ckt->flush_timer || !evtimer_pending(ckt->flush_timer, NULL))) { bufferevent_disable(bev, EV_WRITE); if (bufferevent_get_enabled(bev) || evbuffer_get_length(bufferevent_get_input(bev)) > 0) { diff --git a/src/protocol/chop.cc b/src/protocol/chop.cc index 5acc010..d7da97f 100644 --- a/src/protocol/chop.cc +++ b/src/protocol/chop.cc @@ -784,8 +784,6 @@ chop_push_to_upstream(circuit_t *c) static int chop_find_or_make_circuit(conn_t *conn, uint64_t circuit_id) { - log_assert(conn->cfg->mode == LSN_SIMPLE_SERVER); - chop_config_t *cfg = static_cast<chop_config_t *>(conn->cfg); chop_circuit_table::value_type in(circuit_id, 0); std::pair<chop_circuit_table::iterator, bool> out = cfg->circuits.insert(in); @@ -954,6 +952,11 @@ chop_config_t::circuit_create(size_t) ckt->recv_crypt = decryptor::create(s2c_key, 16); while (!ckt->circuit_id) rng_bytes((uint8_t *)&ckt->circuit_id, sizeof(uint64_t)); + + chop_circuit_table::value_type in(ckt->circuit_id, 0); + std::pair<chop_circuit_table::iterator, bool> out = circuits.insert(in); + log_assert(out.second); + out.first->second = ckt; } return ckt; } @@ -996,17 +999,20 @@ chop_circuit_t::~chop_circuit_t() free(p); } - if (this->cfg->mode == LSN_SIMPLE_SERVER) { - /* The IDs for old circuits are preserved for a while (at present, - indefinitely; FIXME: purge them on a timer) against the - possibility that we'll get a junk connection for one of them - right after we close it (same deal as the TIME_WAIT state in TCP). */ - chop_config_t *cfg = static_cast<chop_config_t *>(this->cfg); - out = cfg->circuits.find(this->circuit_id); - log_assert(out != cfg->circuits.end()); - log_assert(out->second == this); - out->second = NULL; - } + /* The IDs for old circuits are preserved for a while (at present, + indefinitely; FIXME: purge them on a timer) against the + possibility that we'll get a junk connection for one of them + right after we close it (same deal as the TIME_WAIT state in + TCP). Note that we can hit this case for the *client* if the + cover protocol includes a mandatory reply to every client + message and the hidden channel closed s->c before c->s: the + circuit will get destroyed on the client side after the c->s FIN, + and the mandatory reply will be to a stale circuit. */ + chop_config_t *cfg = static_cast<chop_config_t *>(this->cfg); + out = cfg->circuits.find(this->circuit_id); + log_assert(out != cfg->circuits.end()); + log_assert(out->second == this); + out->second = NULL; } void

1 0

[stegotorus/master] Yet another missed shutdown case.
by zwol＠torproject.org 20 Jul '12

20 Jul '12

commit e06cb9a45c52f0190f37be0a9109f25decda126e Author: Zack Weinberg <zackw(a)panix.com> Date: Thu Feb 2 08:43:18 2012 -0800 Yet another missed shutdown case. --- src/protocol/chop.cc | 19 +++++++++++++++++++ 1 files changed, 19 insertions(+), 0 deletions(-) diff --git a/src/protocol/chop.cc b/src/protocol/chop.cc index d7da97f..89a9ccc 100644 --- a/src/protocol/chop.cc +++ b/src/protocol/chop.cc @@ -1282,6 +1282,25 @@ chop_conn_t::recv() if (evbuffer_get_length(bufferevent_get_input(c->up_buffer))) c->send(); + /* If we're at EOF, close all connections (sending first if + necessary). If we're the client we have to keep trying to talk + as long as we haven't both sent and received a FIN, or we might + deadlock. */ + else if (ckt->sent_fin && ckt->received_fin) { + circuit_disarm_flush_timer(ckt); + for (unordered_set<conn_t *>::iterator i = ckt->downstreams.begin(); + i != ckt->downstreams.end(); i++) { + chop_conn_t *conn = static_cast<chop_conn_t*>(*i); + if (conn->must_transmit_timer && + evtimer_pending(conn->must_transmit_timer, NULL)) + must_transmit_timer_cb(-1, 0, conn); + conn_send_eof(conn); + } + } else { + if (ckt->cfg->mode != LSN_SIMPLE_SERVER) + circuit_arm_flush_timer(ckt, ckt->flush_interval()); + } + return 0; }

1 0

[stegotorus/master] Monkeyin' with the test suite.
by zwol＠torproject.org 20 Jul '12

20 Jul '12

commit dda19c5a05404677b14ca8460b7cebf7d330b91a Author: Zack Weinberg <zackw(a)panix.com> Date: Thu Feb 2 09:51:58 2012 -0800 Monkeyin' with the test suite. --- src/test/test_tl.py | 28 +++++++++++++--------------- 1 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/test/test_tl.py b/src/test/test_tl.py index a91beca..6d632f8 100644 --- a/src/test/test_tl.py +++ b/src/test/test_tl.py @@ -70,21 +70,19 @@ class TimelineTest(object): "127.0.0.1:5010","nosteg_rr", )) - # def test_chop_nosteg_rr2(self): - # self.doTest("chop", - # ("chop", "server", "127.0.0.1:5001", - # "127.0.0.1:5010","nosteg_rr","127.0.0.1:5011","nosteg_rr", - # "chop", "client", "127.0.0.1:4999", - # "127.0.0.1:5010","nosteg_rr","127.0.0.1:5011","nosteg_rr", - # )) - - # def test_chop_http(self): - # self.doTest("chop", - # ("chop", "server", "127.0.0.1:5001", - # "127.0.0.1:5010","http","127.0.0.1:5011","http", - # "chop", "client", "127.0.0.1:4999", - # "127.0.0.1:5010","http","127.0.0.1:5011","http", - # )) + def test_chop_nosteg_rr2(self): + self.doTest("chop", + ("chop", "server", "127.0.0.1:5001", + "127.0.0.1:5010","nosteg_rr","127.0.0.1:5011","nosteg_rr", + "chop", "client", "127.0.0.1:4999", + "127.0.0.1:5010","nosteg_rr","127.0.0.1:5011","nosteg_rr", + )) + + # NOTE: 'embed' steg presently cannot be tested using this system + # because it runs out of trace data before any of the tests complete. + + # NOTE: 'http' steg presently cannot be tested using this system + # because the trace pools are process-global rather than per-listener. # Synthesize TimelineTest+TestCase subclasses for every 'tl_*' file in # the test directory.

1 0

[stegotorus/master] first version of stegotorus... binary name still obfsproxy
by zwol＠torproject.org 20 Jul '12

20 Jul '12

commit 2e5825fee2123ad6fcac1fd5aff7b85ea65b5b06 Author: Vinod Yegneswaran <vinod(a)csl.sri.com> Date: Mon Oct 31 22:33:46 2011 +0000 first version of stegotorus... binary name still obfsproxy js + flash steg enabled. git-svn-id: svn+ssh://spartan.csl.sri.com/svn/private/DEFIANCE@107 a58ff0ac-194c-e011-a152-003048836090 --- Makefile.am | 10 +- NOTES | 20 + run-autogen.csh | 3 + src/steg/cookies.c | 230 ++++++++ src/steg/cookies.h | 17 + src/steg/crc32.c | 82 +++ src/steg/crc32.h | 18 + src/steg/jsSteg.c | 1200 ++++++++++++++++++++++++++++++++++++++ src/steg/jsSteg.h | 65 +++ src/steg/payloads.c | 1486 ++++++++++++++++++++++++++++++++++++++++++++++++ src/steg/payloads.h | 159 ++++++ src/steg/pdfSteg.c | 618 ++++++++++++++++++++ src/steg/pdfSteg.h | 29 + src/steg/swfSteg.c | 282 +++++++++ src/steg/swfSteg.c.old | 264 +++++++++ src/steg/swfSteg.h | 42 ++ src/steg/x_http.c.old | 337 +++++++++++ src/steg/x_http2.c | 700 +++++++++++++++++++++++ src/steg/zpack.c | 408 +++++++++++++ src/steg/zpack.h | 14 + start-client.csh | 8 + start-server.csh | 6 + torrc | 12 + 23 files changed, 6009 insertions(+), 1 deletions(-) diff --git a/Makefile.am b/Makefile.am index bcb6e88..e8b68f5 100644 --- a/Makefile.am +++ b/Makefile.am @@ -22,7 +22,15 @@ PROTOCOLS = \ src/protocols/x_rr.c STEGANOGRAPHERS = \ - src/steg/x_http.c + src/steg/x_http.c \ + src/steg/x_http2.c \ + src/steg/payloads.c \ + src/steg/cookies.c \ + src/steg/jsSteg.c \ + src/steg/swfSteg.c \ + src/steg/zpack.c \ + src/steg/crc32.c \ + src/steg/pdfSteg.c libobfsproxy_a_SOURCES = \ src/connections.c \ diff --git a/NOTES b/NOTES new file mode 100644 index 0000000..5b5f23c --- /dev/null +++ b/NOTES @@ -0,0 +1,20 @@ +To Test ObfsProxy: + +start server (start-server.csh) +start client (start-client.csh) +start Tor (copy torrc.sample to etc/tor) + + +Now test tor: +curl --socks4a 127.0.0.1:9060 -o - http://check.torproject.org + + +To add new steg modules: + +1. Add a file below src/steg which implements a steg module; note that +the STEG_DEFINE_MODULE boilerplate macro is mandatory. The name of +the file should be the same as the name of the module (as set by +STEG_DEFINE_MODULE) plus the .c extension. +2. Add the file to the STEGANOGRAPHERS list in Makefile.am. + +That should be all that is necessary. diff --git a/run-autogen.csh b/run-autogen.csh new file mode 100644 index 0000000..9fe8c44 --- /dev/null +++ b/run-autogen.csh @@ -0,0 +1,3 @@ +#!/bin/csh +setenv PATH /usr/local/bin:`echo $PATH` +./autogen.sh diff --git a/src/steg/cookies.c b/src/steg/cookies.c new file mode 100644 index 0000000..e8d43b9 --- /dev/null +++ b/src/steg/cookies.c @@ -0,0 +1,230 @@ + +#include "cookies.h" + +int unwrap_cookie(unsigned char* inbuf, unsigned char* outbuf, int buflen) { + int i,j; + j = 0; + + for (i=0; i < buflen; i++) { + char c = inbuf[i]; + + if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')) + outbuf[j++] = c; + } + + return j; + +} + + + + + + +/* valid cookie characters: anything between between 33 and 126, with the exception of "=" and ';'*/ +/* writes a line of the form XXXX=YYYYY of length cookielen to outbuf, outbuf length >= cookielen */ +/* returns data consumed if success. datalen assumed to be greater than cookielen*/ + + +int gen_one_cookie(unsigned char* outbuf, int cookielen, unsigned char* data, int datalen) { + int sofar = 0; + unsigned char c; + int namelen, vlen; + int data_consumed = 0; + + if (cookielen < 4) + return -1; + + + + if (cookielen > 13) + namelen = rand() % 10 + 1; + else + namelen = rand() % (cookielen - 3) + 1; + + vlen = cookielen - namelen; + + + + while (sofar < namelen) { + c = rand() % (127 - 33) + 33; + if (c == '=' || c == ';' || c == '`' || c == '\'' || c == '%') + continue; + + if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') || (rand () % 4 != 0)) { + if (data_consumed < datalen) + outbuf[sofar++] = data[data_consumed++]; + } + else + outbuf[sofar++] = c; + } + + + outbuf[sofar++] = '='; + + + while (sofar < cookielen) { + c = rand() % (127 - 33) + 33; + if (c == '=' || c == ';' || c == '`' || c == '\'' || c == '%') + continue; + + if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') || (rand() % 4 != 0)) { + if (data_consumed < datalen) + outbuf[sofar++] = data[data_consumed++]; + } + else + outbuf[sofar++] = c; + } + + + + return data_consumed; + +} + + + + +/* dummy version for testing */ +int gen_one_cookie2(unsigned char* outbuf, int cookielen, unsigned char* data, int datalen) { + int i; + + if (cookielen < 4) + return -1; + + if (datalen >= cookielen) { + memcpy(outbuf, data, cookielen); + return cookielen; + } + + memcpy(outbuf, data, datalen); + for (i=datalen; i < cookielen; i++) + outbuf[i] = '+'; + + return datalen; + +} + +/* returns data consumed */ +int gen_cookie_field(unsigned char* outbuf, int total_cookie_len, unsigned char* data, int datalen) { + int rem_cookie_len = total_cookie_len; + int consumed = 0; + + + if (total_cookie_len < 4) { + fprintf(stderr, "error: cookie length too small\n"); + return -1; + } + + while (rem_cookie_len > 4) { + int cookielen = 4 + rand() % (rem_cookie_len - 3); + + int cnt = gen_one_cookie(outbuf, cookielen, data + consumed, datalen - consumed); + + if (cnt < 0) { + fprintf(stderr, "error: couldn't create cookie %d\n", cnt); + return cnt; + } + + + + consumed += cnt; + // fprintf(stderr, "cnt = %d %d %d; consumed = %d\n", cnt, rem_cookie_len, cookielen, consumed); + rem_cookie_len = rem_cookie_len - cookielen; + outbuf += cookielen; + + + + if (rem_cookie_len == 0) { + break; + } + else if (rem_cookie_len <= 5) { + int i = 0; + if ((consumed < datalen) && (consumed % 2 == 1)) { + outbuf[0] = data[consumed]; + consumed++; + outbuf++; + rem_cookie_len--; + } + + for (i=0; i < rem_cookie_len; i++) + outbuf[i] = "ghijklmnopqrstuvwxyzGHIJKLMNOPQRSTUVWXYZ"[rand() % 40]; + + return consumed; + } + + + outbuf[0] = ';'; + outbuf++; + rem_cookie_len--; + } + + + if (consumed % 2 == 1) { + if ((outbuf[-1] >= '0' && outbuf[-1] <= '9') || (outbuf[-1] >= 'a' && outbuf[-1] <= 'f')) { + outbuf[-1] = '*'; + consumed--; + } + else { + outbuf[-1] = data[consumed]; + consumed++; + } + } + + + return consumed; +} + + +/* dummy version for testing */ +int gen_cookie_field2(unsigned char* outbuf, int total_cookie_len, unsigned char* data, int datalen) { + int i; + if (datalen >= total_cookie_len) { + memcpy(outbuf, data, total_cookie_len); + return total_cookie_len; + } + + memcpy(outbuf, data, datalen); + for (i=datalen; i < total_cookie_len; i++) + outbuf[i] = '*'; + + return datalen; +} + + + + +/* + +int main () { + char outbuf[200]; + char data[52] = "1a239023820389023802380389abc2322132321932847203aedf"; + char data2[200]; + // srand(time(NULL)); + srand (20); + + + + int i=0; + + for (i=0; i < 1000000; i++) { + int cookielen = rand()%50 + 5; + bzero(outbuf, sizeof(outbuf)); + int len = gen_cookie_field(outbuf, cookielen, data, sizeof(data)); + // printf("len = %d cookie = %s %d\n", len, outbuf, cookielen); + bzero(data2, sizeof(data2)); + int len2 = unwrap_cookie(outbuf, data2, cookielen); + // printf("unwrapped datalen = %d data = %s\n", len, data2); + + if (len != len2) + printf("hello %d\n", i); + } + + + + +} + +*/ + + diff --git a/src/steg/cookies.h b/src/steg/cookies.h new file mode 100644 index 0000000..4970776 --- /dev/null +++ b/src/steg/cookies.h @@ -0,0 +1,17 @@ +#ifndef _COOKIES_H +#define _COOKIES_H + + + +#include <stdio.h> +#include <strings.h> +#include <stdlib.h> + +int unwrap_cookie(unsigned char* inbuf, unsigned char* outbuf, int buflen); +int gen_cookie_field(unsigned char* outbuf, int total_cookie_len, unsigned char* data, int datalen); +int gen_one_cookie(unsigned char* outbuf, int cookielen, unsigned char* data, int datalen); +int gen_one_cookie2(unsigned char* outbuf, int cookielen, unsigned char* data, int datalen); +int gen_cookie_field2(unsigned char* outbuf, int total_cookie_len, unsigned char* data, int datalen); + + +#endif diff --git a/src/steg/crc32.c b/src/steg/crc32.c new file mode 100644 index 0000000..7fdc847 --- /dev/null +++ b/src/steg/crc32.c @@ -0,0 +1,82 @@ +#include "crc32.h" + +#define CRC32C(c,d) (c=(c>>8)^crc_c[(c^(d))&0xFF]) + +static const unsigned int crc_c[256] = { + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, + 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, + 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, + 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, + 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, + 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, + 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, + 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, + 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, + 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, + 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, + 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, + 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, + 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, + 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, + 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, + 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, + 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, + 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, + 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, + 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, + 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, + 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, + 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, + 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, + 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, + 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, + 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, + 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, + 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, + 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, + 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, + 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, + 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, + 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, + 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, + 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, + 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, + 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, + 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, + 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, + 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, + 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, + 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, + 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, + 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, + 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, + 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, + 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, + 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, + 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, + 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, + 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, + 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, + 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, + 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, + 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d, +}; + + +unsigned int generate_crc32c(char *buffer, size_t length) { + size_t i; + unsigned int crc32 = ~0L; + + for (i = 0; i < length; i++){ + CRC32C(crc32, (unsigned char)buffer[i]); + } + return ~crc32; +} + diff --git a/src/steg/crc32.h b/src/steg/crc32.h new file mode 100644 index 0000000..780e7bd --- /dev/null +++ b/src/steg/crc32.h @@ -0,0 +1,18 @@ +#ifndef __crc32cr_table_h__ +#define __crc32cr_table_h__ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include <sys/types.h> + +#if defined HAVE_STDINT_H +# include <stdint.h> +#elif defined HAVE_INTTYPES_H +# include <inttypes.h> +#endif + +unsigned int generate_crc32c(char *string, size_t length); + +#endif diff --git a/src/steg/jsSteg.c b/src/steg/jsSteg.c new file mode 100644 index 0000000..5946062 --- /dev/null +++ b/src/steg/jsSteg.c @@ -0,0 +1,1200 @@ +#include "payloads.h" +#include "jsSteg.h" +#include "cookies.h" + + +/* + * jsSteg: A Javascript-based steganography module + * + */ + + +/* + * int isxString(char *str) + * + * description: + * return 1 if all char in str are hexadecimal + * return 0 otherwise + * + */ +int isxString(char *str) { + unsigned int i; + char *dp = str; + for (i=0; i<strlen(str); i++) { + if (! isxdigit(*dp) ) { + return 0; + } + } + return 1; +} + + +/* + * int encode(char *data, char *jTemplate, char *jData, + * unsigned int dlen, unsigned int jtlen, unsigned int jdlen) + * + * description: + * embed hex-encoded data (data) in the input Javascript (jTemplate) + * and put the result in jData + * function returns the number of characters in data successfully + * embedded in jData, or returns one of the error codes + * + * approach: + * replaces characters in jTemplate that are hexadecimal (i.e., {0-9,a-f,A-F}) + * with those in data, and leave the non-hex char in place + * + * input: + * - data[] : hex data to hide + * - dlen : size of data + * - jTemplate[] : Javascript + * - jlen : size of jTemplate + * - jdlen : size of jData, output buffer + * + * output: + * - jData : result of encoding data in jTemplate + * + * assumptions: + * - data is hex-encoded + * + * exceptions: + * - if (jdlen < jtlen) return INVALID_BUF_SIZE + * - if (data contains non-hex char) return INVALID_DATA_CHAR + * + * example: + * data = "0123456789ABCDEF" + * jTemplate = "dfp_ord=Math.random()*10000000000000000; dfp_tile = 1;" + * encode() returns 16 + * jData = "01p_or2=M3th.r4n5om()*6789ABCDEF0000000; dfp_tile = 1;" + * + */ +int encode(char *data, char *jTemplate, char *jData, + unsigned int dlen, unsigned int jtlen, unsigned int jdlen ) +{ + unsigned int encCnt = 0; /* num of data encoded in jData */ + char *dp, *jtp, *jdp; /* current pointers for data, jTemplate, and jData */ + + unsigned int j; + + /* + * insanity checks + */ + if (jdlen < jtlen) { return INVALID_BUF_SIZE; } + + dp = data; jtp = jTemplate; jdp = jData; + + if (! isxString(dp) ) { return INVALID_DATA_CHAR; } + + /* handling boundary case: dlen == 0 */ + if (dlen < 1) { return 0; } + + + for (j=0; j<jtlen; j++) { + /* found a hex char in jTemplate that can be used for encoding data */ + if ( isxdigit(*jtp) ) { + *jdp = *dp; + dp++; + encCnt++; + if (encCnt == dlen) { + jtp++; jdp++; + break; + } + } else { + *jdp = *jtp; + } + jtp++; jdp++; + } + + + /* copying the rest of jTemplate to jdata */ + while (jtp < (jTemplate+jtlen)) { + *jdp++ = *jtp++; + } + + return encCnt; +} + + +#define startScriptTypeJS "<script type=\"text/javascript\">" +#define endScriptTypeJS "</script>" +// #define JS_DELIMITER "?" +// #define JS_DELIMITER_REPLACEMENT "." + + +/* + * similar to encode(), but uses offset2Hex to look for usable hex char + * in JS for encoding. See offset2Hex for what hex char are considered + * usable. encode() also converts JS_DELIMITER that appears in the + * the JS to JS_DELIMITER_REPLACEMENT, before all the data is encoded. + * + * Output: + * fin - signal the caller whether all data has been encoded and + * a JS_DELIMITER has been added + */ +int encode2(char *data, char *jTemplate, char *jData, + unsigned int dlen, unsigned int jtlen, + unsigned int jdlen, int *fin) +{ + unsigned int encCnt = 0; /* num of data encoded in jData */ + char *dp, *jtp, *jdp; /* current pointers for data, jTemplate, and jData */ + int i,j; + + /* + * insanity checks + */ + if (jdlen < jtlen) { return INVALID_BUF_SIZE; } + + dp = data; jtp = jTemplate; jdp = jData; + + if (! isxString(dp) ) { return INVALID_DATA_CHAR; } + + /* handling boundary case: dlen == 0 */ + if (dlen < 1) { return 0; } + + + i = offset2Hex(jtp, (jTemplate+jtlen)-jtp, 0); + while (encCnt < dlen && i != -1) { + // copy next i char from jtp to jdp, + // except that if *jtp==JS_DELIMITER, copy + // JS_DELIMITER_REPLACEMENT to jdp instead + j = 0; + while (j < i) { + if (*jtp == JS_DELIMITER) { + *jdp = JS_DELIMITER_REPLACEMENT; + } else { + *jdp = *jtp; + } + jtp = jtp + 1; jdp = jdp + 1; j++; + } + + *jdp = *dp; + encCnt++; + dp = dp + 1; jtp = jtp + 1; jdp = jdp + 1; + + i = offset2Hex(jtp, (jTemplate+jtlen)-jtp, 1); + } + + + + // copy the rest of jTemplate to jdata + // if we've encoded all data, replace the first + // char in jTemplate by JS_DELIMITER, if needed, + // to signal the end of data encoding + +#ifdef DEBUG2 + printf("encode2: encCnt = %d; dlen = %d\n", encCnt, dlen); +#endif + + *fin = 0; + if (encCnt == dlen) { + // replace the next char in jTemplate by JS_DELIMITER + if (jtp < (jTemplate+jtlen)) { + *jdp = JS_DELIMITER; + } + jdp = jdp+1; jtp = jtp+1; + *fin = 1; + } + + while (jtp < (jTemplate+jtlen)) { + if (*jtp == JS_DELIMITER) { + if (encCnt < dlen) { + *jdp = JS_DELIMITER_REPLACEMENT; + } else { + *jdp = *jtp; + } + // else if (isxdigit(*jtp)) { + // if (encCnt < dlen && *fin == 0) { + // *jdp = JS_DELIMITER; + // *fin = 1; + // } else { + // *jdp = *jtp; + // } + // } + } else { + *jdp = *jtp; + } + jdp = jdp+1; jtp = jtp+1; + } + +#ifdef DEBUG2 + printf("encode2: encCnt = %d; dlen = %d\n", encCnt, dlen); + printf("encode2: fin= %d\n", *fin); +#endif + + return encCnt; + +} + + + +int encodeHTTPBody(char *data, char *jTemplate, char *jData, + unsigned int dlen, unsigned int jtlen, + unsigned int jdlen, int mode) +{ + char *dp, *jtp, *jdp; // current pointers for data, jTemplate, and jData + unsigned int encCnt = 0; // num of data encoded in jData + int n; // tmp for updating encCnt + char *jsStart, *jsEnd; + int skip; + int scriptLen; + int fin; + unsigned int dlen2 = dlen; + dp = data; + jtp = jTemplate; + jdp = jData; + + + if (mode == CONTENT_JAVASCRIPT) { + // assumption: the javascript pertaining to jTemplate has enough capacity + // to encode jData. thus, we only invoke encode() once here. + encCnt = encode2(dp, jtp, jdp, dlen, jtlen, jdlen, &fin); + // ensure that all dlen char from data have been encoded in jData +#ifdef DEBUG + if (encCnt != dlen || fin == 0) { + printf("Problem encoding all data to the JS\n"); + } +#endif + return encCnt; + + } + + else if (mode == CONTENT_HTML_JAVASCRIPT) { + while (encCnt < dlen2) { + jsStart = strstr(jtp, startScriptTypeJS); + if (jsStart == NULL) { +#ifdef DEBUG + printf("lack of usable JS; can't find startScriptType\n"); +#endif + return encCnt; + } + skip = strlen(startScriptTypeJS)+jsStart-jtp; +#ifdef DEBUG2 + printf("copying %d (skip) char from jtp to jdp\n", skip); +#endif + memcpy(jdp, jtp, skip); + jtp = jtp+skip; jdp = jdp+skip; + jsEnd = strstr(jtp, endScriptTypeJS); + if (jsEnd == NULL) { +#ifdef DEBUG + printf("lack of usable JS; can't find endScriptType\n"); +#endif + return encCnt; + } + + // the JS for encoding data is between jsStart and jsEnd + scriptLen = jsEnd - jtp; + // n = encode2(dp, jtp, jdp, dlen, jtlen, jdlen, &fin); + n = encode2(dp, jtp, jdp, dlen, scriptLen, jdlen, &fin); + // update encCnt, dp, and dlen based on n + if (n > 0) { + encCnt = encCnt+n; dp = dp+n; dlen = dlen-n; + } + // update jtp, jdp, jdlen + skip = jsEnd-jtp; + jtp = jtp+skip; jdp = jdp+skip; jdlen = jdlen-skip; + skip = strlen(endScriptTypeJS); + memcpy(jdp, jtp, skip); + jtp = jtp+skip; jdp = jdp+skip; jdlen = jdlen-skip; + } + + // copy the rest of jTemplate to jdp + skip = jTemplate+jtlen-jtp; + + // handling the boundary case in which JS_DELIMITER hasn't been + // added by encode() + if (fin == 0 && dlen == 0) { + if (skip > 0) { + *jtp = JS_DELIMITER; + jtp = jtp+1; jdp = jdp+1; + skip--; + } + } + memcpy(jdp, jtp, skip); + return encCnt; + + } else { + log_warn("Unknown mode (%d) for encode2()", mode); + return 0; + } + + +} + +/* + * int decode(char *jData, char *dataBuf, + * unsigned int jdlen, unsigned int dlen, unsigned int dataBufSize) + * + * description: + * extract hex char from Javascript embedded with data (jData) + * and put the result in dataBuf + * function returns the number of hex char extracted from jData + * to dataBuf, or returns one of the error codes + * + * input: + * - jData[]: Javascript embedded with hex-encoded data + * - jdlen : size of jData + * - dlen : size of data to recover + * - dataBufSize : size of output data buffer (dataBuf) + * + * output: + * - dataBuf[] : output buffer for recovered data + * + * assumptions: + * - data is hex-encoded + * + * exceptions: + * - if (dlen > dataBufSize) return INVALID_BUF_SIZE + * + * example: + * jData = "01p_or2=M3th.r4n5om()*6789ABCDEF0000000; dfp_tile = 1;" + * jdlen = 54 + * dlen = 16 + * dataBufSize = 1000 + * decode() returns 16 + * dataBuf= "0123456789ABCDEF" + * + */ +int decode (char *jData, char *dataBuf, unsigned int jdlen, + unsigned int dlen, unsigned int dataBufSize ) +{ + unsigned int decCnt = 0; /* num of data decoded */ + char *dp, *jdp; /* current pointers for dataBuf and jData */ + unsigned int j; + + if (dlen > dataBufSize) { return INVALID_BUF_SIZE; } + + dp = dataBuf; jdp = jData; + for (j=0; j<jdlen; j++) { + if ( isxdigit(*jdp) ) { + if (decCnt < dlen) { + decCnt++; + *dp++ = *jdp++; + } else { + break; + } + } else { + jdp++; + } + } + return decCnt; +} + + +/* + * decode2() is similar to decode(), but uses offset2Hex to look for + * applicable hex char in JS for decoding. Also, the decoding process + * stops when JS_DELIMITER is encountered. + */ +int decode2 (char *jData, char *dataBuf, unsigned int jdlen, + unsigned int dataBufSize, int *fin ) +{ + unsigned int decCnt = 0; /* num of data decoded */ + char *dp, *jdp; /* current pointers for dataBuf and jData */ + int i,j; + int cjdlen = jdlen; + + *fin = 0; + dp = dataBuf; jdp = jData; + + i = offset2Hex(jdp, cjdlen, 0); + while (i != -1) { + // return if JS_DELIMITER exists between jdp and jdp+i + for (j=0; j<i; j++) { + if (*jdp == JS_DELIMITER) { + *fin = 1; + return decCnt; + } + jdp = jdp+1; cjdlen--; + } + // copy hex data from jdp to dp + if (dataBufSize <= 0) { + return decCnt; + } + *dp = *jdp; + jdp = jdp+1; cjdlen--; + dp = dp+1; dataBufSize--; + decCnt++; + + // find the next hex char + i = offset2Hex(jdp, cjdlen, 1); + } + + // look for JS_DELIMITER between jdp to jData+jdlen + while (jdp < jData+jdlen) { + if (*jdp == JS_DELIMITER) { + *fin = 1; + break; + } + jdp = jdp+1; + } + + return decCnt; +} + + +int decodeHTTPBody (char *jData, char *dataBuf, unsigned int jdlen, + unsigned int dataBufSize, int *fin, int mode ) +{ + char *jsStart, *jsEnd; + char *dp, *jdp; // current pointers for data and jData + int scriptLen; + int decCnt = 0; + int n; + int dlen = dataBufSize; + dp = dataBuf; jdp = jData; + + if (mode == CONTENT_JAVASCRIPT) { + decCnt = decode2(jData, dataBuf, jdlen, dataBufSize, fin); + if (*fin == 0) { + log_warn("Unable to find JS_DELIMITER"); + } + } + else if (mode == CONTENT_HTML_JAVASCRIPT) { + *fin = 0; + while (*fin == 0) { + jsStart = strstr(jdp, startScriptTypeJS); + if (jsStart == NULL) { +#ifdef DEBUG + printf("Can't find startScriptType for decoding data inside script type JS\n"); +#endif + return decCnt; + } + jdp = jsStart+strlen(startScriptTypeJS); + jsEnd = strstr(jdp, endScriptTypeJS); + if (jsEnd == NULL) { +#ifdef DEBUG + printf("Can't find endScriptType for decoding data inside script type JS\n"); +#endif + return decCnt; + } + + // the JS for decoding data is between jsStart and jsEnd + scriptLen = jsEnd - jdp; + n = decode2(jdp, dp, scriptLen, dlen, fin); + if (n > 0) { + decCnt = decCnt+n; dlen=dlen-n; dp=dp+n; + } + jdp = jsEnd+strlen(endScriptTypeJS); + } // while (*fin==0) + } else { + log_warn("Unknown mode (%d) for encode2()", mode); + return 0; + } + + return decCnt; +} + + + + + +void printerr(int errno) { + if (errno == INVALID_BUF_SIZE) { + printf ("Error: Output buffer too small\n"); + } + else if (errno == INVALID_DATA_CHAR) { + printf ("Error: Non-hex char in data\n"); + } + else { + printf ("Unknown error: %i\n", errno); + } +} + + +int testEncode(char *data, char *js, char *outBuf, unsigned int dlen, unsigned int jslen, + unsigned int outBufLen, int testNum) { + int r; + + printf ("***** Start of testEncode (%i) *****\n", testNum); + printf ("Input:\n"); + printf ("data = %s\n", data); + printf ("data len = %i\n", dlen); + printf ("js = %s\n", js); + printf ("js len = %i\n", jslen); + r = encode (data, js, outBuf, dlen, jslen, outBufLen); + if (r < 0) { + printerr(r); + } else { + printf ("\nOutput:\n"); + printf ("%i char of data embedded in outBuf\n", r); + outBuf[jslen] = '\0'; + printf ("outBuf = %s\n", outBuf); + } + printf ("***** End of testEncode (%i) *****\n", testNum); + return r; +} + +int testDecode(char *inBuf, char *outBuf, unsigned int inBufSize, unsigned int dlen, + unsigned int outBufSize, int testNum) { + + int r; + + printf ("***** Start of testDecode (%i) *****\n", testNum); + printf ("Input:\n"); + printf ("inBuf = %s\n", inBuf); + printf ("inBuf size = %i\n", inBufSize); + printf ("data len = %i\n", dlen); + printf ("outBuf size = %i\n", outBufSize); + r = decode(inBuf, outBuf, inBufSize, dlen, outBufSize); + if (r < 0) { + printerr(r); + } else { + printf ("\nOutput:\n"); + printf ("%i char of data recovered from inBuf (to outBuf)\n", r); + outBuf[r] = '\0'; + printf ("outBuf = %s\n", outBuf); + } + printf ("***** End of testDecode (%i) *****\n", testNum); + return r; +} + + +int testEncode2(char *data, char *js, char *outBuf, + unsigned int dlen, unsigned int jslen, unsigned int outBufLen, + int mode, int testNum) { + int r; + // int fin; + + printf ("***** Start of testEncode2 (%i) *****\n", testNum); + printf ("Input:\n"); + printf ("data = %s\n", data); + printf ("data len = %i\n", dlen); + printf ("js = %s\n", js); + printf ("js len = %i\n", jslen); + // r = encode2(data, js, outBuf, dlen, jslen, outBufLen, &fin); + r = encodeHTTPBody(data, js, outBuf, dlen, jslen, outBufLen, mode); + + if (r < 0) { + printerr(r); + } + else { + printf ("\nOutput:\n"); + printf ("%i char of data embedded in outBuf\n", r); + // printf ("fin = %d\n", fin); + outBuf[jslen] = '\0'; + printf ("outBuf = %s\n", outBuf); + + if ((unsigned int) r < dlen) { + printf ("Incomplete data encoding\n"); + } + } + printf ("***** End of testEncode (%i) *****\n", testNum); + return r; +} + + + + +int testDecode2(char *inBuf, char *outBuf, + unsigned int inBufSize, unsigned int outBufSize, + int mode, int testNum) { + int r; + int fin; + + printf ("***** Start of testDecode2 (%i) *****\n", testNum); + printf ("Input:\n"); + printf ("inBuf = %s\n", inBuf); + printf ("inBuf size = %i\n", inBufSize); + printf ("outBuf size = %i\n", outBufSize); + r = decodeHTTPBody(inBuf, outBuf, inBufSize, outBufSize, &fin, mode); + if (r < 0) { + printerr(r); + } else { + printf ("\nOutput:\n"); + printf ("%i char of data recovered from inBuf (to outBuf)\n", r); + outBuf[r] = '\0'; + printf ("outBuf = %s\n", outBuf); + } + printf ("***** End of testDecode2 (%i) *****\n", testNum); + return r; +} + + +int +x_http2_server_JS_transmit (steg_t* s, struct evbuffer *source, conn_t *conn) { + + struct evbuffer_iovec *iv; + int nv; + struct evbuffer *dest = conn_get_outbound(conn); + size_t sbuflen = evbuffer_get_length(source); + char outbuf[HTTP_MSG_BUF_SIZE]; + char data[(int) sbuflen*2]; + unsigned int datalen; + + size_t sofar = 0; + unsigned int cnt = 0; + int r; + + + + // by dynamically configuring the size returned by x_http2_transmit_room(), + // we can ensure that the size of data (source) is less than the max + // data length the JavaScript (jsTemplate) can encode + + + + // do { + int i; + unsigned int jsLen; + int mode; + char *hend; + unsigned int hLen; + unsigned int mjs; + + char *jsTemplate = NULL; + int jsTemplateSize = 0; + + + + + /* int hdrLen; + int content_len; + int fin2; + + char* tmp; + char data2[20000]; + int decCnt; + */ + + datalen = 0; + + + log_debug("sbuflen = %d sofar = %d\n", (int) sbuflen, (int) sofar); + + // log_debug("SERVER: dumping data with length %d:", (int) sbuflen); + // evbuffer_dump(source, stderr); + // Convert data in 'source' to hexadecimal and write it to data + + nv = evbuffer_peek(source, sbuflen - sofar, NULL, NULL, 0); + iv = xzalloc(sizeof(struct evbuffer_iovec) * nv); + + if (evbuffer_peek(source, sbuflen - sofar, NULL, iv, nv) != nv) { + free(iv); + return -1; + } + + // jsTemplate should be init already, by x_http2_new or the previous invocation + // of this function + + mjs = get_max_JS_capacity(); + + if (mjs <= 0) { + log_debug("SERVER ERROR: (server_transmit) No JavaScript found in jsTemplate\n"); + return -1; + } + + if (sbuflen > (size_t) mjs) { + log_debug("SERVER ERROR: (server_transmit) jsTemplate cannot accommodate data %d %dn", + (int) sbuflen, (int) mjs); + return -1; + } + + + cnt = 0; + + for (i = 0; i < nv; i++) { + const unsigned char *p = iv[i].iov_base; + const unsigned char *limit = p + iv[i].iov_len; + char c; + + while (p < limit && cnt < sbuflen-sofar) { + c = *p++; + data[datalen] = "0123456789abcdef"[(c & 0xF0) >> 4]; + data[datalen+1] = "0123456789abcdef"[(c & 0x0F) >> 0]; + datalen += 2; + cnt++; + } + } + + log_debug("SERVER encoded data in hex string (len %d):", datalen); + // buf_dump((unsigned char*)data, datalen, stderr); + + free(iv); + + + + if (get_payload(HTTP_CONTENT_JAVASCRIPT, datalen, &jsTemplate, &jsTemplateSize) == 1) { + log_debug("SERVER found the next HTTP response template with size %d", jsTemplateSize); + } else { + log_debug("SERVER couldn't find the next HTTP response template; reusing the previous one"); + } + + log_debug("MJS %d %d", datalen, mjs); + + + + if (jsTemplate == NULL) { + fprintf(stderr, "NO suitable payload found %d %d\n", datalen, mjs); + exit(-1); + } + + + // use encodeHTTPBody to embed data in the JS jsTemplate + // assumption: jsTemplate is null-terminated + + log_debug("strlen(jsTemplate) = %d", (int)strlen(jsTemplate)); + log_debug("jsTemplateSize = %d", jsTemplateSize); + // unsigned int jsLen = strlen(jsTemplate); + jsLen = jsTemplateSize; + + + mode = has_eligible_HTTP_content (jsTemplate, jsLen, HTTP_CONTENT_JAVASCRIPT); + hend = strstr(jsTemplate, "\r\n\r\n"); + if (hend == NULL) { + log_warn("Unable to find end of header in the HTTP template"); + return -1; + } + + log_debug("SERVER: using HTTP resp template of length = %d\n", jsLen); + // log_debug("HTTP resp tempmlate:"); + // buf_dump((unsigned char*)jsTemplate, jsLen, stderr); + // fprintf(stderr, "==========================\n"); + + hLen = hend+4-jsTemplate; + r = encodeHTTPBody(data, hend+4, outbuf, datalen, jsLen-hLen, HTTP_MSG_BUF_SIZE, mode); + + + + + /// NEW STUFF + + +/* hdrLen = strstr(jsTemplate, "\r\n\r\n") - jsTemplate + 4; + tmp = strstr(jsTemplate, "Content-Length: ") + strlen("Content-Length: "); + + content_len = atoi(tmp); + + + decCnt = decodeHTTPBody(jsTemplate + hdrLen, data2, content_len, HTTP_MSG_BUF_SIZE, &fin2, mode); + + + if (decCnt == (int) datalen) + fprintf(stderr, "cnts match\n"); + else + fprintf(stderr, "cnts don't match %d %d\n", decCnt, datalen); + +*/ + + + if (r < 0 || ((unsigned int) r < datalen)) { + fprintf(stderr, "incomplete data encoding\n"); + exit(-1); + log_debug("SERVER ERROR: Incomplete data encoding"); + return -1; + } + + // note: the transformation is length-preserving for now + log_debug("SERVER: HTTP body with encoded data:"); + // buf_dump((unsigned char*)outbuf, jsLen-hLen, stderr); + // fprintf(stderr, "==========================\n"); + + if (evbuffer_add(dest, jsTemplate, hLen)) { + log_debug("SERVER ERROR: x_http2_server_transmit: evbuffer_add() fails for jsTemplate"); + return -1; + } + + // fprintf(stderr, "HELLO ==========================\n"); + + if (evbuffer_add(dest, outbuf, jsLen-hLen)) { + log_debug("SERVER ERROR: x_http2_server_transmit: evbuffer_add() fails for outbuf"); + return -1; + } + + sofar += datalen/2; + evbuffer_drain(source, datalen/2); + // } while (sbuflen > sofar); + + + + // fprintf(stderr, "SERVER TRANSMITTED payload of size %d\n", (int) sbuflen); + + // obtain a usable HTTP response template for the next data, and + // modify jsTemplateCapacity + + + log_debug("SERVER finding the next HTTP response template"); + + + + + + // conn_cease_transmission(conn); + conn_close_after_transmit(conn); + // downcast_steg(s)->have_transmitted = 1; + return 0; +} + + + + + + +int +x_http2_handle_client_JS_receive(steg_t *s, conn_t *conn, struct evbuffer *dest, struct evbuffer* source) { + struct evbuffer_ptr s2; + unsigned int response_len = 0; + unsigned int content_len = 0; + unsigned int hdrLen; + char buf[10]; + char respMsg[HTTP_MSG_BUF_SIZE]; + char data[HTTP_MSG_BUF_SIZE]; + + unsigned char *field; + unsigned char *fieldStart; + unsigned char* fieldEnd; + unsigned char *fieldValStart; + char *httpBody; + + ev_ssize_t r; + int mode, decCnt, fin; + struct evbuffer * scratch; + int i,j,k; + char c; + + + s2 = evbuffer_search(source, "\r\n\r\n", sizeof ("\r\n\r\n") -1 , NULL); + if (s2.pos == -1) { + log_debug("CLIENT Did not find end of HTTP header %d", (int) evbuffer_get_length(source)); + // evbuffer_dump(source, stderr); + return RECV_INCOMPLETE; + } + + log_debug("CLIENT received response header with len %d", (int)s2.pos); + + response_len = 0; + hdrLen = s2.pos + strlen("\r\n\r\n"); + response_len += hdrLen; + + // get content length, e.g., Content-Length: 22417 + field = evbuffer_pullup(source, s2.pos); + if (field == NULL) { + log_debug("CLIENT unable to pullup the complete HTTP header"); + return RECV_BAD; + } + + fieldStart = (unsigned char*) strstr((char*) field, "Content-Length: "); + if (fieldStart == NULL) { + log_debug("CLIENT unable to find Content-Length in the header"); + return RECV_BAD; + } + + fieldEnd = (unsigned char*) strstr((char *)fieldStart, "\r\n"); + if (fieldEnd == NULL) { + log_debug("CLIENT unable to find end of line for Content-Length"); + return RECV_BAD; + } + + fieldValStart = fieldStart+strlen("Content-Length: "); + if ((unsigned int) (fieldEnd-fieldValStart) > (sizeof(buf)-1)) { + log_debug("CLIENT: Value of Content-Length too large"); + return RECV_BAD; + } + memcpy(buf, fieldValStart, fieldEnd-fieldValStart); + buf[fieldEnd-fieldValStart] = 0; + + content_len = atoi(buf); + log_debug("CLIENT received Content-Length = %d\n", content_len); + + response_len += content_len; + + if (response_len > evbuffer_get_length(source)) + return RECV_INCOMPLETE; + + + + // read the entire HTTP resp + if (response_len < HTTP_MSG_BUF_SIZE) { + r = evbuffer_copyout(source, respMsg, response_len); + log_debug("CLIENT %d char copied from source to respMsg (expected %d)", (int)r, response_len); + if (r < 0) { + log_debug("CLIENT ERROR: evbuffer_copyout fails"); + return RECV_INCOMPLETE; + } + if (r < response_len) { + log_debug("CLIENT: evbuffer_copyout incomplete; got %d instead of %d", (int)r, response_len); + return RECV_INCOMPLETE; + } + respMsg[response_len] = 0; + } else { + log_debug("CLIENT: HTTP response too large to handle"); + return RECV_BAD; + } + + log_debug("CLIENT received HTTP response with length %d\n", response_len); + log_debug("HTTP response:"); + // buf_dump((unsigned char*)respMsg, response_len, stderr); + // fprintf(stderr, "==========================\n"); + + httpBody = respMsg + hdrLen; + + log_debug("CLIENT Before has_eligible HTTP content\n"); + // find out if the resp is a JavaScript or JS embedded in HTML doc + mode = has_eligible_HTTP_content (respMsg, response_len, HTTP_CONTENT_JAVASCRIPT); + if (mode != 1 && mode != 2) { + log_debug("CLIENT ERROR: HTTP response not useful for jsSteg (mode %d)", mode); + return RECV_BAD; + } + + log_debug("CLIENT Before decodeHTTPBody; mode: %d\n", mode); + + // call decodeHTTPBody + decCnt = decodeHTTPBody(httpBody, data, response_len-hdrLen, HTTP_MSG_BUF_SIZE, &fin, mode); + data[decCnt] = 0; + + log_debug("After decodeHTTPBody; decCnt: %d\n", decCnt); + + // decCnt is an odd number or data is not a hex string + if (decCnt % 2) { + log_debug("CLIENT ERROR: An odd number of hex characters received"); + // buf_dump((unsigned char*)data, decCnt, stderr); + return RECV_BAD; + } + + if (! isxString(data)) { + log_debug("CLIENT ERROR: Data received not hex"); + // buf_dump((unsigned char*)data, decCnt, stderr); + return RECV_BAD; + } + + log_debug("Hex data received:"); + // buf_dump ((unsigned char*)data, decCnt, stderr); + + // get a scratch buffer + scratch = evbuffer_new(); + if (!scratch) return RECV_BAD; + + if (evbuffer_expand(scratch, decCnt/2)) { + log_debug("CLIENT ERROR: Evbuffer expand failed \n"); + evbuffer_free(scratch); + return RECV_BAD; + } + + // convert hex data back to binary + for (i=0, j=0; i< decCnt; i=i+2, ++j) { + sscanf(&data[i], "%2x", (unsigned int*) &k); + c = (char)k; + evbuffer_add(scratch, &c, 1); + } + + log_debug("CLIENT Done converting hex data to binary:\n"); + // evbuffer_dump(scratch, stderr); + + + // fprintf(stderr, "CLIENT RECEIVED payload of size %d\n", (int) evbuffer_get_length(scratch)); + // add the scratch buffer (which contains the data) to dest + + if (evbuffer_add_buffer(dest, scratch)) { + evbuffer_free(scratch); + log_debug("CLIENT ERROR: Failed to transfer buffer"); + return RECV_BAD; + } + log_debug("Added scratch (buffer) to dest\n"); + + evbuffer_free(scratch); + + + if (response_len <= evbuffer_get_length(source)) { + if (evbuffer_drain(source, response_len) == -1) { + log_debug("CLIENT ERROR: Added scratch (buffer) to dest\n"); + return RECV_BAD; + } + } + else { + log_warn("response_len > buffer size... can't drain"); + exit(-1); + } + + + log_debug("Drained source for %d char\n", response_len); + + // downcast_steg(s)->have_received = 1; + conn_expect_close(conn); + + return RECV_GOOD; +} + + + +//int +//x_http2_handle_server_JS_receive(steg_t *s, conn_t *conn, struct evbuffer *dest, struct evbuffer* source) { +// +// int cnt = 0; +// +// do { +// struct evbuffer_ptr s2 = evbuffer_search(source, "\r\n\r\n", sizeof ("\r\n\r\n") -1 , NULL); +// unsigned char* data; +// unsigned char* limit; +// unsigned char *p; +// int unwrapped_cookie_len; +// struct evbuffer *scratch; +// unsigned char c, h, secondhalf; +// unsigned char buf[evbuffer_get_length(source)]; +// +// +// if (s2.pos == -1) { +// log_debug("Did not find end of request %d", (int) evbuffer_get_length(source)); +// // evbuffer_dump(source, stderr); +// return RECV_INCOMPLETE; +// } +// +// log_debug("SERVER received request header of length %d", (int)s2.pos); +// +// data = evbuffer_pullup(source, s2.pos); +// if (data == NULL) { +// log_debug("SERVER evbuffer_pullup fails"); +// return RECV_BAD; +// } +// +// limit = data + s2.pos; +// +// data = (unsigned char*) strstr((char*) data, "Cookie:"); +// +// if (data == NULL || memcmp(data, "Cookie:", sizeof "Cookie:"-1)) { +// log_debug("Unexpected HTTP verb: %.*s", 5, data); +// return RECV_BAD; +// } +// +// p = data + sizeof "Cookie: "-1; +// unwrapped_cookie_len = unwrap_cookie(p, buf, (int) (limit - p)); +// +// log_debug("SERVER: received cookie of length = %d %d\n", unwrapped_cookie_len, (int) (limit-p)); +// // buf_dump(buf, unwrapped_cookie_len, stderr); +// // fprintf(stderr, "==========================\n"); +// // buf_dump(p, (int) (limit-p), stderr); +// +// +// // log_debug("hello SERVER received %d cnt = %d\n", (int) (limit - p), cnt); +// // buf_dump(p, (int) (limit-p), stderr); +// +// /* We need a scratch buffer here because the contract is that if +// we hit a decode error we *don't* write anything to 'dest'. */ +// scratch = evbuffer_new(); +// +// if (!scratch) return RECV_BAD; +// +// +// if (evbuffer_expand(scratch, unwrapped_cookie_len/2)) { +// log_debug("Evbuffer expand failed \n"); +// evbuffer_free(scratch); +// return RECV_BAD; +// } +// p = buf; +// +// +// secondhalf = 0; +// while ((int) (p - buf) < unwrapped_cookie_len) { +// if (!secondhalf) c = 0; +// if ('0' <= *p && *p <= '9') h = *p - '0'; +// else if ('a' <= *p && *p <= 'f') h = *p - 'a' + 10; +// else if ('A' <= *p && *p <= 'F') h = *p - 'A' + 10; +// else if (*p == '=' && !secondhalf) { +// p++; +// continue; +// } else { +// evbuffer_free(scratch); +// log_debug("Decode error: unexpected URI characterasdfaf %d", *p); +// return RECV_BAD; +// } +// +// c = (c << 4) + h; +// if (secondhalf) { +// evbuffer_add(scratch, &c, 1); +// // log_debug("adding to scratch"); +// cnt++; +// } +// secondhalf = !secondhalf; +// p++; +// } +// +// +// +// if (evbuffer_add_buffer(dest, scratch)) { +// evbuffer_free(scratch); +// log_debug("Failed to transfer buffer"); +// return RECV_BAD; +// } +// evbuffer_drain(source, s2.pos + sizeof("\r\n\r\n") - 1); +// evbuffer_free(scratch); +// } while (evbuffer_get_length(source)); +// +// +// log_debug("SERVER RECEIVED payload %d\n", cnt); +// // downcast_steg(s)->have_received = 1; +// conn_transmit_soon(conn, 100); +// return RECV_GOOD; +//} + + + + +/***** + int + main() { + int jDataSize = 1000; + char jData[jDataSize]; + int outDataBufSize = 1000; + char outDataBuf[outDataBufSize]; + + int r; + // test case 1: data embedded in javascript + r = testEncode2(data1, js1, jData, strlen(data1), strlen(js1), jDataSize, + CONTENT_JAVASCRIPT, 1); + if (r > 0) { testDecode2(jData, outDataBuf, strlen(js1), outDataBufSize, CONTENT_JAVASCRIPT, 1); } + + // test case 4: data embedded in one script type javascript + r = testEncode2(data1, js4, jData, strlen(data1), strlen(js4), jDataSize, + CONTENT_HTML_JAVASCRIPT, 4); + if (r > 0) { testDecode2(jData, outDataBuf, strlen(js4), outDataBufSize, CONTENT_HTML_JAVASCRIPT, 4); } + + // test case 5: data embedded in one script type javascript + r = testEncode2(data1, js5, jData, strlen(data1), strlen(js5), jDataSize, + CONTENT_HTML_JAVASCRIPT, 5); + if (r > 0) { testDecode2(jData, outDataBuf, strlen(js5), outDataBufSize, CONTENT_HTML_JAVASCRIPT, 5); } + + + return 0; + } +*****/ + +/***** + int + main() { + int jDataSize = 1000; + char jData[jDataSize]; + int jDataSmallSize = 5; + char jDataSmall[jDataSmallSize]; + + int outDataBufSize = 1000; + char outDataBuf[outDataBufSize]; + int outDataSmallSize = 5; + char outDataSmall[outDataSmallSize]; + + int r; + + // test case 1: data embedded in javascript + r = testEncode(data1, js1, jData, strlen(data1), strlen(js1), jDataSize, 1); + if (r > 0) { testDecode(jData, outDataBuf, strlen(js1), r, outDataBufSize, 1); } + + // test case 2: data embedded in javascript + r = testEncode(data1, js2, jData, strlen(data1), strlen(js2), jDataSize, 2); + if (r > 0) { testDecode(jData, outDataBuf, strlen(js2), r, outDataBufSize, 2); } + + // test case 3: data partially embedded in javascript; num of hex char in js < data len + r = testEncode(data1, js3, jData, strlen(data1), strlen(js3), jDataSize, 3); + if (r > 0) { testDecode(jData, outDataBuf, strlen(js3), r, outDataBufSize, 3); } + + // test case 4: data embedded in javascript; larger data + r = testEncode(data2, js1, jData, strlen(data2), strlen(js1), jDataSize, 4); + if (r > 0) { testDecode(jData, outDataBuf, strlen(js1), r, outDataBufSize, 4); } + + // test case 5 (for encode): err for non-hex data + testEncode(nonhexstr, js1, jData, strlen(nonhexstr), strlen(js1), jDataSize, 5); + + // test case 6 (for encode): err for small output buf + testEncode(data1, js1, jDataSmall, strlen(data1), strlen(js1), jDataSmallSize, 6); + + // test case 7 (for decode): err for small output buf + r = testEncode(data1, js1, jData, strlen(data1), strlen(js1), jDataSize, 7); + if (r > 0) { testDecode(jData, outDataSmall, strlen(js1), r, outDataSmallSize, 7); } + } +*****/ + diff --git a/src/steg/jsSteg.h b/src/steg/jsSteg.h new file mode 100644 index 0000000..3c5f6ae --- /dev/null +++ b/src/steg/jsSteg.h @@ -0,0 +1,65 @@ +#ifndef _JSSTEG_H +#define _JSSTEG_H + + +#include <stdio.h> +#include <string.h> +#include <ctype.h> +#include "util.h" +#include "connections.h" +#include "steg.h" +#include <event2/buffer.h> + + +/* error codes */ +#define INVALID_BUF_SIZE -1 +#define INVALID_DATA_CHAR -2 + + +int encodeHTTPBody(char *data, char *jTemplate, char *jData,unsigned int dlen, + unsigned int jtlen, unsigned int jdlen, int mode); +int isxString(char *str); +int decodeHTTPBody (char *jData, char *dataBuf, unsigned int jdlen, + unsigned int dataBufSize, int *fin, int mode); + +int encode(char *data, char *jTemplate, char *jData, + unsigned int dlen, unsigned int jtlen, unsigned int jdlen ); + +int encode2(char *data, char *jTemplate, char *jData, + unsigned int dlen, unsigned int jtlen, + unsigned int jdlen, int *fin); + +int decode (char *jData, char *dataBuf, unsigned int jdlen, + unsigned int dlen, unsigned int dataBufSize ); + +int decode2 (char *jData, char *dataBuf, unsigned int jdlen, + unsigned int dataBufSize, int *fin ); + +void printerr(int errno); + +int testEncode(char *data, char *js, char *outBuf, unsigned int dlen, unsigned int jslen, + unsigned int outBufLen, int testNum); + +int testDecode(char *inBuf, char *outBuf, unsigned int inBufSize, unsigned int dlen, + unsigned int outBufSize, int testNum); + +int testEncode2(char *data, char *js, char *outBuf, + unsigned int dlen, unsigned int jslen, unsigned int outBufLen, + int mode, int testNum); + +int testDecode2(char *inBuf, char *outBuf, + unsigned int inBufSize, unsigned int outBufSize, + int mode, int testNum); + + +int +x_http2_server_JS_transmit (steg_t* s, struct evbuffer *source, conn_t *conn); + +int +x_http2_handle_client_JS_receive(steg_t *s, conn_t *conn, struct evbuffer *dest, struct evbuffer* source); + + + +#endif + + diff --git a/src/steg/payloads.c b/src/steg/payloads.c new file mode 100644 index 0000000..2782aa6 --- /dev/null +++ b/src/steg/payloads.c @@ -0,0 +1,1486 @@ +#include "util.h" +#include "payloads.h" +#include "swfSteg.h" + + +/* These variables below are write-once, hence they should be race-safe */ + +static int initTypePayload[MAX_CONTENT_TYPE]; +static int typePayloadCount[MAX_CONTENT_TYPE]; +static int typePayload[MAX_CONTENT_TYPE][MAX_PAYLOADS]; +static int typePayloadCap[MAX_CONTENT_TYPE][MAX_PAYLOADS]; + + +static unsigned int max_JS_capacity = 0; +static unsigned int max_PDF_capacity = 0; + + + +pentry_header payload_hdrs[MAX_PAYLOADS]; +char* payloads[MAX_PAYLOADS]; +int payload_count = 0; + + +unsigned int get_max_JS_capacity() { + return max_JS_capacity; +} + +unsigned int get_max_PDF_capacity() { + return max_PDF_capacity; +} + + + +/* + * fixContentLen corrects the Content-Length for an HTTP msg that + * has been ungzipped, and removes the "Content-Encoding: gzip" + * field from the header. + * + * The function returns -1 if no change to the HTTP msg has been made, + * when the msg wasn't gzipped or an error has been encountered + * If fixContentLen changes the msg header, it will put the new HTTP + * msg in buf and returns the length of the new msg + * + * Input: + * payload - pointer to the (input) HTTP msg + * payloadLen - length of the (input) HTTP msg + * + * Ouptut: + * buf - pointer to the buffer containing the new HTTP msg + * bufLen - length of buf + * + */ +int fixContentLen (char* payload, int payloadLen, char *buf, int bufLen) { + + int gzipFlag=0, clFlag=0, clZeroFlag=0; + char* ptr = payload; + char* clPtr = payload; + char* gzipPtr = payload; + char* end; + + + char *cp, *clEndPtr; + int hdrLen, bodyLen, r, len; + + + + + + // note that the ordering between the Content-Length and the Content-Encoding + // in an HTTP msg may be different for different msg + + // if payloadLen is larger than the size of our buffer, + // stop and return -1 + if (payloadLen > bufLen) { return -1; } + + while (1) { + end = strstr(ptr, "\r\n"); + if (end == NULL) { + // log_debug("invalid header %d %d %s \n", payloadLen, (int) (ptr - payload), payload); + return -1; + } + + if (!strncmp(ptr, "Content-Encoding: gzip\r\n", 24)) { + gzipFlag = 1; + gzipPtr = ptr; + } else if (!strncmp(ptr, "Content-Length: 0", 17)) { + clZeroFlag = 1; + } else if (!strncmp(ptr, "Content-Length:", 15)) { + clFlag = 1; + clPtr = ptr; + } + + if (!strncmp(end, "\r\n\r\n", 4)){ + break; + } + ptr = end+2; + } + + // stop if zero Content-Length or Content-Length not found + if (clZeroFlag || ! clFlag) return -1; + + // end now points to the end of the header, before "\r\n\r\n" + cp=buf; + bodyLen = (int)(payloadLen - (end+4-payload)); + + clEndPtr = strstr(clPtr, "\r\n"); + if (clEndPtr == NULL) { + log_debug("unable to find end of line for Content-Length"); + return -1; + } + if (gzipFlag && clFlag) { + if (gzipPtr < clPtr) { // Content-Encoding appears before Content-Length + + // copy the part of the header before Content-Encoding + len = (int)(gzipPtr-payload); + memcpy(cp, payload, len); + cp = cp+len; + + // copy the part of the header between Content-Encoding and Content-Length + // skip 24 char, the len of "Content-Encoding: gzip\r\n" + // *** this is temporary; we'll remove this after the obfsproxy can perform gzip + len = (int)(clPtr-(gzipPtr+24)); + memcpy(cp, gzipPtr+24, len); + cp = cp+len; + + // put the new Content-Length + memcpy(cp, "Content-Length: ", 16); + cp = cp+16; + r = sprintf(cp, "%d\r\n", bodyLen); + if (r < 0) { + log_debug("sprintf fails"); + return -1; + } + cp = cp+r; + + // copy the part of the header after Content-Length, if any + if (clEndPtr != end) { // there is header info after Content-Length + len = (int)(end-(clEndPtr+2)); + memcpy(cp, clEndPtr+2, len); + cp = cp+len; + memcpy(cp, "\r\n\r\n", 4); + cp = cp+4; + } else { // Content-Length is the last hdr field + memcpy(cp, "\r\n", 2); + cp = cp+2; + } + + hdrLen = cp-buf; + +/**** +log_debug("orig: hdrLen = %d, bodyLen = %d, payloadLen = %d", (int)(end+4-payload), bodyLen, payloadLen); +log_debug("new: hdrLen = %d, bodyLen = %d, payloadLen = %d", hdrLen, bodyLen, hdrLen+bodyLen); + ****/ + + // copy the HTTP body + memcpy(cp, end+4, bodyLen); + return (hdrLen+bodyLen); + + } else { // Content-Length before Content-Encoding + // copy the part of the header before Content-Length + len = (int)(clPtr-payload); + memcpy(cp, payload, len); + cp = cp+len; + + // put the new Content-Length + memcpy(cp, "Content-Length: ", 16); + cp = cp+16; + r = sprintf(cp, "%d\r\n", bodyLen); + if (r < 0) { + log_debug("sprintf fails"); + return -1; + } + cp = cp+r; + + // copy the part of the header between Content-Length and Content-Encoding + len = (int)(gzipPtr-(clEndPtr+2)); + memcpy(cp, clEndPtr+2, len); + cp = cp+len; + + // copy the part of the header after Content-Encoding + // skip 24 char, the len of "Content-Encoding: gzip\r\n" + // *** this is temporary; we'll remove this after the obfsproxy can perform gzip + if (end > (gzipPtr+24)) { // there is header info after Content-Encoding + len = (int)(end-(gzipPtr+24)); + memcpy(cp, gzipPtr+24, len); + cp = cp+len; + memcpy(cp, "\r\n\r\n", 4); + cp = cp+4; + } else { // Content-Encoding is the last field in the hdr + memcpy(cp, "\r\n", 2); + cp = cp+2; + } + hdrLen = cp-buf; + +/**** +log_debug("orig: hdrLen = %d, bodyLen = %d, payloadLen = %d", (int)(end+4-payload), bodyLen, payloadLen); +log_debug("new: hdrLen = %d, bodyLen = %d, payloadLen = %d", hdrLen, bodyLen, hdrLen+bodyLen); + ****/ + + // copy the HTTP body + memcpy(cp, end+4, bodyLen); + return (hdrLen+bodyLen); + } + } + return -1; +} + +void load_payloads(const char* fname) { + FILE* f; + char buf[HTTP_MSG_BUF_SIZE]; + char buf2[HTTP_MSG_BUF_SIZE]; + pentry_header pentry; + int pentryLen; + int r; + + if (payload_count != 0) + return; + + srand(time(NULL)); + f = fopen(fname, "r"); + if (f == NULL) { + fprintf(stderr, "Cannot open trace file %s. Exiting\n", fname); + exit(1); + } + + bzero(payload_hdrs, sizeof(payload_hdrs)); + + while (payload_count < MAX_PAYLOADS) { + + if (fread(&pentry, 1, sizeof(pentry_header), f) < sizeof(pentry_header)) { + break; + } + + pentryLen = ntohl(pentry.length); + if((unsigned int) pentryLen > sizeof(buf)) { +#ifdef DEBUG + // fprintf(stderr, "pentry too big %d %d\n", pentry.length, ntohl(pentry.length)); + fprintf(stderr, "pentry too big %d\n", pentryLen); +#endif + // skip to the next pentry + if (fseek(f, pentryLen, SEEK_CUR)) { + fprintf(stderr, "skipping to next pentry fails\n"); + } + continue; + // exit(0); + } + + pentry.length = pentryLen; + pentry.ptype = ntohs(pentry.ptype); + + if (fread(buf, 1, pentry.length, f) < (unsigned int) pentry.length) + break; + + // todo: + // fixed content length for gzip'd HTTP msg + // fixContentLen returns -1, if no change to the msg + // otherwise, it put the new HTTP msg (with hdr changed) in buf2 + // and returns the size of the new msg + + r = -1; + if (pentry.ptype == TYPE_HTTP_RESPONSE) { + r = fixContentLen (buf, pentry.length, buf2, HTTP_MSG_BUF_SIZE); + // log_debug("for payload_count %d, fixContentLen returns %d", payload_count, r); + } + // else { + // log_debug("for payload_count %d, pentry.ptype = %d", payload_count, pentry.ptype); + // } + + if (r < 0) { + payloads[payload_count] = malloc(pentry.length + 1); + memcpy(payloads[payload_count], buf, pentry.length); + } else { + pentry.length = r; + payloads[payload_count] = malloc(pentry.length + 1); + memcpy(payloads[payload_count], buf2, pentry.length); + } + payload_hdrs[payload_count] = pentry; + payloads[payload_count][pentry.length] = 0; + payload_count++; + } // while + +#ifdef DEBUG + printf("loading payload count = %d\n", payload_count); +#endif + + fclose(f); +} + + + + + +void gen_rfc_1123_date(char* buf, int buf_size) { + time_t t = time(NULL); + struct tm *my_tm = gmtime(&t); + strftime(buf, buf_size, "Date: %a, %d %b %Y %H:%M:%S GMT\r\n", my_tm); +} + + + +/* sample response header */ +/* + +HTTP/1.1 200 OK +Date: Fri, 21 Oct 2011 19:09:49 GMT +Server: Apache +Content-Length: 1441 +* Content-Encoding: gzip +Content-Type: text/html;charset=UTF-8 +* Keep-Alive: timeout=15, max=93 +Connection: close + +*/ + + +int gen_response_header(char* content_type, int gzip, int length, char* buf, int buflen) { + char* ptr; + + // conservative assumption here.... + if (buflen < 256) { + fprintf(stderr, "gen_response_header: buflen too small\n"); + return -1; + } + + sprintf(buf, "HTTP/1.1 200 OK\r\n"); + ptr = buf + strlen("HTTP/1.1 200 OK\r\n"); + gen_rfc_1123_date(ptr, buflen - (ptr - buf)); + ptr = ptr + strlen(ptr); + + if (gzip) + sprintf(ptr, "Server: Apache\r\nContent-Length: %d\r\nContent-Encoding: gzip\r\nContent-Type: %s\r\nConnection: close\r\n\r\n", length, content_type); + else + sprintf(ptr, "Server: Apache\r\nContent-Length: %d\r\nContent-Type: %s\r\nConnection: close\r\n\r\n", length, content_type); + + ptr += strlen(ptr); + + return ptr - buf; +} + + + + + + +int parse_client_headers(char* inbuf, char* outbuf, int len) { + // client-side + // remove Host: field + // remove referrer fields? + + char* ptr = inbuf; + int outlen = 0; + + while (1) { + // char* end = strstr(ptr, "\r\n", len - (ptr - inbuf)); + char* end = strstr(ptr, "\r\n"); + if (end == NULL) { + fprintf(stderr, "invalid client header %d %d %s \n PTR = %s\n", len, (int) (len - (ptr - inbuf)), inbuf, ptr); + // fprintf(stderr, "HERE %s\n", ptr); + break; + } + + if (!strncmp(ptr, "Host:", 5) || + !strncmp(ptr, "Referer:", 8) || + !strncmp(ptr, "Cookie:", 7)) { + goto next; + } + + memcpy(outbuf + outlen, ptr, end - ptr + 2); + outlen += end - ptr + 2; + + next: + if (!strncmp(end, "\r\n\r\n", 4)){ + break; + } + ptr = end+2; + } + + return outlen; + + // server-side + // fix date fields + // fix content-length + + + +} + + + + +/* first line is of the form.... + GET /XX/XXXX.swf[?YYYY] HTTP/1.1\r\n +*/ + + +int +find_uri_type(char* buf) { + + char* uri; + int uri_len; + char* ext; + + if (strncmp(buf, "GET", 3) != 0 && strncmp(buf, "POST", 4) != 0) + return -1; + + buf = strchr(buf, ' ') + 1; + uri_len = strchr(buf, ' ') - buf; + uri = malloc(uri_len + 1); + + strncpy(uri, buf, uri_len); + uri[uri_len] = 0; + + if (strchr(uri, '?')) + ext = strchr(uri, '?') - 4; + else + ext = uri + uri_len - 4; + + + if (!strncmp(ext, ".pdf", 4) || !strncmp(ext, ".PDF", 4)) + return HTTP_CONTENT_PDF; + + if (!strncmp(ext, ".swf", 4) || !strncmp(ext, ".SWF", 4)) + return HTTP_CONTENT_SWF; + + // if (!strncmp(ext, ".js", 3) || !strncmp(ext, ".JS", 3)) + return HTTP_CONTENT_JAVASCRIPT; + + if (!strncmp(ext-1, "html", 4) || !strncmp(ext, "htm", 3) || strchr(ext-1, '.') == NULL) + return HTTP_CONTENT_HTML; + + return -1; + +} + + + + + + + + + + + +unsigned int find_client_payload(char* buf, int len, int type) { + int r = rand() % payload_count; + int cnt = 0; + char* inbuf; + +#ifdef DEBUG + fprintf(stderr, "TRYING payload %d \n", r); +#endif + while (1) { + pentry_header* p = &payload_hdrs[r]; + if (p->ptype == type) { + inbuf = payloads[r]; + if (find_uri_type(inbuf) != HTTP_CONTENT_SWF && + find_uri_type(inbuf) != HTTP_CONTENT_JAVASCRIPT) + + goto next; + + if (p->length > len) { + fprintf(stderr, "BUFFER TOO SMALL... \n"); + goto next; + } + else + len = p->length; + break; + } + next: + r = (r+1) % payload_count; + + + // no matching payloads... + if (cnt++ == payload_count) { + fprintf(stderr, "NO MATCHING PAYLOADS... \n"); + return 0; + } + } + + inbuf[len] = 0; + + // clean up the buffer... + return parse_client_headers(inbuf, buf, len); + +} + + +/* + * skipJSPattern returns the number of characters to skip when + * the input pointer matches the start of a common JavaScript + * keyword + * + * todo: + * Use a more efficient algo (e.g., Aho-Corasick) in the next iteration + */ +int skipJSPattern (char *cp, int len) { + + // log_debug("Turning off skipJSPattern for debugging"); + return 0; + + if (len < 1) return 0; + + if (len > 8) { + // "function " and "function(" + if (cp[0] == 'f' && + !strncmp(cp+1, "un", 2) && + isxdigit(cp[3]) && + !strncmp(cp+4, "tion", 4) && + (cp[8] == ' ' || cp[8] == '(')) + return 9; + } + + if (len > 6) { + // "return " + if (cp[0] == 'r' && + isxdigit(cp[1]) && + !strncmp(cp+2, "turn ", 5)) + return 7; + // "switch " + if (cp[0] == 's' && + !strncmp(cp+1, "wit", 3) && + isxdigit(cp[4]) && + !strncmp(cp+5, "h ", 2)) + return 7; + } + + if (len > 5) { + // "while " and "while(" + if (cp[0] == 'w' && + !strncmp(cp+1, "hil", 3) && + isxdigit(cp[4]) && + (cp[5] == ' ' || cp[5] == '(')) + return 6; + } + + if (len > 4) { + // "else " and "else{" + if (cp[0] == 'e' && + !strncmp(cp, "ls", 2) && + isxdigit(cp[3]) && + (cp[4] == ' ' || cp[4] == '{')) + return 5; + } + + if (len > 3) { + // "var " + if (cp[0] == 'v' && + isxdigit(cp[1]) && + cp[2] == 'r' && + cp[3] == ' ') + return 4; + } + + return 0; +} + + +int isalnum_ (char c) { + if (isalnum(c) || c == '_') return 1; + else return 0; +} + +int offset2Alnum_ (char *p, int range) { + char *cp = p; + + while ((cp < (p+range)) && !isalnum_(*cp)) { + cp++; + } + + if (cp < (p+range)) { + return (cp-p); + } else { + return -1; + } +} + +/* + * offset2Hex returns the offset to the next usable hex char. + * usable here refer to char that our steg module can use to encode + * data. in particular, words that correspond to common JavaScript keywords + * are not used for data encoding (see skipJSPattern). Also, because + * JS var name must start with an underscore or a letter (but not a digit) + * we don't use the first char of a word for encoding data + * + * e.g., the JS statement "var a;" won't be used for encoding data + * because "var" is a common JS keyword and "a" is the first char of a word + * + * Input: + * p - ptr to the starting pos + * range - max number of char to look + * isLastCharHex - is the char pointed to by (p-1) a hex char + * + * Output: + * offset2Hex returns the offset to the next usable hex char + * between p and (p+range), if it exists; + * otherwise, it returns -1 + * + */ +int offset2Hex (char *p, int range, int isLastCharHex) { + char *cp = p; + int i,j; + int isFirstWordChar = 1; + + if (range < 1) return -1; + + // case 1: last char is hexadecimal + if (isLastCharHex) { + if (isxdigit(*cp)) return 0; // base case + else { + cp++; + i = offset2Alnum_(cp, p+range-cp); + while (i == 0) { + if (isxdigit(*cp)) return (cp-p); + cp++; + i = offset2Alnum_(cp, p+range-cp); + } + if (i == -1) return -1; + // if (i > 0), fallthru and handle case 2 + } + } + + // case 2: + // find the next word that starts with alnum or underscore, + // which could be a variable, keyword, or literal inside a string + + i = offset2Alnum_(cp, p+range-cp); + if (i == -1) return -1; + // isFirstWordChar = 1; + + while (cp < (p+range) && i != -1) { + + if (i == 0) { + j = skipJSPattern(cp, p+range-cp); + if (j > 0) { + // cp points a word that should not be used (e.g., JS keyword) + cp = cp+j; isFirstWordChar = 1; + } else { // j == 0 + if (isFirstWordChar) { + cp++; isFirstWordChar = 0; // skip the 1st char of a word + } + else { // we are in the middle of a word; no need to invoke skipJSPattern + if (isxdigit(*cp)) return (cp-p); + else cp++; + } + } + } else { // i > 0 + cp += i; isFirstWordChar = 1; + } + i = offset2Alnum_(cp, p+range-cp); + + } // while + + // cannot find next usable hex char + return -1; + +} + +/* + * capacityJS returns the number of usable hex char + * in the input HTTP message for data encoding + * + * Input: + * buf - pointer to HTTP message + * len - sizeof buf + * mode - + * CONTENT_JAVASCRIPT (case 1); HTTP message body is a JavaScript + * (e.g., Content-Type: {text/javascript, application/x-javascript, + * application/javascript}) + * CONTENT_HTML_JAVASCRIPT (case 2) + * Content-Type: text/html and HTTP message body contains + * <script type="text/javascript"> ... </script> + * + * Output: + * capacityJS returns the number of usable hex char that can be embedded + * in the HTTP message + * + * Note: + * This is a prototype for the simple version (all hex char are usable) + * will refine this to skip JavaScript keywords in the next iteration + * + */ + + + +unsigned int capacityJS (char* buf, int len, int mode) { + char *hEnd, *bp, *jsStart, *jsEnd; + int cnt=0; + + // jump to the beginning of the body of the HTTP message + hEnd = strstr(buf, "\r\n\r\n"); + if (hEnd == NULL) { + // cannot find the separator between HTTP header and HTTP body + return 0; + } + bp = hEnd + 4; + + if (mode == CONTENT_JAVASCRIPT) { + while (bp < (buf+len)) { + if (isxdigit(*bp)) { + cnt++; +#ifdef DEBUG + printf("%c", *bp); +#endif + } + bp++; + } +#ifdef DEBUG + printf("\n"); +#endif + return cnt; + } else if (mode == CONTENT_HTML_JAVASCRIPT) { + while (bp < (buf+len)) { + jsStart = strstr(bp, "<script type=\"text/javascript\">"); + if (jsStart == NULL) break; + bp = jsStart+31; + jsEnd = strstr(bp, "</script>"); + if (jsEnd == NULL) break; + // count the number of usable hex char between jsStart+31 and jsEnd + while (bp < jsEnd) { + if (isxdigit(*bp)) { + cnt++; +#ifdef DEBUG + printf("%c", *bp); +#endif + } + bp++; + } + bp += 9; + } +#ifdef DEBUG + printf("\n"); +#endif + return cnt; + } else { + fprintf(stderr, "Unknown mode (%d) for capacityJS() ... \n", mode); + return 0; + } + +} + +/* + * capacityJS3 is the next iteration for capacityJS + */ +unsigned int capacityJS3 (char* buf, int len, int mode) { + char *hEnd, *bp, *jsStart, *jsEnd; + int cnt=0; + int j; + + // jump to the beginning of the body of the HTTP message + hEnd = strstr(buf, "\r\n\r\n"); + if (hEnd == NULL) { + // cannot find the separator between HTTP header and HTTP body + return 0; + } + bp = hEnd + 4; + + + if (mode == CONTENT_JAVASCRIPT) { + j = offset2Hex(bp, (buf+len)-bp, 0); + while (j != -1) { + cnt++; + if (j == 0) { + bp = bp+1; + } else { + bp = bp+j+1; + } +// #ifdef DEBUG +// printf("got |%c|\n", *(bp-1)); +// #endif + j = offset2Hex(bp, (buf+len)-bp, 1); + } // while + return cnt; + } else if (mode == CONTENT_HTML_JAVASCRIPT) { + while (bp < (buf+len)) { + jsStart = strstr(bp, "<script type=\"text/javascript\">"); + if (jsStart == NULL) break; + bp = jsStart+31; + jsEnd = strstr(bp, "</script>"); + if (jsEnd == NULL) break; + // count the number of usable hex char between jsStart+31 and jsEnd + + j = offset2Hex(bp, jsEnd-bp, 0); + while (j != -1) { + cnt++; + if (j == 0) { + bp = bp+1; + } else { + bp = bp+j+1; + } +#ifdef DEBUG +printf("got |%c|\n", *(bp-1)); +#endif + j = offset2Hex(bp, jsEnd-bp, 1); + } // while (j != -1) + + bp += 9; + } // while (bp < (buf+len)) + return cnt; + } else { + fprintf(stderr, "Unknown mode (%d) for capacityJS() ... \n", mode); + return 0; + } +} + + +/* + * strInBinary looks for char array pattern of length patternLen in a char array + * blob of length blobLen + * + * return a pointer for the first occurrence of pattern in blob, if found + * otherwise, return NULL + * + */ +char * +strInBinary (const char *pattern, unsigned int patternLen, + const char *blob, unsigned int blobLen) { + int found = 0; + char *cp = (char *)blob; + + while (1) { + if (blob+blobLen-cp < patternLen) break; + if (*cp == pattern[0]) { + if (memcmp(cp, pattern, patternLen) == 0) { + found = 1; + break; + } + } + cp++; + } + if (found) return cp; + else return NULL; +} + + +/* + * has_eligible_HTTP_content() identifies if the input HTTP message + * contains a specified type of content, used by a steg module to + * select candidate HTTP message as cover traffic + */ + +// for JavaScript, there are two cases: +// 1) If Content-Type: has one of the following values +// text/javascript +// application/x-javascript +// application/javascript +// 2) Content-Type: text/html and +// HTTP body contains <script type="text/javascript"> ... </script> +// #define CONTENT_JAVASCRIPT 1 (for case 1) +// #define CONTENT_HTML_JAVASCRIPT 2 (for case 2) +// +// for pdf, we look for the msgs whose Content-Type: has one of the +// following values +// 1) application/pdf +// 2) application/x-pdf +// + +int has_eligible_HTTP_content (char* buf, int len, int type) { + char* ptr = buf; + char* matchptr; + int tjFlag=0, thFlag=0, ceFlag=0, teFlag=0, http304Flag=0, clZeroFlag=0, pdfFlag=0, swfFlag=0; + char* end, *cp; + +#ifdef DEBUG + fprintf(stderr, "TESTING availabilty of js in payload ... \n"); +#endif + + if (type != HTTP_CONTENT_JAVASCRIPT && + type != HTTP_CONTENT_PDF && type != HTTP_CONTENT_SWF) + return 0; + + + // assumption: buf is null-terminated + if (!strstr(buf, "\r\n\r\n")) + return 0; + + + while (1) { + end = strstr(ptr, "\r\n"); + if (end == NULL) { + break; + } + + if (!strncmp(ptr, "Content-Type:", 13)) { + + if (!strncmp(ptr+14, "text/javascript", 15) || + !strncmp(ptr+14, "application/javascript", 22) || + !strncmp(ptr+14, "application/x-javascript", 24)) { + tjFlag = 1; + } + if (!strncmp(ptr+14, "text/html", 9)) { + thFlag = 1; + } + if (!strncmp(ptr+14, "application/pdf", 15) || + !strncmp(ptr+14, "application/x-pdf", 17)) { + pdfFlag = 1; + } + if (!strncmp(ptr+14, "application/x-shockwave-flash", strlen("application/x-shockwave-flash"))) { + swfFlag = 1; + } + + } else if (!strncmp(ptr, "Content-Encoding:", 17)) { + ceFlag = 1; + } else if (!strncmp(ptr, "Transfer-Encoding:", 18)) { + teFlag = 1; + } else if (!strncmp(ptr, "HTTP/1.1 304 ", 13)) { + http304Flag = 1; + } else if (!strncmp(ptr, "Content-Length: 0", 17)) { + clZeroFlag = 1; + } + + if (!strncmp(end, "\r\n\r\n", 4)){ + break; + } + ptr = end+2; + } + +#ifdef DEBUG + printf("tjFlag=%d; thFlag=%d; ceFlag=%d; teFlag=%d; http304Flag=%d; clZeroFlag=%d\n", + tjFlag, thFlag, ceFlag, teFlag, http304Flag, clZeroFlag); +#endif + + if (type == HTTP_CONTENT_JAVASCRIPT) { + // empty body if it's HTTP not modified (304) or zero Content-Length + if (http304Flag || clZeroFlag) return 0; + + // for now, we're not dealing with Transfer-Encoding (e.g., chunked) + // if (teFlag) return 0; + // if (ceFlag) return 0; // for now, we remove "Content-Encoding: gzip" in fixContentLen + if (teFlag || ceFlag) return 0; +/***** + if (teFlag) return 0; + *****/ + + if (tjFlag && ceFlag && end != NULL) { + log_debug("(JS) gzip flag detected with hdr len %d", (int)(end-buf+4)); + } else if (thFlag && ceFlag && end != NULL) { + log_debug("(HTML) gzip flag detected with hdr len %d", (int)(end-buf+4)); + } + + // case 1 + if (tjFlag) return 1; + + // case 2: check if HTTP body contains <script type="text/javascript"> + if (thFlag) { + matchptr = strstr(ptr, "<script type=\"text/javascript\">"); + if (matchptr != NULL) { + return 2; + } + } + } + + if (type == HTTP_CONTENT_PDF && pdfFlag) { + // reject msg with empty body: HTTP not modified (304) or zero Content-Length + if (http304Flag || clZeroFlag) return 0; + + // for now, we're not dealing with Transfer-Encoding (e.g., chunked) + // or Content-Encoding (e.g., gzip) + // if (teFlag) return 0; + // if (ceFlag) return 0; + if (teFlag || ceFlag) return 0; + + // check if HTTP body contains "endstream"; + // strlen("endstream") == 9 + + cp = strInBinary("endstream", 9, ptr, buf+len-ptr); + if (cp != NULL) { + // log_debug("Matched endstream!"); + return 1; + } + } + + if (type == HTTP_CONTENT_SWF && swfFlag == 1 && + ((len + buf - end) > SWF_SAVE_FOOTER_LEN + SWF_SAVE_HEADER_LEN + 8)) + return 1; + + return 0; +} + + + +unsigned int capacityPDF (char* buf, int len) { + char *hEnd, *bp, *streamStart, *streamEnd; + int cnt=0; + int size; + + // jump to the beginning of the body of the HTTP message + hEnd = strstr(buf, "\r\n\r\n"); + if (hEnd == NULL) { + // cannot find the separator between HTTP header and HTTP body + return 0; + } + bp = hEnd + 4; + + while (bp < (buf+len)) { + streamStart = strInBinary("stream", 6, bp, (buf+len)-bp); + // streamStart = strstr(bp, "stream"); + if (streamStart == NULL) break; + bp = streamStart+6; + streamEnd = strInBinary("endstream", 9, bp, (buf+len)-bp); + // streamEnd = strstr(bp, "endstream"); + if (streamEnd == NULL) break; + // count the number of char between streamStart+6 and streamEnd + size = streamEnd - (streamStart+6) - 2; // 2 for \r\n before streamEnd + if (size > 0) { + cnt = cnt + size; + log_debug("capacity of pdf increase by %d", size); + } + bp += 9; + } + return cnt; +} + + + + + + + + + +/* + * init_payload_pool initializes the arrays pertaining to + * message payloads for the specified content type + * + * Specifically, it populates the following arrays + * static int initTypePayload[MAX_CONTENT_TYPE]; + * static int typePayloadCount[MAX_CONTENT_TYPE]; + * static int typePayload[MAX_CONTENT_TYPE][MAX_PAYLOADS]; + * static int typePayloadCap[MAX_CONTENT_TYPE][MAX_PAYLOADS]; + * + * Input: + * len - max length of payload + * type - ptype field value in pentry_header + * contentType - (e.g, HTTP_CONTENT_JAVASCRIPT for JavaScript content) + */ + + + + +int init_JS_payload_pool(int len, int type, int minCapacity) { + + // stat for usable payload + int minPayloadSize = 0, maxPayloadSize = 0; + int sumPayloadSize = 0; + int minPayloadCap = 0, maxPayloadCap = 0; + int sumPayloadCap = 0; + + unsigned int contentType = HTTP_CONTENT_JAVASCRIPT; + + int cnt = 0; + int r; + pentry_header* p; + char* msgbuf; + int cap; + int mode; + + + + if (payload_count == 0) { + log_debug("payload_count == 0; forgot to run load_payloads()?\n"); + return 0; + } + + if (initTypePayload[contentType] != 0) return 1; // init is done already + + + for (r = 0; r < payload_count; r++) { + p = &payload_hdrs[r]; + if (p->ptype != type || p->length > len) { + continue; + } + + msgbuf = payloads[r]; + + mode = has_eligible_HTTP_content(msgbuf, p->length, HTTP_CONTENT_JAVASCRIPT); + if (mode > 0) { + + cap = capacityJS3(msgbuf, p->length, mode); + if (cap < JS_DELIMITER_SIZE) + continue; + + cap = (cap - JS_DELIMITER_SIZE)/2; + + if (cap > minCapacity) { + typePayloadCap[contentType][cnt] = cap; // (cap-JS_DELIMITER_SIZE)/2; + // because we use 2 hex char to encode every data byte, the available + // capacity for encoding data is divided by 2 + typePayload[contentType][cnt] = r; + cnt++; + + // update stat + if (cnt == 1) { + minPayloadSize = p->length; maxPayloadSize = p->length; + minPayloadCap = cap; maxPayloadCap = cap; + } + else { + if (minPayloadSize > p->length) minPayloadSize = p->length; + if (maxPayloadSize < p->length) maxPayloadSize = p->length; + if (minPayloadCap > cap) minPayloadCap = cap; + if (maxPayloadCap < cap) { + maxPayloadCap = cap; + } + + } + sumPayloadSize += p->length; sumPayloadCap += cap; + } + } + } + + + max_JS_capacity = maxPayloadCap; + + + initTypePayload[contentType] = 1; + typePayloadCount[contentType] = cnt; + log_debug("init_payload_pool: typePayloadCount for contentType %d = %d", + contentType, typePayloadCount[contentType]); + log_debug("minPayloadSize = %d", minPayloadSize); + log_debug("maxPayloadSize = %d", maxPayloadSize); + log_debug("avgPayloadSize = %f", (float)sumPayloadSize/(float)cnt); + log_debug("minPayloadCap = %d", minPayloadCap); + log_debug("maxPayloadCap = %d", maxPayloadCap); + log_debug("avgPayloadCap = %f", (float)sumPayloadCap/(float)cnt); + return 1; +} + + + + + + + + + +int init_PDF_payload_pool(int len, int type, int minCapacity) { + + // stat for usable payload + int minPayloadSize = 0, maxPayloadSize = 0; + int sumPayloadSize = 0; + int minPayloadCap = 0, maxPayloadCap = 0; + int sumPayloadCap = 0; + + int cnt = 0; + int r; + pentry_header* p; + char* msgbuf; + int cap; + int mode; + unsigned int contentType = HTTP_CONTENT_PDF; + + + if (payload_count == 0) { + fprintf(stderr, "payload_count == 0; forgot to run load_payloads()?\n"); + return 0; + } + + if (initTypePayload[contentType] != 0) return 1; // init is done already + + + for (r = 0; r < payload_count; r++) { + p = &payload_hdrs[r]; + if (p->ptype != type || p->length > len) { + continue; + } + + msgbuf = payloads[r]; + + mode = has_eligible_HTTP_content(msgbuf, p->length, HTTP_CONTENT_PDF); + if (mode > 0) { + // use capacityPDF() to find out the amount of data that we + // can encode in the pdf doc + // cap = minCapacity+1; + cap = capacityPDF(msgbuf, p->length); + log_debug("got pdf (index %d) with capacity %d", r, cap); + if (cap > minCapacity) { + log_debug("pdf (index %d) greater than mincapacity %d", cnt, minCapacity); + typePayloadCap[contentType][cnt] = (cap-PDF_DELIMITER_SIZE)/2; + typePayload[contentType][cnt] = r; + cnt++; + + // update stat + if (cnt == 1) { + minPayloadSize = p->length; maxPayloadSize = p->length; + minPayloadCap = cap; maxPayloadCap = cap; + } + else { + if (minPayloadSize > p->length) minPayloadSize = p->length; + if (maxPayloadSize < p->length) maxPayloadSize = p->length; + if (minPayloadCap > cap) minPayloadCap = cap; + if (maxPayloadCap < cap) maxPayloadCap = cap; + } + sumPayloadSize += p->length; sumPayloadCap += cap; + } + } + } + + max_PDF_capacity = maxPayloadCap; + initTypePayload[contentType] = 1; + typePayloadCount[contentType] = cnt; + log_debug("init_payload_pool: typePayloadCount for contentType %d = %d", + contentType, typePayloadCount[contentType]); + log_debug("minPayloadSize = %d", minPayloadSize); + log_debug("maxPayloadSize = %d", maxPayloadSize); + log_debug("avgPayloadSize = %f", (float)sumPayloadSize/(float)cnt); + log_debug("minPayloadCap = %d", minPayloadCap); + log_debug("maxPayloadCap = %d", maxPayloadCap); + log_debug("avgPayloadCap = %f", (float)sumPayloadCap/(float)cnt); + return 1; +} + + + + + +int init_SWF_payload_pool(int len, int type, int minCapacity) { + + // stat for usable payload + int minPayloadSize = 0, maxPayloadSize = 0; + int sumPayloadSize = 0; + + int cnt = 0; + int r; + pentry_header* p; + char* msgbuf; + int mode; + unsigned int contentType = HTTP_CONTENT_SWF; + + + if (payload_count == 0) { + fprintf(stderr, "payload_count == 0; forgot to run load_payloads()?\n"); + return 0; + } + + if (initTypePayload[contentType] != 0) return 1; // init is done already + + + for (r = 0; r < payload_count; r++) { + p = &payload_hdrs[r]; + if (p->ptype != type || p->length > len) { + continue; + } + + msgbuf = payloads[r]; + // found a payload corr to the specified contentType + + mode = has_eligible_HTTP_content(msgbuf, p->length, HTTP_CONTENT_SWF); + if (mode > 0) { + typePayload[contentType][cnt] = r; + cnt++; + // update stat + if (cnt == 1) { + minPayloadSize = p->length; + maxPayloadSize = p->length; + } + else { + if (minPayloadSize > p->length) + minPayloadSize = p->length; + if (maxPayloadSize < p->length) + maxPayloadSize = p->length; + } + sumPayloadSize += p->length; + } + } + + initTypePayload[contentType] = 1; + typePayloadCount[contentType] = cnt; + log_debug("init_payload_pool: typePayloadCount for contentType %d = %d", + contentType, typePayloadCount[contentType]); + log_debug("minPayloadSize = %d", minPayloadSize); + log_debug("maxPayloadSize = %d", maxPayloadSize); + log_debug("avgPayloadSize = %f", (float)sumPayloadSize/(float)cnt); + return 1; +} + + + + + + + + + +int get_next_payload (int contentType, char** buf, int* size, int* cap) { + int r; + + log_debug("get_next_payload: contentType = %d, initTypePayload = %d, typePayloadCount = %d", + contentType, initTypePayload[contentType], typePayloadCount[contentType]); + + + if (contentType <= 0 || + contentType >= MAX_CONTENT_TYPE || + initTypePayload[contentType] == 0 || + typePayloadCount[contentType] == 0) + return 0; + + r = rand() % typePayloadCount[contentType]; +// int r = 1; +// log_debug("SERVER: *** always choose the same payload ***"); + + log_debug("SERVER: picked payload with index %d", r); + *buf = payloads[typePayload[contentType][r]]; + *size = payload_hdrs[typePayload[contentType][r]].length; + *cap = typePayloadCap[contentType][r]; + return 1; +} + + + + + + + + +int get_payload (int contentType, int cap, char** buf, int* size) { + int r; + unsigned int i = 0; + unsigned int cnt = 0; + + log_debug("get_payload: contentType = %d, initTypePayload = %d, typePayloadCount = %d", + contentType, initTypePayload[contentType], typePayloadCount[contentType]); + + + if (contentType <= 0 || + contentType >= MAX_CONTENT_TYPE || + initTypePayload[contentType] == 0 || + typePayloadCount[contentType] == 0) + return 0; + + + cnt = typePayloadCount[contentType]; + r = rand() % cnt; + + for (i=0; i < cnt; i++) { + + if (typePayloadCap[contentType][(r+i) % cnt] <= cap) + continue; + + *buf = payloads[typePayload[contentType][(r+i)%cnt]]; + *size = payload_hdrs[typePayload[contentType][(r+i)%cnt]].length; + return 1; + } + + + + return 0; + +} + + + + +int +find_content_length (char *hdr, int hlen) { + char *clStart; + char* clEnd; + char *clValStart; + int valLen; + int contentLen; + char buf[10]; + + clStart = strstr(hdr, "Content-Length: "); + if (clStart == NULL) { + log_debug("Unable to find Content-Length in the header"); + return -1; + } + + clEnd = strstr((char *)clStart, "\r\n"); + if (clEnd == NULL) { + log_debug("Unable to find end of line for Content-Length"); + return -1; + } + + // clValStart = clStart+strlen("Content-Length: "); + clValStart = clStart+16; + + valLen = clEnd-clValStart; + if (valLen > 9) return -1; + memcpy(buf, clValStart, valLen); + buf[valLen] = 0; + contentLen = atoi(buf); + return contentLen; +} + + + + + + +/* + +void testOffset2Alnum_skipJSPattern () { + char s1[] = "for (i=0; i<10; i++) { print i; }"; + + char s2[] = "***abcde*****"; + int d, i; + + printf("s1 = %s\n", s1); + printf("s2 = %s\n", s2); + + + d = offset2Alnum_(s1, strlen(s1)); + printf ("offset2Alnum_ for s1 = %d\n", d); + d = offset2Alnum_(s2, strlen(s2)); + printf ("offset2Alnum_ for s2 = %d\n", d); + + i = skipJSPattern (s1, strlen(s1)); + printf ("skipJSPattern for s1 = %d\n", i); + i = skipJSPattern (s2, strlen(s2)); + printf ("skipJSPattern for s2 = %d\n", i); +} + + + + +void testOffset2Hex () { + int d; + char s3[] = "for (bc=0; bc<10; bc++) { ad=2*bc+ad; }"; + printf("len(s3)=%d; s3 = |%s|\n", (int)strlen(s3), s3); + + d = offset2Alnum_(s3, strlen(s3)); + printf ("offset2Alnum_ for s3 = %d\n", d); + d = offset2Hex(s3, strlen(s3), 0); + printf ("offset2Hex for s3 = %d\n", d); +} + + +void testCapacityJS () { + int d; + char s4[] = "\r\n\r\n abc = abc + 1;"; + char s6[] = "\r\n\r\n <script type=\"text/javascript\">abc = abc + 1;</script>"; + + printf("\nTest for CONTENT_JAVASCRIPT:\n"); + printf("len(s4)=%d; s4 = |%s|\n", (int)strlen(s4), s4); + + d = offset2Alnum_(s4, strlen(s4)); + printf ("offset2Alnum_ for s4 = %d\n", d); + d = offset2Hex(s4, strlen(s4), 0); + printf ("offset2Hex for s4 = %d\n", d); + + printf("capacityJS (JS) returns %d\n", capacityJS(s4, strlen(s4), CONTENT_JAVASCRIPT)); + printf("capacityJS3 (JS) returns %d\n", capacityJS3(s4, strlen(s4), CONTENT_JAVASCRIPT)); + + printf("\nTest for CONTENT_HTML_JAVASCRIPT:\n"); + printf("len(s6)=%d; s6 = |%s|\n", (int)strlen(s6), s6); + + d = offset2Alnum_(s6, strlen(s6)); + printf ("offset2Alnum_ for s6 = %d\n", d); + d = offset2Hex(s6, strlen(s6), 0); + printf ("offset2Hex for s6 = %d\n", d); + + printf("capacityJS (HTML) returns %d\n", capacityJS(s6, strlen(s6), CONTENT_HTML_JAVASCRIPT)); + printf("capacityJS3 (HTML) returns %d\n", capacityJS3(s6, strlen(s6), CONTENT_HTML_JAVASCRIPT)); +} +*/ + + +/***** +int main() { + char buf[HTTP_MSG_BUF_SIZE]; + bzero(buf, sizeof(buf)); + // test for TYPE_HTTP_REQUEST + // load_payloads("../../traces/client.out"); + // int len = find_client_payload(buf, 10000, TYPE_HTTP_REQUEST); + // printf("%s\n", buf); + + // test for TYPE_HTTP_RESPONSE + // load_payloads("../../traces/server-cnn-nogzip.out"); + // load_payloads("../../traces/server-portals.out"); // ptype==1? + + // testOffset2Alnum_skipJSPattern(); + // testOffset2Hex(); + // testCapacityJS(); + + load_payloads("../../traces/server.out"); + // int r; + // r = find_server_payload(&buf, sizeof(buf), TYPE_HTTP_RESPONSE, HTTP_CONTENT_JAVASCRIPT); + // if (r > 0) { + // printf("Available payload capablity %d\n", r); + // } + // return r; + + return 0; +} + *****/ + diff --git a/src/steg/payloads.h b/src/steg/payloads.h new file mode 100644 index 0000000..b3fcc9d --- /dev/null +++ b/src/steg/payloads.h @@ -0,0 +1,159 @@ +#ifndef _PAYLOADS_H +#define _PAYLOADS_H + +#include <stdio.h> +#include <stdlib.h> +#include <time.h> +#include <string.h> +#include <sys/types.h> +#include <arpa/inet.h> +#include <ctype.h> + + +/* three files: + server_data, client data, protocol data +*/ + + + +#define RECV_GOOD 0 +#define RECV_INCOMPLETE 0 +#define RECV_BAD -1 + + + +#define CONN_DATA_REQUEST 1 /* payload packet sent by client */ +#define CONN_DATA_REPLY 2 /* payload packet sent by server */ + +#define NO_NEXT_STATE -1 + +#define MAX_PAYLOADS 10000 +// #define HTTP_MSG_BUF_SIZE 100000 + +// jsSteg-specific defines +#define JS_DELIMITER '?' +// a JavaScript delimiter is used to signal the end of encoding +// to facilitate the decoding process +#define JS_DELIMITER_REPLACEMENT '.' +// JS_DELIMITER that exists in the JavaScript before the end of +// data encoding will be replaced by JS_DELIMITER_REPLACEMENT +#define JS_DELIMITER_SIZE 1 + +#define JS_MIN_AVAIL_SIZE 2050 +// JS_MIN_AVAIL_SIZE should reflect the min number of data bytes +// a JavaScript may encapsulate +// Using hex-based encoding, it takes 2 hex char in JS +// to encode 1 data byte. Thus the size of data that can be encoded +// is about half this value + + +#define PDF_DELIMITER_SIZE 2 +#define PDF_MIN_AVAIL_SIZE 10240 +// PDF_MIN_AVAIL_SIZE should reflect the min number of data bytes +// a pdf doc can encode + +// specifying the type of contents as an input argument +// for has_eligible_HTTP_content() +#define HTTP_CONTENT_JAVASCRIPT 1 +#define HTTP_CONTENT_PDF 2 +#define HTTP_CONTENT_SWF 3 +#define HTTP_CONTENT_ENCRYPTEDZIP 4 +#define HTTP_CONTENT_HTML 5 + +// used by the JavaScript steg module to distinguish two cases in which +// JS may appear in the HTTP msg +// 1) CONTENT-TYPE in HTTP header specifies that the HTTP body is a JS +// 2) CONTENT-TYPE corresponds to HTML, and the HTTP body contains JS +// denoted by script type for JS +#define CONTENT_JAVASCRIPT 1 +#define CONTENT_HTML_JAVASCRIPT 2 + + +// payloads for specific content type +// +// MAX_CONTENT_TYPE specifies the maximum number of supported content types +// (e.g. HTTP_CONTENT_JAVASCRIPT is a content type) +// +// initTypePayload[x] specifies whether the arrays typePayloadCount and +// typePayloads for content type x +// +// typePayloadCount[x] specifies the number of available payloads for +// content type x +// +// typePayload[x][] contains references to the corresponding entries in +// payload_hdrs[] and payloads[] +// +// typePayloadCap[x][] specifies the capacity for typePayload[x][] + +#define MAX_CONTENT_TYPE 11 + + + +typedef int SID; +typedef short PacketType; +typedef short StateFlag; + +#define TYPE_SERVICE_DATA 0x1 +#define TYPE_HTTP_REQUEST 0x2 +#define TYPE_HTTP_RESPONSE 0x4 +#define BEGIN_STATE_FLG 0x1 +#define END_STATE_FLG 0x2 + + +/* struct for reading in the payload_gen dump file */ +typedef struct { + PacketType ptype; + int length; + ushort port; /* network format */ +}pentry_header; + + + + +typedef struct service_state { + SID id; + PacketType data_type; + SID next_state; + // double* probabilities; + StateFlag flg; + int dir; +}state; + + +#define HTTP_MSG_BUF_SIZE 100000 +void load_payloads(const char* fname); +unsigned int find_client_payload(char* buf, int len, int type); +unsigned int find_server_payload(char** buf, int len, int type, int contentType); + +int init_JS_payload_pool(int len, int type, int minCapacity); +int init_SWF_payload_pool(int len, int type, int minCapacity); +int init_PDF_payload_pool(int len, int type,int minCapacity); + + +int get_next_payload (int contentType, char** buf, int* size, int* cap); + +int get_payload (int contentType, int cap, char** buf, int* size); + +int has_eligible_HTTP_content (char* buf, int len, int type); +int fixContentLen (char* payload, int payloadLen, char *buf, int bufLen); +void gen_rfc_1123_date(char* buf, int buf_size); +int parse_client_headers(char* inbuf, char* outbuf, int len); +int skipJSPattern (char *cp, int len); +int isalnum_ (char c); +int offset2Alnum_ (char *p, int range); +int offset2Hex (char *p, int range, int isLastCharHex); +unsigned int capacityJS (char* buf, int len, int mode); +unsigned int capacityJS3 (char* buf, int len, int mode); +unsigned int get_max_JS_capacity(void); + +char * strInBinary (const char *pattern, unsigned int patternLen, const char *blob, unsigned int blobLen); + + +unsigned int capacityPDF (char* buf, int len); +unsigned int get_max_PDF_capacity(void); +int find_content_length (char *hdr, int hlen); +int find_uri_type(char* buf); + +int gen_response_header(char* content_type, int gzip, int length, char* buf, int buflen); + +#endif diff --git a/src/steg/pdfSteg.c b/src/steg/pdfSteg.c new file mode 100644 index 0000000..58147fa --- /dev/null +++ b/src/steg/pdfSteg.c @@ -0,0 +1,618 @@ +#include "payloads.h" +#include "pdfSteg.h" + +void buf_dump(unsigned char* buf, int len, FILE *out); + +#define STREAM_BEGIN ">>stream" +#define STREAM_BEGIN_SIZE 8 +#define STREAM_END "endstream" +#define STREAM_END_SIZE 9 + +#define DEBUG + + +/* + * pdfSteg: A PDF-based steganography module + * + */ + + +/* + * addDelimiter processes the input buffer (inbuf) of length inbuflen, + * copies it to output buffer (outbuf) of size outbufsize, + * and adds a two-char-long, end-of-data pattern at the end of outbuf + * based on delimiter1 and delimiter2. + * + * The end-of-data pattern consists of delimiter1 followed by a char + * that is not delimiter1. Thus, delimiter1 and delimiter2 must be + * different. + * + * If delimiter1 appears in the input buffer, addDelimiter puts two + * delimiter1 char in output buffer (to enable removeDelimiter to perform + * the back transformation) + * + * addDelimiter returns the length of the data written to outbuf, including + * the end-of-data pattern, if the transformation succeeds; + * otherwise, it returns -1 + * + */ +int +addDelimiter(char *inbuf, int inbuflen, char *outbuf, int outbuflen, + const char delimiter1, const char delimiter2) +{ + int cnt; + char *ibp, ic, rc; + + if (delimiter1 == delimiter2) return -1; + + cnt = 0; + ibp = inbuf; + while ((ibp-inbuf)<inbuflen && cnt<(outbuflen-2)) { + ic = *(ibp++); + if (ic != delimiter1) { + outbuf[cnt++] = ic; + } else { + outbuf[cnt++] = delimiter1; + outbuf[cnt++] = delimiter1; + } + } + + // error if outbuf is no large enough for storing the resulting data + if (cnt >= (outbuflen-2)) return -1; + + // put delimiter1 and a char that is not a delimiter1 + // as the end-of-data pattern at the end of outbuf + outbuf[cnt++] = delimiter1; + // try to get a random char (that is not delimiter1) + rc = (char) (rand() % 256); + if (rc != delimiter1) { + outbuf[cnt++] = rc; + } else { // unable to get a rand char != delimiter1, use delimiter2 + outbuf[cnt++] = delimiter2; + } + return cnt; +} + + +/* + * removeDelimiter performs the reverse transformation of addDelimiter. + * + * returns the length of data written to outbuf, if succeed; + * otherwise, it returns -1 + * endFlag indicates whether the end-of-encoding byte pattern (i.e., + * delimiter1 followed by non-delimiter1) is detected + */ +int +removeDelimiter(char *inbuf, int inbuflen, char *outbuf, int outbuflen, + const char delimiter1, int *endFlag, int *escape) +{ + int cnt; + char *ibp, ic1, ic2; + + cnt = 0; + *endFlag = 0; + ibp = inbuf; + + if (inbuflen <= 0) return -1; + + // special case: 2-char, end-of-data pattern could be in two buffers + // if *escape == true, we need to see if + // 1) (*ibp == delimiter1) -> put delimiter1 in outbuf + // 2) (*ibp != delimiter1) -> end-of-data detected + if (*escape) { + ic1 = *ibp; + if (ic1 == delimiter1) { + outbuf[cnt++] = ic1; ibp++; + } else { + *endFlag = 1; + return 0; + } + } + + *escape = 0; + while ((ibp-inbuf+1)<inbuflen && cnt<outbuflen) { + ic1 = *(ibp++); + if (ic1 != delimiter1) { + // *escape = 0; + outbuf[cnt++] = ic1; + } else { + // *escape = 1; + // lookahead 1 char + ic2 = *ibp; + // if the next char is delimiter1 + if (ic2 == delimiter1) { + outbuf[cnt++] = delimiter1; ibp++; + // *escape = 0; + } else { // end-of-data pattern detected + *endFlag = 1; + break; + } + } + } + + // if (*escape) { + // *escape = 0; + // return cnt; + // } + if (ibp-inbuf == inbuflen) return cnt; + + // handling the last char in inbuf, if needed + ic1 = *ibp; + if (ic1 != delimiter1) { + outbuf[cnt++] = ic1; + } else { + // look at the next stream obj to handle the special cases + *escape = 1; + } + + return cnt; +} + + + +/* + * pdfWrap embeds data of length dlen inside the stream objects of the PDF + * document (length plen) that appears in the body of a HTTP msg, and + * stores the result in the output buffer of size outsize + * + * pdfWrap returns the length of the pdf document with the data embedded + * inside, if succeed; otherwise, it returns -1 to indicate an error + * + */ +int +pdfWrap (char *data, unsigned int dlen, + char *pdfTemplate, unsigned int plen, + char *outbuf, unsigned int outbufsize) +{ + char data2[dlen*2+2]; + char *tp, *dp, *op, *streamStart, *streamEnd, *plimit; + int data2len, cnt, size, size2; + + // assumption: pdfWrap is length-preserving + if (outbufsize < plen) return -1; + + data2len = addDelimiter(data, dlen, data2, HTTP_MSG_BUF_SIZE, PDF_DELIMITER, PDF_DELIMITER2); + if (data2len < 1) return -1; + + + op = outbuf; // current pointer for output buffer + tp = pdfTemplate; // current pointer for http msg template + dp = data2; // current pointer for data2 + cnt = 0; // number of data char encoded + plimit = pdfTemplate+plen; + + while (tp < plimit) { + // find the next stream obj + streamStart = strInBinary(STREAM_BEGIN, STREAM_BEGIN_SIZE, tp, plimit-tp); + if (streamStart == NULL) { + log_warn("Cannot find stream in pdf"); + return -1; + } + + // copy everything between tp and "stream" (inclusive) to outbuf + size = streamStart - tp + STREAM_BEGIN_SIZE; + memcpy(op, tp, size); + op += size; + tp = streamStart + STREAM_BEGIN_SIZE; + + streamEnd = strInBinary(STREAM_END, STREAM_END_SIZE, tp, plimit-tp); + if (streamEnd == NULL) { + log_warn("Cannot find endstream in pdf"); + return -1; + } + + // count the number of usable char between tp and streamEnd + size = streamEnd-tp; + + // encoding data in the stream obj + if (size > 0) { + size2 = data2len - cnt; + if (size < size2) { + memcpy(op, dp, size); + op += size; tp += size; dp += size; + memcpy(op, tp, STREAM_END_SIZE); + op += STREAM_END_SIZE; tp += STREAM_END_SIZE; + cnt += size; + } else { // done encoding data + memcpy(op, dp, size2); + op += size2; tp += size2; dp += size2; + cnt += size2; + printf("Encoded %d char in pdf. Done encoding\n", size2); + break; + } + log_debug("Encoded %d char in pdf", size); + } else { // empty stream + memcpy(op, tp, STREAM_END_SIZE); + op += STREAM_END_SIZE; tp += STREAM_END_SIZE; + } + + if (cnt >= data2len) break; // this shouldn't happen ... + } + + // copy the rest of pdfTemplate to outbuf + size = plimit-tp; + log_debug("copying the rest of pdfTemplate to outbuf (size %d)", size); + memcpy(op, tp, size); + op += size; + return (op-outbuf); +} + + + + +/* + * pdfUnwrap is the inverse operation of pdfWrap + */ +int +pdfUnwrap (char *data, unsigned int dlen, + char *outbuf, unsigned int outbufsize) +{ + char *dp, *op, *streamStart, *streamEnd, *dlimit, *olimit; + int cnt, size, size2, endFlag; + int escape = 0; + + dp = data; // current pointer for data + op = outbuf; // current pointer for outbuf + cnt = 0; // number of char decoded + dlimit = data+dlen; + olimit = outbuf+outbufsize; + + while (dp < dlimit) { + // find the next stream obj + streamStart = strInBinary(STREAM_BEGIN, STREAM_BEGIN_SIZE, dp, dlimit-dp); + if (streamStart == NULL) { + log_warn("Cannot find stream in pdf"); + return -1; + } + + dp = streamStart + STREAM_BEGIN_SIZE; + streamEnd = strInBinary(STREAM_END, STREAM_END_SIZE, dp, dlimit-dp); + if (streamEnd == NULL) { + log_warn("Cannot find endstream in pdf"); + return -1; + } + + // count the number of usable char between tp and streamEnd + size = streamEnd-dp; + + if (size > 0) { + size2 = removeDelimiter(dp, size, op, olimit-op, PDF_DELIMITER, &endFlag, &escape); + if (size2 < 0) { + return -1; + } + cnt += size2; + if (endFlag) { // Done decoding + break; + } else { // Continue decoding + op += size2; + dp = streamEnd + STREAM_END_SIZE; + } + } else { // empty stream obj + dp = streamEnd + STREAM_END_SIZE; + } + } + + return cnt; +} + + + + + +int x_http2_server_PDF_transmit (steg_t* s, struct evbuffer *source, conn_t *conn) { + + struct evbuffer *dest = conn_get_outbound(conn); + size_t sbuflen = evbuffer_get_length(source); + unsigned int mpdf; + char *pdfTemplate = NULL, *hend; + int pdfTemplateSize = 0; + // char data1[HTTP_MSG_BUF_SIZE]; + char data1[(int) sbuflen]; + char outbuf[HTTP_MSG_BUF_SIZE]; + int cnt, hLen, outbuflen, i; + + struct evbuffer_iovec *iv; + int nv; + + // for debugging pdfWrap and pdfUnwrap + // char data2[(int) sbuflen]; + // int data2len; + + log_debug("Entering SERVER PDF transmit with sbuflen %d", (int)sbuflen); + + nv = evbuffer_peek(source, sbuflen, NULL, NULL, 0); + iv = xzalloc(sizeof(struct evbuffer_iovec) * nv); + + if (evbuffer_peek(source, sbuflen, NULL, iv, nv) != nv) { + free(iv); + return -1; + } + + cnt = 0; + for (i = 0; i < nv; i++) { + const unsigned char *p = iv[i].iov_base; + const unsigned char *limit = p + iv[i].iov_len; + while (p < limit && cnt < (int)sbuflen) { + data1[cnt++] = *p++; + } + } + + free(iv); + + log_debug("SERVER sbuflen = %d; cnt = %d", (int)sbuflen, cnt); + + mpdf = get_max_PDF_capacity(); + + if (mpdf <= 0) { + log_warn("SERVER ERROR: No pdfTemplate found\n"); + return -1; + } + + if (sbuflen > (size_t) mpdf) { + log_warn("SERVER ERROR: pdfTemplate cannot accommodate data %d %dn", + (int) sbuflen, (int) mpdf); + return -1; + } + + if (get_payload(HTTP_CONTENT_PDF, sbuflen, &pdfTemplate, &pdfTemplateSize) == 1) { + log_debug("SERVER found the next HTTP response template with size %d", pdfTemplateSize); + } else { + log_warn("SERVER couldn't find the next HTTP response template"); + return -1; + } + + hend = strstr(pdfTemplate, "\r\n\r\n"); + if (hend == NULL) { + log_warn("SERVER unable to find end of header in the HTTP template"); + return -1; + } + + hLen = hend+4-pdfTemplate; + + log_debug("SERVER calling pdfWrap for data1 with length %d", cnt); + outbuflen = pdfWrap(data1, cnt, hend+4, pdfTemplateSize-hLen, outbuf, HTTP_MSG_BUF_SIZE); + if (outbuflen < 0) { + log_warn("SERVER pdfWrap fails"); + return -1; + } + log_debug("SERVER pdfSteg sends resp with hdr len %d body len %d", hLen, outbuflen); + + + // debugging + // buf_dump((unsigned char *)data1, cnt, stderr); + + // data2len = pdfUnwrap(outbuf, outbuflen, data2, sbuflen); + // if ((int)sbuflen == data2len) { + // log_warn("sbuflen == data2len == %d", (int)sbuflen); + // if (memcmp(data1, data2, sbuflen) == 0) { + // log_warn("data1 and data2 match"); + // } else { + // log_warn("data1 and data2 DO NOT match!! Dumping data1 ..."); + // buf_dump((unsigned char *)data1, cnt, stderr); + // log_warn("data1 and data2 DO NOT match!! Dumping data2..."); + // buf_dump((unsigned char *)data2, data2len, stderr); + // } + // } else { + // log_warn("*** sbuflen = %d, data2len = %d *** Dumping data1 ...", (int)sbuflen, data2len); + // buf_dump((unsigned char *)data1, cnt, stderr); + // log_warn("*** sbuflen = %d, data2len = %d *** Dumping data2 ...", (int)sbuflen, data2len); + // buf_dump((unsigned char *)data2, data2len, stderr); + // } + + + if (evbuffer_add(dest, pdfTemplate, hLen)) { + log_warn("SERVER ERROR: evbuffer_add() fails for pdfTemplate"); + return -1; + } + if (evbuffer_add(dest, outbuf, outbuflen)) { + log_warn("SERVER ERROR: evbuffer_add() fails for outbuf"); + return -1; + } + + evbuffer_drain(source, sbuflen); + + conn_close_after_transmit(conn); + // downcast_steg(s)->have_transmitted = 1; + return 0; +} + + + +int +x_http2_handle_client_PDF_receive(steg_t *s, conn_t *conn, struct evbuffer *dest, struct evbuffer* source) { + struct evbuffer_ptr s2; + unsigned int response_len = 0, hdrLen; + char outbuf[HTTP_MSG_BUF_SIZE]; + int content_len = 0, outbuflen; + char *httpHdr, *httpBody; + + log_debug("Entering CLIENT PDF receive"); + + s2 = evbuffer_search(source, "\r\n\r\n", sizeof ("\r\n\r\n") -1 , NULL); + if (s2.pos == -1) { + log_warn("CLIENT Did not find end of HTTP header %d", (int) evbuffer_get_length(source)); + // evbuffer_dump(source, stderr); + return RECV_INCOMPLETE; + } + + log_debug("CLIENT received response header with len %d", (int)s2.pos); + + response_len = 0; + hdrLen = s2.pos + strlen("\r\n\r\n"); + response_len += hdrLen; + + httpHdr = (char *) evbuffer_pullup(source, s2.pos); + if (httpHdr == NULL) { + log_warn("CLIENT unable to pullup the complete HTTP header"); + return RECV_BAD; + } + + content_len = find_content_length(httpHdr, hdrLen); + if (content_len < 0) { + log_warn("CLIENT unable to find content length"); + return RECV_BAD; + } + log_debug("CLIENT received Content-Length = %d\n", content_len); + + response_len += content_len; + + if (response_len > evbuffer_get_length(source)) + return RECV_INCOMPLETE; + + httpHdr = (char *) evbuffer_pullup(source, response_len); + httpBody = httpHdr + hdrLen; + + + outbuflen = pdfUnwrap(httpBody, content_len, outbuf, HTTP_MSG_BUF_SIZE); + if (outbuflen < 0) { + log_warn("CLIENT ERROR: pdfUnwrap fails\n"); + return RECV_BAD; + } + + log_debug("CLIENT unwrapped data of length %d:", outbuflen); + + + // debugging + // buf_dump((unsigned char *)outbuf, outbuflen, stderr); + // ***** not sure why there is an extra char at the end of outbuf + outbuflen--; + + + if (evbuffer_add(dest, outbuf, outbuflen)) { + log_warn("CLIENT ERROR: evbuffer_add to dest fails\n"); + return RECV_BAD; + } + + // log_debug("Drained source for %d char\n", response_len); + if (evbuffer_drain(source, response_len) == -1) { + log_warn("CLIENT ERROR: failed to drain source\n"); + return RECV_BAD; + } + + // downcast_steg(s)->have_received = 1; + conn_expect_close(conn); + return RECV_GOOD; +} + + + + +/***** +int main() { + char data1[] = "this is a test?? yes!"; + char data2[100]; + char data3[100]; + int dlen1, dlen2, dlen3, end; + char last = ' '; + printf("hello world\n"); + + dlen2 = addDelimiter(data1, strlen(data1), data2, 100, '?', '.'); + printf("dlen2 = %d\n", dlen2); + dlen3 = removeDelimiter(data2, dlen2, data3, 100, '?', &end, &last); + printf("endflag = %d", end); + printf("dlen3 = %d\n", dlen3); + if (memcmp(data1, data3, dlen3) == 0) { + data1[dlen3] = 0; + printf("removeDelimiter(addDelimiter(x)) == x for |%s|\n", data1); + } else { + printf("removeDelimiter(addDelimiter(x)) != x for |%s|\n", data1); + } + return 1; +} + *****/ + +/***** +int main() { + char data1[] = "12345"; + char data2[] = "123456789012"; + char data3[] = "12345678901"; + char data4[] = "1234567890?"; + char pdf1[] = "[PDFHDR][STUFFS1]>>streamABCDEFGHIJYYendstream[STUFFS2]>>streamABCDEFGHIJYYendstream[STUFF3][PDFTRAILER]"; + char out[200]; + char orig[200]; + int r1, r2; + + printf("********************\n"); + printf("pdfwrap for %s\n", data1); + printf("strlen(pdf1) = %d\n", (int)strlen(pdf1)); + r1 = pdfWrap(data1, strlen(data1), pdf1, strlen(pdf1), out, (int)sizeof(out)); + if (r1 > 0) { + printf("pdfWrap returns %d\n", r1); + out[r1] = 0; + printf("out[] contains |%s|\n", out); + } else { + printf("pdfWrap returns %d\n", r1); + } + + r2 = pdfUnwrap(out, r1, orig, (int)sizeof(orig)); + if (r2 > 0) { + printf("pdfUnwrap returns %d\n", r2); + orig[r2] = 0; + printf("orig[] contains |%s|\n", orig); + } else { + printf("pdfUnwrap returns %d\n", r2); + } + + printf("********************\n"); + printf("pdfwrap for %s\n", data2); + r1 = pdfWrap(data2, strlen(data2), pdf1, strlen(pdf1), out, (int)sizeof(out)); + if (r1 > 0) { + printf("pdfWrap returns %d\n", r1); + out[r1] = 0; + printf("out[] contains |%s|\n", out); + } else { + printf("pdfWrap returns %d\n", r1); + } + + r2 = pdfUnwrap(out, r1, orig, (int)sizeof(orig)); + if (r2 > 0) { + printf("pdfUnwrap returns %d\n", r2); + orig[r2] = 0; + printf("orig[] contains |%s|\n", orig); + } else { + printf("pdfUnwrap returns %d\n", r2); + } + + printf("********************\n"); + printf("pdfwrap for %s\n", data3); + r1 = pdfWrap(data3, strlen(data3), pdf1, strlen(pdf1), out, (int)sizeof(out)); + if (r1 > 0) { + printf("pdfWrap returns %d\n", r1); + out[r1] = 0; + printf("out[] contains |%s|\n", out); + } else { + printf("pdfWrap returns %d\n", r1); + } + + r2 = pdfUnwrap(out, r1, orig, (int)sizeof(orig)); + if (r2 > 0) { + printf("pdfUnwrap returns %d\n", r2); + orig[r2] = 0; + printf("orig[] contains |%s|\n", orig); + } else { + printf("pdfUnwrap returns %d\n", r2); + } + + printf("********************\n"); + printf("pdfwrap for %s\n", data4); + r1 = pdfWrap(data4, strlen(data4), pdf1, strlen(pdf1), out, (int)sizeof(out)); + if (r1 > 0) { + printf("pdfWrap returns %d\n", r1); + out[r1] = 0; + printf("out[] contains |%s|\n", out); + } else { + printf("pdfWrap returns %d\n", r1); + } + + r2 = pdfUnwrap(out, r1, orig, (int)sizeof(orig)); + if (r2 > 0) { + printf("pdfUnwrap returns %d\n", r2); + orig[r2] = 0; + printf("orig[] contains |%s|\n", orig); + } else { + printf("pdfUnwrap returns %d\n", r2); + } + + return 0; +} + *****/ diff --git a/src/steg/pdfSteg.h b/src/steg/pdfSteg.h new file mode 100644 index 0000000..7e48449 --- /dev/null +++ b/src/steg/pdfSteg.h @@ -0,0 +1,29 @@ +#ifndef _PDFSTEG_H +#define _PDFSTEG_H + + +#include <stdio.h> +#include <string.h> +#include <ctype.h> +#include "util.h" +#include "connections.h" +#include "steg.h" +#include <event2/buffer.h> + + + +#define PDF_DELIMITER '?' +#define PDF_DELIMITER2 '.' + +int pdfWrap (char *data, unsigned int dlen, char *pdfTemplate, unsigned int plen, char *outbuf, unsigned int outbufsize); +int pdfUnwrap (char *data, unsigned int dlen, char *outbuf, unsigned int outbufsize); + +int addDelimiter(char *inbuf, int inbuflen, char *outbuf, int outbuflen, const char delimiter1, const char delimiter2); +int removeDelimiter(char *inbuf, int inbuflen, char *outbuf, int outbuflen, const char delimiter1, int* endFlag, int* escape); + +int x_http2_server_PDF_transmit (steg_t* s, struct evbuffer *source, conn_t *conn); +int +x_http2_handle_client_PDF_receive(steg_t *s, conn_t *conn, struct evbuffer *dest, struct evbuffer* source); + +#endif + diff --git a/src/steg/swfSteg.c b/src/steg/swfSteg.c new file mode 100644 index 0000000..ad3d5c8 --- /dev/null +++ b/src/steg/swfSteg.c @@ -0,0 +1,282 @@ +#include "swfSteg.h" + + +static const char http_response_1[] = + "HTTP/1.1 200 OK\r\n" + "Expires: Thu, 01 Jan 1970 00:00:00 GMT\r\n" + "Cache-Control: no-store\r\n" + "Connection: close\r\n" + "Content-Type: application/x-shockwave-flash\r\n" + "Content-Length: "; + + + + + + + + + + + +unsigned int +swf_wrap(char* inbuf, int in_len, char* outbuf, int out_sz) { + + char* swf; + int in_swf_len; + + char* tmp_buf; + int out_swf_len; + + char* resp; + int resp_len; + + char hdr[512]; + unsigned int hdr_len; + + char* tmp_buf2; + + + + if (!get_payload(HTTP_CONTENT_SWF, -1, &resp, &resp_len)) { + log_warn("swfsteg: no suitable payload found\n"); + return -1; + } + + swf = strstr(resp, "\r\n\r\n") + 4; + in_swf_len = resp_len - (swf - resp); + + + + + if (out_sz - in_len < (SWF_SAVE_FOOTER_LEN + SWF_SAVE_HEADER_LEN + 8 + 512)) { + fprintf(stderr, "swfsteg: outbuf too small %d \n", out_sz - in_len); + + log_warn("swfsteg: outbuf too small\n"); + return -1; + } + + + tmp_buf = malloc(in_len + SWF_SAVE_HEADER_LEN + SWF_SAVE_FOOTER_LEN); + + if (tmp_buf == NULL) { + log_warn("swfsteg: malloc failed\n"); + return -1; + } + + + tmp_buf2 = malloc(in_len + SWF_SAVE_HEADER_LEN + SWF_SAVE_FOOTER_LEN + 512); + + if (tmp_buf2 == NULL) { + free(tmp_buf); + log_warn("swfsteg: malloc failed\n"); + return -1; + } + + + memcpy(tmp_buf, swf+8, SWF_SAVE_HEADER_LEN); + memcpy(tmp_buf+SWF_SAVE_HEADER_LEN, inbuf, in_len); + memcpy(tmp_buf+SWF_SAVE_HEADER_LEN+in_len, swf + in_swf_len - SWF_SAVE_FOOTER_LEN, SWF_SAVE_FOOTER_LEN); + out_swf_len = def(tmp_buf, SWF_SAVE_HEADER_LEN + in_len + SWF_SAVE_FOOTER_LEN, tmp_buf2+8, + in_len + SWF_SAVE_HEADER_LEN + SWF_SAVE_FOOTER_LEN + 512-8, + Z_DEFAULT_COMPRESSION); + + // sprintf(hdr, "%s%d\r\n\r\n", http_response_1, out_swf_len + 8); + + + + // fprintf(stderr, "out_swf_len = %d\n", out_swf_len); + + + hdr_len = gen_response_header((char*) "application/x-shockwave-flash", 0, out_swf_len + 8, hdr, sizeof(hdr)); + + // fprintf(stderr, "hdr = %s\n", hdr); + + memcpy(tmp_buf2, swf, 4); + ((int*) (tmp_buf2))[1] = out_swf_len; + + memcpy(outbuf, hdr, hdr_len); + memcpy(outbuf+hdr_len, tmp_buf2, out_swf_len + 8); + + free(tmp_buf); + free(tmp_buf2); + return out_swf_len + 8 + hdr_len; +} + + + + +unsigned int +swf_unwrap(char* inbuf, int in_len, char* outbuf, int out_sz) { + char* tmp_buf; + int inf_len; + + tmp_buf = malloc(in_len * 8); + + inf_len = inf(inbuf + 8, in_len - 8, tmp_buf, in_len * 8); + + // fprintf(stderr, "in_swf_len = %d\n", in_len -8 ); + + + if (inf_len < 0 || out_sz < inf_len - SWF_SAVE_HEADER_LEN - SWF_SAVE_FOOTER_LEN) { + fprintf(stderr, "inf_len = %d\n", inf_len); + free(tmp_buf); + // buf_dump((unsigned char*) (inbuf+8), in_len -8, stderr); + + + + return -1; + } + + memcpy(outbuf, tmp_buf + SWF_SAVE_HEADER_LEN, inf_len - SWF_SAVE_HEADER_LEN - SWF_SAVE_FOOTER_LEN); + return inf_len - SWF_SAVE_HEADER_LEN - SWF_SAVE_FOOTER_LEN; +} + + + + + +int +x_http2_server_SWF_transmit (steg_t* s, struct evbuffer *source, conn_t *conn) { + + struct evbuffer *dest = conn_get_outbound(conn); + size_t sbuflen = evbuffer_get_length(source); + char* inbuf; + char* outbuf; + int outlen; + + + + inbuf = malloc(sbuflen); + + if (inbuf == NULL) { + log_warn("malloc inbuf failed\n"); + return -1; + } + + + if (evbuffer_remove(source, inbuf, sbuflen) == -1) { + log_debug("evbuffer_remove failed in x_http2_server_SWF_transmit"); + return -1; + } + + outbuf = malloc(4*sbuflen + SWF_SAVE_FOOTER_LEN + SWF_SAVE_HEADER_LEN + 512); + + if (outbuf == NULL) { + free(inbuf); + log_warn("malloc outbuf failed\n"); + return -1; + } + + // fprintf(stderr, "server wrapping swf len %d\n", (int) sbuflen); + outlen = swf_wrap(inbuf, sbuflen, outbuf, 4*sbuflen + SWF_SAVE_FOOTER_LEN + SWF_SAVE_HEADER_LEN + 512); + + if (outlen < 0) { + log_warn("swf_wrap failed\n"); + // fprintf(stderr, "swf_wrap failed\n"); + free(inbuf); + free(outbuf); + return -1; + } + + + if (evbuffer_add(dest, outbuf, outlen)) { + log_debug("SERVER ERROR: x_http2_server_transmit: evbuffer_add() fails for jsTemplate"); + free(inbuf); + free(outbuf); + return -1; + } + + + // conn_cease_transmission(conn); + conn_close_after_transmit(conn); + + + free(inbuf); + free(outbuf); + return 0; +} + + + + +int +x_http2_handle_client_SWF_receive(steg_t *s, conn_t *conn, struct evbuffer *dest, struct evbuffer* source) { + struct evbuffer_ptr s2; + unsigned int response_len = 0, hdrLen; + char outbuf[HTTP_MSG_BUF_SIZE]; + int content_len = 0, outbuflen; + char *httpHdr, *httpBody; + + + + s2 = evbuffer_search(source, "\r\n\r\n", sizeof ("\r\n\r\n") -1 , NULL); + if (s2.pos == -1) { + log_warn("CLIENT Did not find end of HTTP header %d", (int) evbuffer_get_length(source)); + fprintf(stderr, "client did not find end of HTTP header\n"); + + // evbuffer_dump(source, stderr); + return RECV_INCOMPLETE; + } + + log_debug("CLIENT received response header with len %d", (int)s2.pos); + + response_len = 0; + hdrLen = s2.pos + strlen("\r\n\r\n"); + response_len += hdrLen; + + httpHdr = (char *) evbuffer_pullup(source, s2.pos); + if (httpHdr == NULL) { + log_warn("CLIENT unable to pullup the complete HTTP header"); + return RECV_BAD; + } + + content_len = find_content_length(httpHdr, hdrLen); + + + + if (content_len < 0) { + log_warn("CLIENT unable to find content length"); + return RECV_BAD; + } + log_debug("CLIENT received Content-Length = %d\n", content_len); + + + + response_len += content_len; + + if (response_len > evbuffer_get_length(source)) + return RECV_INCOMPLETE; + + + + httpHdr = (char *) evbuffer_pullup(source, response_len); + httpBody = httpHdr + hdrLen; + + + outbuflen = swf_unwrap(httpBody, content_len, outbuf, HTTP_MSG_BUF_SIZE); + + if (outbuflen < 0) { + fprintf(stderr, "swf_unwrap failed\n"); + log_debug("CLIENT ERROR: swf_unwrap failed\n"); + return RECV_BAD; + } + + // fprintf(stderr, "CLIENT unwrapped data of length %d:", outbuflen); + // buf_dump(outbuf, outbuflen, stderr); + + if (evbuffer_add(dest, outbuf, outbuflen)) { + log_debug("CLIENT ERROR: evbuffer_add to dest fails\n"); + return RECV_BAD; + } + + // log_debug("Drained source for %d char\n", response_len); + if (evbuffer_drain(source, response_len) == -1) { + log_debug("CLIENT ERROR: failed to drain source\n"); + return RECV_BAD; + } + + // downcast_steg(s)->have_received = 1; + conn_expect_close(conn); + return RECV_GOOD; +} diff --git a/src/steg/swfSteg.c.old b/src/steg/swfSteg.c.old new file mode 100644 index 0000000..833ea9f --- /dev/null +++ b/src/steg/swfSteg.c.old @@ -0,0 +1,264 @@ +#include "swfSteg.h" + + +static const char http_response_1[] = + "HTTP/1.1 200 OK\r\n" + "Expires: Thu, 01 Jan 1970 00:00:00 GMT\r\n" + "Cache-Control: no-store\r\n" + "Connection: close\r\n" + "Content-Type: application/x-shockwave-flash\r\n" + "Content-Length: "; + + + + + + + + + + + +unsigned int +swf_wrap(char* inbuf, int in_len, char* outbuf, int out_sz) { + + char* swf; + int in_swf_len; + + char* tmp_buf; + int out_swf_len; + + char* resp; + int resp_len; + + char hdr[512]; + unsigned int hdr_len; + + char* tmp_buf2; + + + + if (!get_payload(HTTP_CONTENT_SWF, -1, &resp, &resp_len)) { + log_warn("swfsteg: no suitable payload found\n"); + return -1; + } + + swf = strstr(resp, "\r\n\r\n") + 4; + in_swf_len = resp_len - (swf - resp); + + + + + if (out_sz - in_len < (SWF_SAVE_FOOTER_LEN + SWF_SAVE_HEADER_LEN + 8 + 512)) { + fprintf(stderr, "swfsteg: outbuf too small %d \n", out_sz - in_len); + + log_warn("swfsteg: outbuf too small\n"); + return -1; + } + + + tmp_buf = malloc(in_len + SWF_SAVE_HEADER_LEN + SWF_SAVE_FOOTER_LEN); + + if (tmp_buf == NULL) { + log_warn("swfsteg: malloc failed\n"); + return -1; + } + + + tmp_buf2 = malloc(in_len + SWF_SAVE_HEADER_LEN + SWF_SAVE_FOOTER_LEN + 512); + + if (tmp_buf2 == NULL) { + free(tmp_buf); + log_warn("swfsteg: malloc failed\n"); + return -1; + } + + + memcpy(tmp_buf, swf+8, SWF_SAVE_HEADER_LEN); + memcpy(tmp_buf+SWF_SAVE_HEADER_LEN, inbuf, in_len); + memcpy(tmp_buf+SWF_SAVE_HEADER_LEN+in_len, swf + in_swf_len - SWF_SAVE_FOOTER_LEN, SWF_SAVE_FOOTER_LEN); + out_swf_len = def(tmp_buf, SWF_SAVE_HEADER_LEN + in_len + SWF_SAVE_FOOTER_LEN, tmp_buf2+8, + in_len + SWF_SAVE_HEADER_LEN + SWF_SAVE_FOOTER_LEN + 512-8, + Z_DEFAULT_COMPRESSION); + + // sprintf(hdr, "%s%d\r\n\r\n", http_response_1, out_swf_len + 8); + + + hdr_len = gen_response_header((char*) "application/x-shockwave-flash", 0, out_swf_len + 8, hdr, sizeof(hdr)); + + // fprintf(stderr, "hdr = %s\n", hdr); + + memcpy(tmp_buf2, swf, 4); + ((int*) (tmp_buf2))[1] = out_swf_len; + + memcpy(outbuf, hdr, hdr_len); + memcpy(outbuf+hdr_len, tmp_buf2, out_swf_len + 8); + + free(tmp_buf); + free(tmp_buf2); + return out_swf_len + 8 + hdr_len; +} + + + +unsigned int +swf_unwrap(char* inbuf, int in_len, char* outbuf, int out_sz) { + char* tmp_buf; + int inf_len; + + tmp_buf = malloc(in_len * 5); + inf_len = inf(inbuf + 8, in_len - 8, tmp_buf, in_len * 5); + + if (inf_len < 0 || out_sz < inf_len - SWF_SAVE_HEADER_LEN - SWF_SAVE_FOOTER_LEN) { + fprintf(stderr, "swf_unwrap failed \n"); + free(tmp_buf); + return -1; + } + + memcpy(outbuf, tmp_buf + SWF_SAVE_HEADER_LEN, inf_len - SWF_SAVE_HEADER_LEN - SWF_SAVE_FOOTER_LEN); + return inf_len - SWF_SAVE_HEADER_LEN - SWF_SAVE_FOOTER_LEN; +} + + + + + +int +x_http2_server_SWF_transmit (steg_t* s, struct evbuffer *source, conn_t *conn) { + + struct evbuffer *dest = conn_get_outbound(conn); + size_t sbuflen = evbuffer_get_length(source); + char* inbuf; + char* outbuf; + int outlen; + + + + inbuf = malloc(sbuflen); + + if (inbuf == NULL) { + log_warn("malloc inbuf failed\n"); + return -1; + } + + + if (evbuffer_remove(source, inbuf, sbuflen) == -1) { + log_debug("evbuffer_remove failed in x_http2_server_SWF_transmit"); + return -1; + } + + outbuf = malloc(4*sbuflen + SWF_SAVE_FOOTER_LEN + SWF_SAVE_HEADER_LEN + 512); + + if (outbuf == NULL) { + free(inbuf); + log_warn("malloc outbuf failed\n"); + return -1; + } + + // fprintf(stderr, "server wrapping swf len %d\n", (int) sbuflen); + outlen = swf_wrap(inbuf, sbuflen, outbuf, 4*sbuflen + SWF_SAVE_FOOTER_LEN + SWF_SAVE_HEADER_LEN + 512); + + if (outlen < 0) { + log_warn("swf_wrap failed\n"); + fprintf(stderr, "swf_wrap failed\n"); + free(inbuf); + free(outbuf); + return -1; + } + + + if (evbuffer_add(dest, outbuf, outlen)) { + log_debug("SERVER ERROR: x_http2_server_transmit: evbuffer_add() fails for jsTemplate"); + free(inbuf); + free(outbuf); + return -1; + } + + + // conn_cease_transmission(conn); + conn_close_after_transmit(conn); + + + free(inbuf); + free(outbuf); + return 0; +} + + + + +int +x_http2_handle_client_SWF_receive(steg_t *s, conn_t *conn, struct evbuffer *dest, struct evbuffer* source) { + struct evbuffer_ptr s2; + unsigned int response_len = 0, hdrLen; + char outbuf[HTTP_MSG_BUF_SIZE]; + int content_len = 0, outbuflen; + char *httpHdr, *httpBody; + + + + s2 = evbuffer_search(source, "\r\n\r\n", sizeof ("\r\n\r\n") -1 , NULL); + if (s2.pos == -1) { + log_warn("CLIENT Did not find end of HTTP header %d", (int) evbuffer_get_length(source)); + fprintf(stderr, "client did not find end of HTTP header\n"); + + // evbuffer_dump(source, stderr); + return RECV_INCOMPLETE; + } + + log_debug("CLIENT received response header with len %d", (int)s2.pos); + + response_len = 0; + hdrLen = s2.pos + strlen("\r\n\r\n"); + response_len += hdrLen; + + httpHdr = (char *) evbuffer_pullup(source, s2.pos); + if (httpHdr == NULL) { + log_warn("CLIENT unable to pullup the complete HTTP header"); + return RECV_BAD; + } + + content_len = find_content_length(httpHdr, hdrLen); + + + + if (content_len < 0) { + log_warn("CLIENT unable to find content length"); + return RECV_BAD; + } + log_debug("CLIENT received Content-Length = %d\n", content_len); + + + + response_len += content_len; + + if (response_len > evbuffer_get_length(source)) + return RECV_INCOMPLETE; + + httpBody = httpHdr + hdrLen; + outbuflen = swf_unwrap(httpBody, content_len, outbuf, HTTP_MSG_BUF_SIZE); + + if (outbuflen < 0) { + fprintf(stderr, "swf_unwrap failed\n"); + log_debug("CLIENT ERROR: swf_unwrap failed\n"); + return RECV_BAD; + } + + // fprintf(stderr, "CLIENT unwrapped data of length %d:", outbuflen); + // buf_dump(outbuf, outbuflen, stderr); + + if (evbuffer_add(dest, outbuf, outbuflen)) { + log_debug("CLIENT ERROR: evbuffer_add to dest fails\n"); + return RECV_BAD; + } + + // log_debug("Drained source for %d char\n", response_len); + if (evbuffer_drain(source, response_len) == -1) { + log_debug("CLIENT ERROR: failed to drain source\n"); + return RECV_BAD; + } + + // downcast_steg(s)->have_received = 1; + conn_expect_close(conn); + return RECV_GOOD; +} diff --git a/src/steg/swfSteg.h b/src/steg/swfSteg.h new file mode 100644 index 0000000..dc6bc04 --- /dev/null +++ b/src/steg/swfSteg.h @@ -0,0 +1,42 @@ +#ifndef _SWFSTEG_H +#define _SWFSTEG_H + + +#include "util.h" +#include "connections.h" +#include "steg.h" +#include "payloads.h" +#include "cookies.h" +#include "pdfSteg.h" +#include "zpack.h" + + +#include <event2/buffer.h> +#include <stdio.h> + + + + + + + +#define SWF_SAVE_HEADER_LEN 1500 +#define SWF_SAVE_FOOTER_LEN 1500 + + +unsigned int +swf_wrap(char* inbuf, int in_len, char* outbuf, int out_sz); + +unsigned int +swf_unwrap(char* inbuf, int in_len, char* outbuf, int out_sz); + +int +x_http2_server_SWF_transmit (steg_t* s, struct evbuffer *source, conn_t *conn); + + +int +x_http2_handle_client_SWF_receive(steg_t *s, conn_t *conn, struct evbuffer *dest, struct evbuffer* source); + +#endif + + diff --git a/src/steg/x_http.c.old b/src/steg/x_http.c.old new file mode 100644 index 0000000..2799078 --- /dev/null +++ b/src/steg/x_http.c.old @@ -0,0 +1,337 @@ +/* Copyright 2011 Zack Weinberg + See LICENSE for other credits and copying information +*/ + +#include "util.h" +#include "connections.h" +#include "steg.h" + +#include <event2/buffer.h> + + + +/* This is an example steganography module. Don't use it to disguise real + traffic! It packages client->server traffic as HTTP GET requests and + server->client traffic as HTTP responses, but makes no actual attempt + to obscure the data proper. */ + +struct x_http_steg_t +{ + steg_t super; + /* no extra stuff is presently necessary */ +}; + +STEG_DEFINE_MODULE(x_http, + 1024, /* client-server max data rate - made up */ + 10240, /* server-client max data rate - ditto */ + 1, /* max concurrent connections per IP */ + 1); /* max concurrent IPs */ + +/* Canned HTTP query and response headers. */ +static const char http_query_1[] = + "GET /"; +static const char http_query_2[] = + " HTTP/1.1\r\n" + "Host: "; +static const char http_query_3[] = + "\r\n" + "Connection: close\r\n\r\n"; + +static const char http_response_1[] = + "HTTP/1.1 200 OK\r\n" + "Expires: Thu, 01 Jan 1970 00:00:00 GMT\r\n" + "Cache-Control: no-store\r\n" + "Connection: close\r\n" + "Content-Type: application/octet-stream\r\n" + "Content-Length: "; +static const char http_response_2[] = + "%lu\r\n" + "\r\n"; + + +steg_t * +x_http_new(rng_t *rng, unsigned int is_clientside) +{ + STEG_NEW(x_http, state, rng, is_clientside); + /* if there were extra stuff to fill in, you would do it here */ + return upcast_steg(state); +} + +void +x_http_del(steg_t *s) +{ + x_http_steg_t *state = downcast_steg(s); + STEG_DEL(s); + /* if there were extra stuff to deallocate, you would do it here */ + free(state); +} + +unsigned int +x_http_detect(conn_t *conn) +{ + struct evbuffer *buf = conn_get_inbound(conn); + unsigned char *data; + + return 0; + + /* Look for the text of http_response_1. */ + if (evbuffer_get_length(buf) >= sizeof http_response_1 - 1) { + data = evbuffer_pullup(buf, sizeof http_response_1 - 1); + if (!memcmp(data, http_response_1, sizeof http_response_1 - 1)) + return 1; + } + + /* The client always transmits "GET /" followed by at least four + characters that are either lowercase hex digits or equals + signs, so we need nine bytes of incoming data. */ + if (evbuffer_get_length(buf) >= 9) { + data = evbuffer_pullup(buf, 9); + if (!memcmp(data, "GET /", 5) && + (ascii_isxdigit(data[5]) || data[5] == '=') && + (ascii_isxdigit(data[6]) || data[6] == '=') && + (ascii_isxdigit(data[7]) || data[7] == '=') && + (ascii_isxdigit(data[8]) || data[8] == '=')) + return 1; + } + + /* Didn't find either the client or the server pattern. */ + return 0; +} + +size_t +x_http_transmit_room(steg_t *s, conn_t *conn) +{ + if (s->is_clientside) + /* per http://www.boutell.com/newfaq/misc/urllength.html, + IE<9 can handle no more than 2048 characters in the path + component of a URL; we're not talking to IE, but this limit + means longer paths look fishy; we hex-encode the path, so + we have to cut the number in half. */ + return 1024; + else + /* no practical limit applies */ + return SIZE_MAX; +} + +int +x_http_transmit(steg_t *s, struct evbuffer *source, conn_t *conn) +{ + struct evbuffer *dest = conn_get_outbound(conn); + + if (s->is_clientside) { + /* On the client side, we have to embed the data in a GET query somehow; + the only plausible places to put it are the URL and cookies. This + presently uses the URL. And it can't be binary. */ + struct evbuffer *scratch; + struct evbuffer_iovec *iv; + int i, nv; + + /* Convert all the data in 'source' to hexadecimal and write it to + 'scratch'. Data is padded to a multiple of four characters with + equals signs. */ + size_t slen = evbuffer_get_length(source); + size_t dlen = slen * 2; + + dlen = dlen + 3 - (dlen-1)%4; + if (dlen == 0) dlen = 4; + + scratch = evbuffer_new(); + if (!scratch) return -1; + if (evbuffer_expand(scratch, dlen)) { + evbuffer_free(scratch); + return -1; + } + + nv = evbuffer_peek(source, slen, NULL, NULL, 0); + iv = xzalloc(sizeof(struct evbuffer_iovec) * nv); + if (evbuffer_peek(source, slen, NULL, iv, nv) != nv) { + evbuffer_free(scratch); + free(iv); + return -1; + } + + for (i = 0; i < nv; i++) { + const unsigned char *p = iv[i].iov_base; + const unsigned char *limit = p + iv[i].iov_len; + char hex[2], c; + while (p < limit) { + c = *p++; + hex[0] = "0123456789abcdef"[(c & 0xF0) >> 4]; + hex[1] = "0123456789abcdef"[(c & 0x0F) >> 0]; + evbuffer_add(scratch, hex, 2); + } + } + free(iv); + while (evbuffer_get_length(scratch) == 0 || + evbuffer_get_length(scratch) % 4 != 0) + evbuffer_add(scratch, "=", 1); + + if (evbuffer_add(dest, http_query_1, sizeof http_query_1-1) || + evbuffer_add_buffer(dest, scratch) || + evbuffer_add(dest, http_query_2, sizeof http_query_2-1) || + evbuffer_add(dest, conn->peername, strlen(conn->peername)) || + evbuffer_add(dest, http_query_3, sizeof http_query_3-1)) { + evbuffer_free(scratch); + return -1; + } + + evbuffer_free(scratch); + evbuffer_drain(source, slen); + conn_cease_transmission(conn); + return 0; + + } else { + /* On the server side, we just fake up some HTTP response headers and + then splat the data we were given. Binary is OK. */ + + if (evbuffer_add(dest, http_response_1, sizeof http_response_1-1)) + return -1; + if (evbuffer_add_printf(dest, http_response_2, + (unsigned long)evbuffer_get_length(source)) == -1) + return -1; + if (evbuffer_add_buffer(dest, source)) + return -1; + + conn_close_after_transmit(conn); + return 0; + } +} + +// enum recv_ret +static int +x_http_receive(steg_t *s, conn_t *conn, struct evbuffer *dest) +{ + struct evbuffer *source = conn_get_inbound(conn); + if (s->is_clientside) { + /* This loop should not be necessary, but we are currently not + enforcing the query-response pattern, so we can get more than + one response per request. */ + do { + /* Linearize the buffer out past the longest possible + Content-Length header and subsequent blank line. 2**64 fits in + 20 characters, and then we have two CRLFs; minus one for the + NUL in sizeof http_response_1. Note that this does _not_ + guarantee that that much data is available. */ + + unsigned char *data = evbuffer_pullup(source, sizeof http_response_1 + 23); + size_t hlen = evbuffer_get_length(source); + if (hlen > sizeof http_response_1 + 23) + hlen = sizeof http_response_1 + 23; + + /* Validate response headers. */ + if (hlen < sizeof http_response_1 - 1) + return 0; // RECV_INCOMPLETE; + if (memcmp(data, http_response_1, sizeof http_response_1 - 1)) + return -1; // RECV_BAD; + + /* There should be an unsigned number immediately after the text of + http_response_1, followed by the four characters \r\n\r\n. + We may not have the complete number yet. */ + unsigned char *p = data + sizeof http_response_1 - 1; + unsigned char *limit = data + hlen; + uint64_t clen = 0; + while (p < limit && '0' <= *p && *p <= '9') { + clen = clen*10 + *p - '0'; + p++; + } + if (p+4 > limit) + return 0; // RECV_INCOMPLETE; + if (p[0] != '\r' || p[1] != '\n' || p[2] != '\r' || p[3] != '\n') + return -1; // RECV_BAD; + + p += 4; + hlen = p - data; + /* Now we know how much data we're expecting after the blank line. */ + if (evbuffer_get_length(source) < hlen + clen) + return 0; // RECV_INCOMPLETE; + + /* we are go */ + if (evbuffer_drain(source, hlen)) + return -1; // RECV_BAD; + + if (evbuffer_remove_buffer(source, dest, clen) != clen) + return -1; // RECV_BAD; + + } while (evbuffer_get_length(source)); + + conn_expect_close(conn); + return 0; // RECV_GOOD; + } else { + /* We need a scratch buffer here because the contract is that if + we hit a decode error we *don't* write anything to 'dest'. */ + struct evbuffer *scratch; + + /* This loop should not be necessary either, but is, for the same + reason given above */ + do { + /* Search for the second and third invariant bits of the query headers + we expect. We completely ignore the contents of the Host header. */ + struct evbuffer_ptr s2 = evbuffer_search(source, http_query_2, + sizeof http_query_2 - 1, + NULL); + if (s2.pos == -1) { + log_debug("Did not find second piece of HTTP query"); + return 0; // RECV_INCOMPLETE; + } + struct evbuffer_ptr s3 = evbuffer_search(source, http_query_3, + sizeof http_query_3 - 1, + &s2); + if (s3.pos == -1) { + log_debug("Did not find third piece of HTTP query"); + return 0; // RECV_INCOMPLETE; + } + obfs_assert(s3.pos + sizeof http_query_3 - 1 + <= evbuffer_get_length(source)); + + unsigned char *data = evbuffer_pullup(source, s2.pos); + if (memcmp(data, "GET /", sizeof "GET /"-1)) { + log_debug("Unexpected HTTP verb: %.*s", 5, data); + return -1; // RECV_BAD; + } + + unsigned char *p = data + sizeof "GET /"-1; + unsigned char *limit = data + s2.pos; + + scratch = evbuffer_new(); + if (!scratch) return -1; // RECV_BAD; + if (evbuffer_expand(scratch, (limit - p)/2)) { + evbuffer_free(scratch); + return -1; // RECV_BAD; + } + + unsigned char c, h, secondhalf = 0; + while (p < limit) { + if (!secondhalf) c = 0; + if ('0' <= *p && *p <= '9') h = *p - '0'; + else if ('a' <= *p && *p <= 'f') h = *p - 'a' + 10; + else if ('A' <= *p && *p <= 'F') h = *p - 'A' + 10; + else if (*p == '=' && !secondhalf) { + p++; + continue; + } else { + evbuffer_free(scratch); + log_debug("Decode error: unexpected URI character %c", *p); + return -1; // RECV_BAD; + } + + c = (c << 4) + h; + if (secondhalf) + evbuffer_add(scratch, &c, 1); + secondhalf = !secondhalf; + p++; + } + + if (evbuffer_add_buffer(dest, scratch)) { + evbuffer_free(scratch); + log_debug("Failed to transfer buffer"); + return -1; // RECV_BAD; + } + evbuffer_drain(source, s3.pos + sizeof http_query_3 - 1); + evbuffer_free(scratch); + + } while (evbuffer_get_length(source)); + + conn_transmit_soon(conn, 100); + return 0; // RECV_GOOD; + } +} diff --git a/src/steg/x_http2.c b/src/steg/x_http2.c new file mode 100644 index 0000000..0710fa7 --- /dev/null +++ b/src/steg/x_http2.c @@ -0,0 +1,700 @@ +/* Copyright (c) 2011, SRI International + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + + * Neither the names of the copyright owners nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + Contributors: Zack Weinberg, Vinod Yegneswaran + See LICENSE for other credits and copying information +*/ + + + +#include "util.h" +#include "connections.h" +#include "steg.h" +#include "payloads.h" +#include "cookies.h" +#include "swfSteg.h" +#include "pdfSteg.h" +#include "jsSteg.h" + +#include <event2/buffer.h> +#include <stdio.h> + + + + +#define MIN_COOKIE_SIZE 128 +#define MAX_COOKIE_SIZE 2048 + + +int +x_http2_server_receive(steg_t *s, conn_t *conn, struct evbuffer *dest, struct evbuffer* source); + +int +lookup_peer_name_from_ip(char* p_ip, char* p_name); + + +static int has_peer_name = 0; +static char peername[512]; + + +struct x_http2_steg_t +{ + steg_t super; + + int have_transmitted; + int have_received; + int type; +}; + + +STEG_DEFINE_MODULE(x_http2, + 1024, /* client-server max data rate - made up */ + 10240, /* server-client max data rate - ditto */ + 1, /* max concurrent connections per IP */ + 1); /* max concurrent IPs */ + + + + + + +int x_http2_client_transmit (steg_t *s, struct evbuffer *source, conn_t *conn); + +void evbuffer_dump(struct evbuffer *buf, FILE *out); +void buf_dump(unsigned char* buf, int len, FILE *out); + + + +void +evbuffer_dump(struct evbuffer *buf, FILE *out) +{ + int nextent = evbuffer_peek(buf, SSIZE_MAX, 0, 0, 0); + struct evbuffer_iovec v[nextent]; + int i; + const unsigned char *p, *limit; + + if (evbuffer_peek(buf, -1, 0, v, nextent) != nextent) + abort(); + + for (i = 0; i < nextent; i++) { + p = v[i].iov_base; + limit = p + v[i].iov_len; + + putc('|', out); + while (p < limit) { + if (*p < 0x20 || *p >= 0x7F || *p == '\\' || *p == '|') + fprintf(out, "\\x%02x", *p); + else + putc(*p, out); + p++; + } + } + putc('|', out); +} + + + + + +void +buf_dump(unsigned char* buf, int len, FILE *out) +{ + int i=0; + putc('|', out); + while (i < len) { + if (buf[i] < 0x20 || buf[i] >= 0x7F || buf[i] == '\\' || buf[i]== '|') + fprintf(out, "\\x%02x", buf[i]); + else + putc(buf[i], out); + i++; + } + putc('|', out); + putc('\n', out); +} + + + + + +steg_t * +x_http2_new(rng_t *rng, unsigned int is_clientside) +{ + + STEG_NEW(x_http2, state, rng, is_clientside); + + if (is_clientside) + load_payloads("traces/client.out"); + else { + load_payloads("traces/server.out"); + init_JS_payload_pool(HTTP_MSG_BUF_SIZE, TYPE_HTTP_RESPONSE, JS_MIN_AVAIL_SIZE); + // init_PDF_payload_pool(HTTP_MSG_BUF_SIZE, TYPE_HTTP_RESPONSE, PDF_MIN_AVAIL_SIZE); + init_SWF_payload_pool(HTTP_MSG_BUF_SIZE, TYPE_HTTP_RESPONSE, 0); + } + + + /* if there were extra stuff to fill in, you would do it here */ + return upcast_steg(state); +} + +void +x_http2_del(steg_t *s) +{ + x_http2_steg_t *state = downcast_steg(s); + + STEG_DEL(s); + + /* if there were extra stuff to deallocate, you would do it here */ + free(state); +} + + +// x_http2_detect determines if a packet should be processed by the http2 steg module +unsigned int +x_http2_detect(conn_t *conn) +{ + struct evbuffer *buf = conn_get_inbound(conn); + unsigned char *data; + + // return 0; +/***** + Here is a list of HTTP response codes extracted from the server-portals.out trace + +7369 HTTP/1.1 200 OK + 470 HTTP/1.1 302 Found + 350 HTTP/1.1 304 Not Modified + 212 HTTP/1.1 302 Moved Temporarily + 184 HTTP/1.1 204 No Content + 451 HTTP/1.0 200 OK + 36 HTTP/1.0 204 No Content + 21 HTTP/1.1 301 Moved Permanently + 19 HTTP/1.1 302 Object moved + 15 HTTP/1.1 404 Not Found + + 7 HTTP/1.0 304 Not Modified + 6 HTTP/1.1 302 Redirect + 3 HTTP/1.0 200 Ok + 2 HTTP/1.1 303 Object Moved + 2 HTTP/1.0 301 Moved Permanently + 2 HTTP/1.0 302 Moved Temporarily + 2 HTTP/1.0 400 Bad request + 2 HTTP/1.0 403 Forbidden + 1 HTTP/1.0 404 Not Found + 1 HTTP/1.1 200 + 1 HTTP/1.1 302 FOUND + 1 HTTP/1.1 304 + 1 HTTP/1.1 400 Bad Request + 1 HTTP/1.1 403 Forbidden + 1 HTTP/1.1 503 Service Unavailable. + *****/ + + // The first part of a valid HTTP response should be of the form + // HTTP/1.x nnn + + if (evbuffer_get_length(buf) >= 12) { + data = evbuffer_pullup(buf, 12); + + if (data != NULL && + ((!memcmp(data, "HTTP/1.1 200", 12)) || + (!memcmp(data, "HTTP/1.1 302", 12)) || + (!memcmp(data, "HTTP/1.1 304", 12)) || + (!memcmp(data, "HTTP/1.1 204", 12)) || + (!memcmp(data, "HTTP/1.0 200", 12)) || + (!memcmp(data, "HTTP/1.0 204", 12)) || + (!memcmp(data, "HTTP/1.1 301", 12)) || + (!memcmp(data, "HTTP/1.1 302", 12)) || + (!memcmp(data, "HTTP/1.1 404", 12)))) { + log_debug("x_http2_detect: valid response"); + return 1; + } + } + + + + + + // SC: if we are only interested in jsSteg, we may want to + // consider HTTP/1.1 and HTTP/1.0 responses whose code is 200 only + + // check to see if this is a valid HTTP request + // + // the following is for HTTP requests used by the http2 steg module + // The client always transmits "GET /" followed by at least four + // characters that are either lowercase hex digits or equals + // signs, so we need nine bytes of incoming data. + + + + if (evbuffer_get_length(buf) >= 9) { + data = evbuffer_pullup(buf, 9); + if (data != NULL && (!memcmp(data, "GET /", 5) || !memcmp(data, "POST /", 5) || !memcmp(data, "Cookie", 6))) { + log_debug("x_http2_detect: valid request"); + return 1; + } + } + + log_debug("x_http2_detect: didn't find either HTTP request or response"); + /* Didn't find either the client or the server pattern. */ + return 0; +} + +size_t +x_http2_transmit_room(steg_t *s, conn_t *conn) +{ + unsigned int mjc; + + if (downcast_steg(s)->have_transmitted) + /* can't send any more on this connection */ + return 0; + + + if (s->is_clientside) { + /* per http://www.boutell.com/newfaq/misc/urllength.html, + IE<9 can handle no more than 2048 characters in the path + component of a URL; we're not talking to IE, but this limit + means longer paths look fishy; we hex-encode the path, so + we have to cut the number in half. */ + return (MIN_COOKIE_SIZE + rand() % (MAX_COOKIE_SIZE - MIN_COOKIE_SIZE)) / 4; + // return 1024; + } + else { + + if (!downcast_steg(s)->have_received) + return 0; + + switch(downcast_steg(s)->type) { + + case HTTP_CONTENT_SWF: + return 1024; + + case HTTP_CONTENT_JAVASCRIPT: + mjc = get_max_JS_capacity() / 2; + if (mjc > 1024) { + // it should be 1024 + ...., but seems like we need to be a little bit smaller (chopper bug?) + int rval = 512 + rand()%(mjc - 1024); + // fprintf(stderr, "returning rval %d, mjc %d\n", rval, mjc); + return rval; + } + log_warn("js capacity too small\n"); + exit(-1); + + case HTTP_CONTENT_PDF: + // return 1024 + rand()%(get_max_PDF_capacity() - 1024) + return PDF_MIN_AVAIL_SIZE; + } + + return SIZE_MAX; + } +} + + + + + + +int +lookup_peer_name_from_ip(char* p_ip, char* p_name) { + struct addrinfo* ailist; + struct addrinfo* aip; + struct addrinfo hint; + char buf[128]; + + hint.ai_flags = AI_CANONNAME; + hint.ai_family = 0; + hint.ai_socktype = 0; + hint.ai_protocol = 0; + hint.ai_addrlen = 0; + hint.ai_canonname = NULL; + hint.ai_addr = NULL; + hint.ai_next = NULL; + + strcpy(buf, p_ip); + buf[strchr(buf, ':') - buf] = 0; + + + if (getaddrinfo(buf, NULL, &hint, &ailist)) { + fprintf(stderr, "error: getaddrinfo() %s\n", p_ip); + exit(1); + } + + for (aip = ailist; aip != NULL; aip = aip->ai_next) { + char buf[512]; + if (getnameinfo(aip->ai_addr, sizeof(struct sockaddr), buf, 512, NULL, 0, 0) == 0) { + sprintf(p_name, "%s", buf); + return 1; + } + } + + return 0; +} + + + + + + + + +int +x_http2_client_transmit (steg_t *s, struct evbuffer *source, conn_t *conn) { + + /* On the client side, we have to embed the data in a GET query somehow; + the only plausible places to put it are the URL and cookies. This + presently uses the URL. And it can't be binary. */ + // struct evbuffer *scratch; + struct evbuffer_iovec *iv; + int i, nv; + struct evbuffer *dest = conn_get_outbound(conn); + size_t sbuflen = evbuffer_get_length(source); + char buf[10000]; + unsigned char data[(int) sbuflen*2]; + // unsigned char outbuf[MAX_COOKIE_SIZE]; + + unsigned char outbuf[(int) sbuflen*8]; + int datalen; + + + // size_t sofar = 0; + size_t cookie_len; + + + /* Convert all the data in 'source' to hexadecimal and write it to + 'scratch'. Data is padded to a multiple of four characters with + equals signs. */ + + + unsigned int len = 0; + unsigned int cnt = 0; + + + + datalen = 0; + cookie_len = 4 * sbuflen + rand() % 4; + + + nv = evbuffer_peek(source, sbuflen, NULL, NULL, 0); + iv = xzalloc(sizeof(struct evbuffer_iovec) * nv); + + if (evbuffer_peek(source, sbuflen, NULL, iv, nv) != nv) { + free(iv); + return -1; + } + + // retry up to 10 times + while (!len) { + len = find_client_payload(buf, sizeof(buf), TYPE_HTTP_REQUEST); + if (cnt++ == 10) return -1; + } + + + if (has_peer_name == 0 && lookup_peer_name_from_ip((char*) conn->peername, peername)) + has_peer_name = 1; + + // if (find_uri_type(buf) != HTTP_CONTENT_SWF) { + // fprintf(stderr, "%s\n", buf); + // exit(-1); + // } + + + + cnt = 0; + + for (i = 0; i < nv; i++) { + const unsigned char *p = iv[i].iov_base; + const unsigned char *limit = p + iv[i].iov_len; + char c; + while (p < limit && cnt < sbuflen) { + c = *p++; + data[datalen] = "0123456789abcdef"[(c & 0xF0) >> 4]; + data[datalen+1] = "0123456789abcdef"[(c & 0x0F) >> 0]; + datalen += 2; + cnt++; + } + } + + free(iv); + + if (cookie_len < 4) cookie_len = 4; + + datalen = gen_cookie_field(outbuf, cookie_len, data, datalen); + log_debug("CLIENT: sending cookie of length = %d %d\n", datalen, (int) cookie_len); + // fprintf(stderr, "CLIENT: sending cookie of length = %d %d\n", datalen, (int) cookie_len); + + if (datalen < 0) { + log_debug("cookie generation failed\n"); + return -1; + } + + + if (evbuffer_add(dest, buf, strstr(buf, "\r\n") - buf + 2) || // add uri field + evbuffer_add(dest, "Host: ", 6) || + evbuffer_add(dest, peername, strlen(peername)) || + evbuffer_add(dest, strstr(buf, "\r\n"), len - (unsigned int) (strstr(buf, "\r\n") - buf)) || // add everything but first line + evbuffer_add(dest, "Cookie: ", 8) || + evbuffer_add(dest, outbuf, cookie_len) || + evbuffer_add(dest, "\r\n\r\n", 4)) { + log_debug("error ***********************"); + return -1; + } + + // sofar += datalen/2; + evbuffer_drain(source, datalen/2); + + log_debug("CLIENT TRANSMITTED payload %d\n", (int) sbuflen); + + conn_cease_transmission(conn); + + downcast_steg(s)->type = find_uri_type(buf); + downcast_steg(s)->have_transmitted = 1; + return 0; +} + + + + + + + + + + + + + + + + + + + + +int +x_http2_transmit(steg_t *s, struct evbuffer *source, conn_t *conn) +{ + // struct evbuffer *dest = conn_get_outbound(conn); + + // fprintf(stderr, "in x_http2_ transmit %d\n", downcast_steg(s)->type); + + + + if (s->is_clientside) { + /* On the client side, we have to embed the data in a GET query somehow; + the only plausible places to put it are the URL and cookies. This + presently uses the URL. And it can't be binary. */ + + return x_http2_client_transmit(s, source, conn); //@@ + } + else { + int rval = -1; + switch(downcast_steg(s)->type) { + + case HTTP_CONTENT_SWF: + rval = x_http2_server_SWF_transmit(s, source, conn); + break; + case HTTP_CONTENT_JAVASCRIPT: + rval = x_http2_server_JS_transmit(s, source, conn); + break; + + case HTTP_CONTENT_PDF: + rval = x_http2_server_PDF_transmit(s, source, conn); + break; + } + + if (rval == 0) downcast_steg(s)->have_transmitted = 1; + return rval; + } +} + + + + + + +int +x_http2_server_receive(steg_t *s, conn_t *conn, struct evbuffer *dest, struct evbuffer* source) { + + int cnt = 0; + unsigned char* data; + int type; + + do { + struct evbuffer_ptr s2 = evbuffer_search(source, "\r\n\r\n", sizeof ("\r\n\r\n") -1 , NULL); + unsigned char* limit; + unsigned char *p; + int unwrapped_cookie_len; + struct evbuffer *scratch; + unsigned char c, h, secondhalf; + unsigned char buf[evbuffer_get_length(source)]; + + + if (s2.pos == -1) { + log_debug("Did not find end of request %d", (int) evbuffer_get_length(source)); + // evbuffer_dump(source, stderr); + return RECV_INCOMPLETE; + } + + log_debug("SERVER received request header of length %d", (int)s2.pos); + + data = evbuffer_pullup(source, s2.pos); + if (data == NULL) { + log_debug("SERVER evbuffer_pullup fails"); + return RECV_BAD; + } + + limit = data + s2.pos; + + type = find_uri_type((char *)data); + log_warn ("*** Got type %d", type); + + /* if (type != 3) { + fprintf(stderr, "type != 3, %d, data = %s \n", find_uri_type2((char *) data), data); + exit(-1); + }*/ + + data = (unsigned char*) strstr((char*) data, "Cookie:"); + + if (data == NULL || memcmp(data, "Cookie:", sizeof "Cookie:"-1)) { + log_debug("Unexpected HTTP verb: %.*s", 5, data); + return RECV_BAD; + } + + p = data + sizeof "Cookie: "-1; + unwrapped_cookie_len = unwrap_cookie(p, buf, (int) (limit - p)); + + log_debug("SERVER: received cookie of length = %d %d\n", unwrapped_cookie_len, (int) (limit-p)); + // buf_dump(buf, unwrapped_cookie_len, stderr); + // fprintf(stderr, "==========================\n"); + // buf_dump(p, (int) (limit-p), stderr); + + + // log_debug("hello SERVER received %d cnt = %d\n", (int) (limit - p), cnt); + // buf_dump(p, (int) (limit-p), stderr); + + /* We need a scratch buffer here because the contract is that if + we hit a decode error we *don't* write anything to 'dest'. */ + scratch = evbuffer_new(); + + if (!scratch) return RECV_BAD; + + + if (evbuffer_expand(scratch, unwrapped_cookie_len/2)) { + log_debug("Evbuffer expand failed \n"); + evbuffer_free(scratch); + return RECV_BAD; + } + p = buf; + + + secondhalf = 0; + while ((int) (p - buf) < unwrapped_cookie_len) { + if (!secondhalf) c = 0; + if ('0' <= *p && *p <= '9') h = *p - '0'; + else if ('a' <= *p && *p <= 'f') h = *p - 'a' + 10; + else if ('A' <= *p && *p <= 'F') h = *p - 'A' + 10; + else if (*p == '=' && !secondhalf) { + p++; + continue; + } else { + evbuffer_free(scratch); + log_debug("Decode error: unexpected URI characterasdfaf %d", *p); + return RECV_BAD; + } + + c = (c << 4) + h; + if (secondhalf) { + evbuffer_add(scratch, &c, 1); + // log_debug("adding to scratch"); + cnt++; + } + secondhalf = !secondhalf; + p++; + } + + + + if (evbuffer_add_buffer(dest, scratch)) { + evbuffer_free(scratch); + log_debug("Failed to transfer buffer"); + return RECV_BAD; + } + evbuffer_drain(source, s2.pos + sizeof("\r\n\r\n") - 1); + evbuffer_free(scratch); + } while (evbuffer_get_length(source)); + + + downcast_steg(s)->have_received = 1; + downcast_steg(s)->type = type; + // fprintf(stderr, "SERVER RECEIVED payload %d %d\n", cnt, type); + + conn_transmit_soon(conn, 100); + return RECV_GOOD; +} + + + + + + + + +static int +x_http2_receive(steg_t *s, conn_t *conn, struct evbuffer *dest) +{ + struct evbuffer *source = conn_get_inbound(conn); + // unsigned int type; + int rval = RECV_BAD; + + + if (s->is_clientside) { + + // fprintf(stderr, "client type = %d\n", downcast_steg(s)->type); + + switch(downcast_steg(s)->type) { + + case HTTP_CONTENT_SWF: + rval = x_http2_handle_client_SWF_receive(s, conn, dest, source); + break; + case HTTP_CONTENT_JAVASCRIPT: + rval = x_http2_handle_client_JS_receive(s, conn, dest, source); + break; + + case HTTP_CONTENT_PDF: + rval = x_http2_handle_client_PDF_receive(s, conn, dest, source); + break; + } + + if (rval == RECV_GOOD) downcast_steg(s)->have_received = 1; + return rval; + + } else { + return x_http2_server_receive(s, conn, dest, source); + } + + +} diff --git a/src/steg/zpack.c b/src/steg/zpack.c new file mode 100644 index 0000000..57cd2b2 --- /dev/null +++ b/src/steg/zpack.c @@ -0,0 +1,408 @@ +#include <stdio.h> +#include <string.h> +#include <assert.h> +#include <time.h> +#include <stdlib.h> +#include "zlib.h" +#include "zpack.h" + + +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(__CYGWIN__) +# include <fcntl.h> +# include <io.h> +# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) +#else +# define SET_BINARY_MODE(file) +#endif + +#define CHUNK 16384 + +/* Compress from file source to file dest until EOF on source. + def() returns Z_OK on success, Z_MEM_ERROR if memory could not be + allocated for processing, Z_STREAM_ERROR if an invalid compression + level is supplied, Z_VERSION_ERROR if the version of zlib.h and the + version of the library linked do not match, or Z_ERRNO if there is + an error reading or writing the files. */ + + +int def(char *source, int slen, char *dest, int dlen, int level) +{ + int ret, flush; + unsigned have; + z_stream strm; + unsigned char in[CHUNK]; + unsigned char out[CHUNK]; + int dlen_orig = dlen; + + /* allocate deflate state */ + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + ret = deflateInit(&strm, level); + if (ret != Z_OK) + return ret; + + /* compress until end of file */ + do { + + if (slen > CHUNK) + strm.avail_in = CHUNK; + else + strm.avail_in = slen; + + memcpy (in, source, strm.avail_in); + slen = slen - strm.avail_in; + source - source + strm.avail_in; + + flush = (slen == 0) ? Z_FINISH : Z_NO_FLUSH; + strm.next_in = in; + + /* run deflate() on input until output buffer not full, finish + compression if all of source has been read in */ + do { + strm.avail_out = CHUNK; + strm.next_out = out; + ret = deflate(&strm, flush); /* no bad return value */ + assert(ret != Z_STREAM_ERROR); /* state not clobbered */ + have = CHUNK - strm.avail_out; + + if ((unsigned int) dlen < have) { + fprintf(stderr, "dest buf too small!\n"); + return Z_ERRNO; + } + + memcpy(dest, out, have); + dest += have; + dlen = dlen - have; + } while (strm.avail_out == 0); + assert(strm.avail_in == 0); /* all input will be used */ + + /* done when last data in file processed */ + } while (flush != Z_FINISH); + assert(ret == Z_STREAM_END); /* stream will be complete */ + + /* clean up and return */ + (void)deflateEnd(&strm); + + printf("hello here...\n"); + return (dlen_orig - dlen); + // return Z_OK; +} + +/* Decompress from file source to file dest until stream ends or EOF. + inf() returns Z_OK on success, Z_MEM_ERROR if memory could not be + allocated for processing, Z_DATA_ERROR if the deflate data is + invalid or incomplete, Z_VERSION_ERROR if the version of zlib.h and + the version of the library linked do not match, or Z_ERRNO if there + is an error reading or writing the files. */ + + + + +int inf(char *source, int slen, char *dest, int dlen) +{ + int ret; + unsigned have; + z_stream strm; + unsigned char in[CHUNK]; + unsigned char out[CHUNK]; + int dlen_orig = dlen; + + + /* allocate inflate state */ + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = 0; + strm.next_in = Z_NULL; + ret = inflateInit(&strm); + if (ret != Z_OK) + return ret; + + /* decompress until deflate stream ends or end of file */ + do { + + if (slen == 0) + break; + + if (slen > CHUNK) + strm.avail_in = CHUNK; + else + strm.avail_in = slen; + + memcpy(in, source, strm.avail_in); + slen = slen - strm.avail_in; + source = source + strm.avail_in; + + + + strm.next_in = in; + + /* run inflate() on input until output buffer not full */ + do { + strm.avail_out = CHUNK; + strm.next_out = out; + ret = inflate(&strm, Z_NO_FLUSH); + assert(ret != Z_STREAM_ERROR); /* state not clobbered */ + switch (ret) { + case Z_NEED_DICT: + ret = Z_DATA_ERROR; /* and fall through */ + case Z_DATA_ERROR: + case Z_MEM_ERROR: + (void)inflateEnd(&strm); + return ret; + } + have = CHUNK - strm.avail_out; + + + if ((unsigned int) dlen < have) { + fprintf(stderr, "dest buf too small!\n"); + return Z_ERRNO; + } + + memcpy(dest, out, have); + dest += have; + dlen = dlen - have; + + } while (strm.avail_out == 0); + + /* done when inflate() says it's done */ + } while (ret != Z_STREAM_END); + + /* clean up and return */ + (void)inflateEnd(&strm); + + if (ret == Z_STREAM_END) + return dlen_orig - dlen; + return Z_DATA_ERROR; +} + +/* report a zlib or i/o error */ +void zerr(int ret) + +{ + fputs("zpipe: ", stderr); + switch (ret) { + case Z_ERRNO: + if (ferror(stdin)) + fputs("error reading stdin\n", stderr); + if (ferror(stdout)) + fputs("error writing stdout\n", stderr); + break; + case Z_STREAM_ERROR: + fputs("invalid compression level\n", stderr); + break; + case Z_DATA_ERROR: + fputs("invalid or incomplete deflate data\n", stderr); + break; + case Z_MEM_ERROR: + fputs("out of memory\n", stderr); + break; + case Z_VERSION_ERROR: + fputs("zlib version mismatch!\n", stderr); + } +} + + + + + + + + +/* assumes that we know there is exactly 10 bytes of gzip header */ + +int gzInflate(char *source, int slen, char *dest, int dlen) +{ + int ret; + unsigned have; + z_stream strm; + unsigned char in[CHUNK]; + unsigned char out[CHUNK]; + int dlen_orig = dlen; + + + /* allocate inflate state */ + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = 0; + strm.next_in = Z_NULL; + + + ret = inflateInit2(&strm, -MAX_WBITS); + if (ret != Z_OK) + return ret; + + source = source + 10; + slen -= 10; + + /* decompress until deflate stream ends or end of file */ + do { + + if (slen == 0) + break; + + if (slen > CHUNK) + strm.avail_in = CHUNK; + else + strm.avail_in = slen; + + memcpy(in, source, strm.avail_in); + slen = slen - strm.avail_in; + source = source + strm.avail_in; + + + + strm.next_in = in; + + /* run inflate() on input until output buffer not full */ + do { + strm.avail_out = CHUNK; + strm.next_out = out; + ret = inflate(&strm, Z_NO_FLUSH); + assert(ret != Z_STREAM_ERROR); /* state not clobbered */ + switch (ret) { + case Z_NEED_DICT: + ret = Z_DATA_ERROR; /* and fall through */ + case Z_DATA_ERROR: + case Z_MEM_ERROR: + (void)inflateEnd(&strm); + return ret; + } + have = CHUNK - strm.avail_out; + + if ((unsigned int) dlen < have) { + fprintf(stderr, "dest buf too small!\n"); + return Z_ERRNO; + } + + memcpy(dest, out, have); + dest += have; + dlen = dlen - have; + + } while (strm.avail_out == 0); + + /* done when inflate() says it's done */ + } while (ret != Z_STREAM_END); + + /* clean up and return */ + (void)inflateEnd(&strm); + + if (ret == Z_STREAM_END) + return dlen_orig - dlen; + return Z_DATA_ERROR; +} + + + + + + + +int gzDeflate(char* start, off_t insz, char *buf, off_t outsz, time_t mtime) { + unsigned char *c; + unsigned long crc; + z_stream z; + + z.zalloc = Z_NULL; + z.zfree = Z_NULL; + z.opaque = Z_NULL; + + if (Z_OK != deflateInit2(&z, + Z_DEFAULT_COMPRESSION, + Z_DEFLATED, + -MAX_WBITS, /* supress zlib-header */ + 8, + Z_DEFAULT_STRATEGY)) { + return -1; + } + + z.next_in = (unsigned char *)start; + z.avail_in = insz; + z.total_in = 0; + + + /* write gzip header */ + + c = (unsigned char *) buf; + c[0] = 0x1f; + c[1] = 0x8b; + c[2] = Z_DEFLATED; + c[3] = 0; /* options */ + c[4] = (mtime >> 0) & 0xff; + c[5] = (mtime >> 8) & 0xff; + c[6] = (mtime >> 16) & 0xff; + c[7] = (mtime >> 24) & 0xff; + c[8] = 0x00; /* extra flags */ + c[9] = 0x03; /* UNIX */ + + z.next_out = c + 10; + z.avail_out = outsz - 10 - 8; + z.total_out = 0; + + if (Z_STREAM_END != deflate(&z, Z_FINISH)) { + deflateEnd(&z); + return -1; + } + + + crc = generate_crc32c(start, insz); + + c = (unsigned char *)buf + 10 + z.total_out; + + c[0] = (crc >> 0) & 0xff; + c[1] = (crc >> 8) & 0xff; + c[2] = (crc >> 16) & 0xff; + c[3] = (crc >> 24) & 0xff; + c[4] = (z.total_in >> 0) & 0xff; + c[5] = (z.total_in >> 8) & 0xff; + c[6] = (z.total_in >> 16) & 0xff; + c[7] = (z.total_in >> 24) & 0xff; + + + + if (Z_OK != deflateEnd(&z)) { + return -1; + } + + return 10 + z.total_out + 8; + +} + + + + + +/* compress or decompress from stdin to stdout */ +/* int main(int argc, char **argv) */ +/* { */ +/* int ret; */ +/* char buf1[32] = "abcasdfadfadfadf23fasdfa23sdfsdf"; */ +/* char buf2[100]; */ +/* char buf3[100]; */ +/* int i; */ + +/* bzero(buf2, sizeof(buf2)); */ +/* bzero(buf3, sizeof(buf3)); */ + + +/* // ret = def(buf1, 3, buf2, 100, Z_DEFAULT_COMPRESSION); */ +/* ret = gzDeflate(buf1, sizeof(buf1), buf2, sizeof(buf2), time(NULL)); */ +/* if (ret <= 0) */ +/* zerr(ret); */ + +/* /\* for (i=0; i < ret; i++) */ +/* putc(buf2[i], stdout); */ +/* *\/ */ + + +/* // printf("len = %d\n", ret); */ + +/* ret = gzInflate(buf2, ret, buf3, 100); */ +/* if (ret <= 0) */ +/* zerr(ret); */ +/* printf("hello %s\n", buf3); */ + + +/* } */ diff --git a/src/steg/zpack.h b/src/steg/zpack.h new file mode 100644 index 0000000..65cd28d --- /dev/null +++ b/src/steg/zpack.h @@ -0,0 +1,14 @@ +#include <stdio.h> +#include <string.h> +#include <assert.h> +#include <time.h> +#include <stdlib.h> +#include "zlib.h" + + +int def(char *source, int slen, char *dest, int dlen, int level); +int inf(char *source, int slen, char *dest, int dlen); +void zerr(int ret); +int gzInflate(char *source, int slen, char *dest, int dlen); +int gzDeflate(char* start, off_t insz, char *buf, off_t outsz, time_t mtime); +unsigned int generate_crc32c(char *buffer, size_t length); diff --git a/start-client.csh b/start-client.csh new file mode 100644 index 0000000..65f4465 --- /dev/null +++ b/start-client.csh @@ -0,0 +1,8 @@ +#!/bin/csh +# ./obfsproxy --log-min-severity=debug x_dsteg socks 127.0.0.1:1080 x_http + +setenv EVENT_NOKQUEUE yes +#./obfsproxy --log-min-severity=debug chop socks 127.0.0.1:1080 127.0.0.1:8080 x_http2 127.0.0.1:8081 x_http2 +# ./obfsproxy --log-min-severity=warn chop socks 127.0.0.1:1080 127.0.0.1:8080 x_http2 127.0.0.1:8081 x_http2 +./obfsproxy --log-min-severity=error chop socks 127.0.0.1:1080 127.0.0.1:8080 x_http2 127.0.0.1:8081 x_http2 + diff --git a/start-server.csh b/start-server.csh new file mode 100644 index 0000000..50b781a --- /dev/null +++ b/start-server.csh @@ -0,0 +1,6 @@ +#!/bin/csh +setenv EVENT_NOKQUEUE yes +#./obfsproxy --log-min-severity=debug chop server 87.73.82.145:8080 127.0.0.1:8080 127.0.0.1:8081 +# ./obfsproxy --log-min-severity=warn chop server 87.73.82.145:8080 127.0.0.1:8080 127.0.0.1:8081 +./obfsproxy --log-min-severity=error chop server 87.73.82.145:8080 127.0.0.1:8080 127.0.0.1:8081 + diff --git a/torrc b/torrc new file mode 100644 index 0000000..ff27e61 --- /dev/null +++ b/torrc @@ -0,0 +1,12 @@ +SocksPort 9060 # what port to open for local application connections +SocksListenAddress 127.0.0.1 # accept connections only from localhost + +SafeLogging 0 +Log info file ./info.log +Log debug file ./debug.log + +Socks4Proxy 127.0.0.1:1080 + +# Bridge 87.73.82.145:8080 +Bridge 127.0.0.1:8080 +UseBridges 1

1 0