- tor-commits - lists.torproject.org

[stegotorus/master] Rough draft benchmarking tools.
by zwol＠torproject.org 20 Jul '12

20 Jul '12

commit 704c053fbae93353976e3f7abf585b6283edb3a9 Author: Zack Weinberg <zackw(a)panix.com> Date: Sun Feb 12 20:36:56 2012 -0800 Rough draft benchmarking tools. --- scripts/benchmark-plot.R | 7 + scripts/benchmark.py | 410 +++++++++++++++++++++++++++++++++++ scripts/bm-fixedrate-cgi.c | 158 ++++++++++++++ scripts/bm-genfiles.py | 162 ++++++++++++++ scripts/bm-mcurl.c | 196 +++++++++++++++++ scripts/tool_urlglob.c | 516 ++++++++++++++++++++++++++++++++++++++++++++ scripts/tool_urlglob.h | 69 ++++++ 7 files changed, 1518 insertions(+), 0 deletions(-) diff --git a/scripts/benchmark-plot.R b/scripts/benchmark-plot.R new file mode 100644 index 0000000..a58ae0a --- /dev/null +++ b/scripts/benchmark-plot.R @@ -0,0 +1,7 @@ +#! /usr/bin/Rscript + +suppressPackageStartupMessages({ + library(ggplot2) +}) + +lf.direct <- read.csv("bench-lf-direct.tab", header=TRUE) diff --git a/scripts/benchmark.py b/scripts/benchmark.py new file mode 100755 index 0000000..c6487b6 --- /dev/null +++ b/scripts/benchmark.py @@ -0,0 +1,410 @@ +#! /usr/bin/python + +# Stegotorus benchmarking script. +# Several different computers are involved: +# +# - the "client" is the machine you run this script on; the workload +# generator will run there, as will the StegoTorus and Tor clients. +# +# - the "proxy" is a machine that you can ssh to with no password. +# It will run the StegoTorus and Tor bridge servers. +# +# - the "target" is the HTTP server that will be contacted in various ways. +# +# bm-genfiles.py must have been run on this server to create file +# trees named 'fixed' and 'pareto' which appear as direct children +# of the root URL. bm-fixedrate-cgi.c must have been compiled for +# that server and appear as /bm-fixedrate.cgi. +# +# Software you need on the client machine: +# +# bwm-ng: http://www.gropp.org/?id=projects&sub=bwm-ng +# curl: http://curl.haxx.se/ +# httperf: http://www.hpl.hp.com/research/linux/httperf/ +# tsocks: http://tsocks.sourceforge.net/about.php +# tor: https://torproject.org/ +# stegotorus: you already have it :) +# +# Software you need on the proxy machine: +# +# nylon: http://monkey.org/~marius/pages/?page=nylon +# tor, stegotorus +# +# You configure this script by setting variables below. + +# Client host + +CLIENT_IP = "99.113.33.155" +CLIENT_IFACE = "eth0" + +# Proxy host + +PROXY = "sandbox03.sv.cmu.edu" +PROXY_IP = "209.129.244.30" # some things won't do DNS for this +PROXY_PORT = "1080" +PROXY_SSH_CMD = ("ssh", PROXY) + +# Target + +TARGET = "storustest.nfshost.com" + +# Fudge factors. For some reason, bm-fixedrate generates data a +# linear factor slower than it was meant to; this is the quick fix. + +FUDGE_FIXEDRATE = 2.5 + +# Programs we need to run. Change these if any binary is not in the +# default path or hasn't got the default name. +# C_ - for the client. P_ - for the proxy. +# You can NOT specify arguments here - if you need to do any +# setup, write a wrapper script. + +C_bwm = "bwm-ng" +C_curl = "curl" +C_httperf = "httperf" +C_storus = "stegotorus-wrapper" +C_tor = "/usr/sbin/tor" +C_tsocks = "/usr/lib/libtsocks.so" + +P_nylon = "nylon" +P_storus = "stegotorus-wrapper" +P_tor = "tor" +P_python = "/usr/local/bin/python" # this must be an absolute path, + # it goes on a shebang line + +# ACTUAL PROGRAM STARTS HERE + +from types import MethodType +import os +import os.path +import pickle +import subprocess +import sys +import time + +def monitor(report, label, period): + """Monitor network utilization (bytes/sec up and down) for a + period of PERIOD seconds, writing the report to REPORT, labeling + each line with LABEL.""" + + bwm = subprocess.Popen((C_bwm, "-o", "csv", "-c", str(period), "-t", "1000", + "-u", "bytes", "-T", "rate", "-I", CLIENT_IFACE), + stdout=subprocess.PIPE, + universal_newlines=True) + try: + n = 1 + for line in bwm.stdout: + (stamp, iface, upbytes, dnbytes, rest) = line.split(';', 4) + if iface == 'total': continue + + # convert to most compact possible form + upbytes = str(float(upbytes)) + dnbytes = str(float(dnbytes)) + + report.write("%s,%d,%s,%s\n" % (label,n,upbytes,dnbytes)) + n += 1 + except: + bwm.terminate() + raise + finally: + bwm.wait() + +class ProxyProcess(object): + """A process running on the proxy host. It has a command line and + an optional config file. It is not expected to produce any output + (if it does, it will get dumped to this script's stdout/stderr) or + require any input (input is redirected from /dev/null). It is + expected to run until it is killed.""" + + @staticmethod + def prepare_remote(): + remote_driver=r"""#! %s +import pickle +import signal +import subprocess +import sys +import traceback + +wrote_rpid = False + +# Remote driver for proxy processes. +try: + data = pickle.load(sys.stdin) + sys.stdin.close() + if data['cfgname']: + f = open(data['cfgname'], "w") + f.write(data['cfgdata']) + f.close() + proc = subprocess.Popen(data['args'], stdin=open("/dev/null", "r"), + stdout=2) # redirect child stdout to our stderr + sys.stdout.write(str(proc.pid) + "\n") + wrote_rpid = True + sys.stdout.close() + proc.wait() + + # the process being killed by SIGTERM is normal + if proc.returncode != 0 and proc.returncode != -signal.SIGTERM: + raise subprocess.CalledProcessError(proc.returncode, data['args'][0]) +except: + traceback.print_exc() + if not wrote_rpid: sys.stdout.write("X\n") + sys.exit(1) + +sys.exit(0) +""" % P_python + remote_setup=r"""newdriver=`mktemp ./driver.py.XXXXXX` || exit 1 +cat > "$newdriver" +if cmp -s "$newdriver" driver.py +then rm -f "$newdriver" +else set -e; mv -f "$newdriver" driver.py; chmod +x driver.py +fi +""" + prep_worker = subprocess.Popen(PROXY_SSH_CMD + (remote_setup,), + stdin=subprocess.PIPE, + stdout=2) + prep_worker.communicate(remote_driver) + if prep_worker.returncode != 0: + raise subprocess.CalledProcessError(prep_worker.returncode, + 'remote_setup script') + + def __init__(self, args, cfgname=None, cfgdata=None): + if ((cfgname is None or cfgdata is None) and + (cfgname is not None or cfgdata is not None)): + raise TypeError("either both or neither of cfgname and cfgdata" + " must be specified") + + self._rpid = "X" + + ProxyProcess.prepare_remote() + self._proc = subprocess.Popen(PROXY_SSH_CMD + ("./driver.py",), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + close_fds=True) + pickle.dump({ 'args' : args, + 'cfgname' : cfgname, + 'cfgdata' : cfgdata }, + self._proc.stdin, 2) + self._proc.stdin.close() + self._rpid = self._proc.stdout.readline().strip() + if self._rpid == "X" or self._rpid == "": + self._rpid = "X" + self._proc.wait() + raise RuntimeError("failed to execute '%s' on proxy host" + % " ".join(args)) + + def terminate(self): + if self._rpid == "X": return + subprocess.check_call(PROXY_SSH_CMD + ("kill", self._rpid)) + + def kill(self): + if self._rpid == "X": return + subprocess.check_call(PROXY_SSH_CMD + ("kill", "-9", self._rpid)) + + # forward everything else to _proc; logic copied verbatim from + # http://code.activestate.com/recipes/519639- + # true-lieberman-style-delegation-in-python/ + def __getattr__(self, aname): + target = self._proc + f = getattr(target, aname) + if isinstance(f, MethodType): + return MethodType(f.im_func, self, target.__class__) + else: + return f + +# Individual proxy-side test runners. +def p_nylon(): + return ProxyProcess((P_nylon, "-f", "-c", "nylon.conf"), + "nylon.conf", + """\ +[General] +No-Simultaneous-Conn=10 +Log=0 +Verbose=0 +PIDfile=nylon.pid + +[Server] +Port=%s +Allow-IP=%s/32 +""" % (PROXY_PORT, CLIENT_IP)) + +def p_tor_direct(): + return ProxyProcess((P_tor, "--quiet", "-f", "tor-direct.conf"), + "tor-direct.conf", + """\ +ORPort %s +SocksPort 0 +BridgeRelay 1 +PublishServerDescriptor 0 +ExitPolicy reject *:* +DataDirectory . +Log err stderr +# unfortunately there doesn't seem to be any way to tell Tor to accept +# OR connections from specific IP addresses only. +""" % PROXY_PORT) + +class ClientProcess(subprocess.Popen): + """A process running on the local machine. This is probably doing + the meat of the work of some benchmark. Basically a shim around + subprocess.Popen to fix constructor arguments.""" + + def __init__(self, argv, envp=None): + if envp is not None: + env = os.environ.copy() + env.update(envp) + subprocess.Popen.__init__(self, argv, + stdin=open("/dev/null", "r"), + stdout=open("/dev/null", "w"), + stderr=subprocess.STDOUT, env=env) + else: + subprocess.Popen.__init__(self, argv, + stdin=open("/dev/null", "r"), + stdout=2) + +def c_tor_direct(): + fp = open("tor-direct-client.conf", "w") + fp.write("""\ +ORPort 0 +SocksPort %s +DataDirectory . +Log err stderr +Bridge %s:%s +UseBridges 1 +SafeSocks 0 +""" % (PROXY_PORT, PROXY_IP, PROXY_PORT)) + fp.close() + return ClientProcess((C_tor, "--quiet", "-f", "tor-direct-client.conf")) + +def c_curl(url, proxyhost): + return ClientProcess((C_curl, "-s", "--socks5-hostname", + proxyhost + ":" + PROXY_PORT, + url, "-o", "/dev/null")) + +def c_httperf(prefix, rate, proxyhost): + fp = open("tsocks.conf", "w") + fp.write("""\ +server = %s +local = %s/255.255.255.255 +server_port = %s +server_type = 5 +""" % (proxyhost, proxyhost, PROXY_PORT)) + fp.close() + return ClientProcess((C_httperf, "--hog", + "--server=" + TARGET, + "--uri=" + prefix, + "--period=" + str(rate), + "--num-calls=5", "--num-conns=2000", + "--wset=10000,1"), + { 'LD_PRELOAD' : C_tsocks, + 'TSOCKS_CONF_FILE' : + os.path.join(os.getcwd(), "tsocks.conf") }) + +# Benchmarks. + +def bench_fixedrate_direct(report): + client = None + proxy = None + try: + proxy = p_nylon() + + for cap in range(10, 810, 10): + sys.stderr.write("fixedrate,direct,%d\n" % (cap * 1000)) + try: + client = c_curl('http://' + TARGET + '/bm-fixedrate.cgi/' + + str(int(cap * 1000 * FUDGE_FIXEDRATE)), + PROXY) + monitor(report, "fixedrate,direct,%d" % (cap * 1000), 60) + finally: + if client is not None: + client.terminate() + client.wait() + client = None + finally: + if proxy is not None: + proxy.terminate() + proxy.wait() + +def bench_fixedrate_tor(report): + client = None + proxy = None + proxyl = None + try: + proxy = p_tor_direct() + proxyl = c_tor_direct() + time.sleep(5) # tor startup is slow + + for cap in range(10,810,10): + sys.stderr.write("fixedrate,tor,%d\n" % (cap * 1000)) + try: + client = c_curl('http://' + TARGET + '/bm-fixedrate.cgi/' + + str(int(cap * 1000 * FUDGE_FIXEDRATE)), + '127.0.0.1') + monitor(report, "fixedrate,tor,%d" % (cap * 1000), 60) + finally: + if client is not None: + client.terminate() + client.wait() + client = None + finally: + if proxy is not None: + proxy.terminate() + proxy.wait() + if proxyl is not None: + proxyl.terminate() + proxyl.wait() + +def bench_files_direct(report, prefix): + client = None + proxy = None + try: + proxy = p_nylon() + + for cps in range(1,81): + sys.stderr.write("files.%s,direct,%d\n" % (prefix, cps)) + try: + client = c_httperf(prefix, 1./cps, PROXY_IP) + monitor(report, "files.%s,direct,%d" % (prefix, cps), 60) + finally: + if client is not None: + client.terminate() + client.wait() + client = None + finally: + if proxy is not None: + proxy.terminate() + proxy.wait() + +def bench_files_tor(report, prefix): + client = None + proxy = None + proxyl = None + try: + proxy = p_tor_direct() + proxyl = c_tor_direct() + time.sleep(5) # tor startup is slow + + for cps in range(1,81): + sys.stderr.write("files.%s,tor,%d\n" % (prefix, cps)) + try: + client = c_httperf(prefix, 1./cps, '127.0.0.1') + monitor(report, "files.%s,tor,%d" % (prefix, cps), 60) + finally: + if client is not None: + client.terminate() + client.wait() + client = None + finally: + if proxy is not None: + proxy.terminate() + proxy.wait() + if proxyl is not None: + proxyl.terminate() + proxyl.wait() + +if __name__ == '__main__': + sys.stdout.write("benchmark,relay,cap,obs,up,down\n") + bench_fixedrate_direct(sys.stdout) + bench_fixedrate_tor(sys.stdout) + bench_files_direct(sys.stdout, "fixed") + bench_files_tor(sys.stdout, "fixed") + bench_files_direct(sys.stdout, "pareto") + bench_files_tor(sys.stdout, "pareto") diff --git a/scripts/bm-fixedrate-cgi.c b/scripts/bm-fixedrate-cgi.c new file mode 100644 index 0000000..2b48f98 --- /dev/null +++ b/scripts/bm-fixedrate-cgi.c @@ -0,0 +1,158 @@ +#define _XOPEN_SOURCE 600 +#define _POSIX_C_SOURCE 200112 + +#include <stdbool.h> +#include <stddef.h> + +#include <errno.h> +#include <math.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <unistd.h> + +/* 1400 bytes is a safe figure for per-packet transmissible payload. */ +#define BLOCKSZ 1400 + + +#if __GNUC__ >= 3 +#define NORETURN void __attribute__((noreturn)) +#else +#define NORETURN void +#endif + +extern char **environ; + +static NORETURN +error_400(const char *msg) +{ + char **p; + printf("Status: 400 Bad Request\nContent-Type: text/plain\n\n" + "400 Bad Request (%s)\nCGI environment dump follows:\n\n", msg); + for (p = environ; *p; p++) + puts(*p); + exit(0); +} + +static NORETURN +error_500(const char *syscall) +{ + printf("Status: 500 Internal Server Error\nContent-Type:text/plain\n\n" + "500 Internal Server Error: %s: %s\n", + syscall, strerror(errno)); + exit(0); +} + +static void +generate(unsigned long rate, bool dryrun) +{ + double interval; + timer_t timerid; + struct sigevent sev; + struct itimerspec its; + sigset_t mask; + int sig; + char *data; + size_t bufsz = BLOCKSZ; + + /* You send data at R bytes per second in 1400-byte blocks by + calling write() every 1/(R/1400) second. However, despite our + use of the high-resolution interval timers, we cannot count on + being scheduled more often than every 1/CLOCKS_PER_SEC seconds, + so if we need to send data faster than that, bump up the block + size instead. */ + interval = 1./(rate/(double)BLOCKSZ); + + if (interval < 1./CLOCKS_PER_SEC) { + interval = 1./CLOCKS_PER_SEC; + bufsz = rate / CLOCKS_PER_SEC; + } + + its.it_value.tv_sec = lrint(floor(interval)); + its.it_value.tv_nsec = lrint((interval - its.it_value.tv_sec) * 1e9); + its.it_interval.tv_sec = its.it_value.tv_sec; + its.it_interval.tv_nsec = its.it_value.tv_nsec; + + if (dryrun) { + printf("Content-Type: text/plain\n\n" + "Goal %lu bytes per second:\n" + "would send %lu bytes every %f seconds\n" + " \" \" \" \" \" %lu sec + %lu nsec\n", + rate, bufsz, interval, + (unsigned long)its.it_value.tv_sec, + (unsigned long)its.it_value.tv_nsec); + return; + } + + data = malloc(bufsz); + if (!data) + error_500("malloc"); + memset(data, 0, bufsz); + + fflush(stdout); + setvbuf(stdout, 0, _IONBF, 0); + fputs("Content-Type: application/octet-stream\n" + "Cache-Control: no-store,no-cache\n\n", stdout); + + sigemptyset(&mask); + sigaddset(&mask, SIGRTMIN); + if (sigprocmask(SIG_SETMASK, &mask, 0)) + error_500("sigprocmask"); + + memset(&sev, 0, sizeof sev); + sev.sigev_notify = SIGEV_SIGNAL; + sev.sigev_signo = SIGRTMIN; + sev.sigev_value.sival_ptr = &timerid; + if (timer_create(CLOCK_MONOTONIC, &sev, &timerid)) + error_500("timer_create"); + + if (timer_settime(timerid, 0, &its, 0)) + error_500("timer_settime"); + + do { + size_t r, n = bufsz; + char *p = data; + do { + r = fwrite(p, 1, n, stdout); + if (r == 0) + exit(1); + n -= r; + p += r; + } while (n > 0); + } while (sigwait(&mask, &sig) == 0); +} + +int +main(void) +{ + unsigned long rate; + char *endp; + bool dryrun; + char *request_method = getenv("REQUEST_METHOD"); + char *query_string = getenv("QUERY_STRING"); + char *path_info = getenv("PATH_INFO"); + + if (!request_method || strcmp(request_method, "GET")) + error_400("method not supported"); + if (query_string && strcmp(query_string, "")) + error_400("no query parameters accepted"); + + if (!path_info || path_info[0] != '/') + error_400("malformed or missing PATH_INFO"); + + rate = strtoul(path_info+1, &endp, 10); + if (endp == path_info+1) + error_400("missing rate (specify bytes per second)"); + + if (endp[0] == '\0') + dryrun = false; + else if (endp[0] == ';' && endp[1] == 'd' && endp[2] == '\0') + dryrun = true; + else + error_400("unrecognized extra arguments"); + + generate(rate, dryrun); + return 0; +} diff --git a/scripts/bm-genfiles.py b/scripts/bm-genfiles.py new file mode 100755 index 0000000..dcd1030 --- /dev/null +++ b/scripts/bm-genfiles.py @@ -0,0 +1,162 @@ +#! /usr/bin/python + +"""Generate files for network performance testing. + +The default behavior is to generate 10,000 files all of which are +exactly 3584 bytes long, because that is approximately how big +Flickr's 75x75px JPEG thumbnails are. You can request a different +size, or you can request that the file sizes instead follow a bounded +Pareto distribution with tunable alpha. + +The files have names compatible with httperf's --wset mode. Since +it insists on .html as a file suffix, the files are syntactically +valid HTML. Their contents are word salad. + +There is one mandatory command line argument: the path to the root +of the tree of files to generate. It is created if it doesn't +already exist. If it already exists, its contents will be erased! +(so don't use '.')""" + +from __future__ import division + +import argparse +import errno +import math +import os +import os.path +import random +import shutil +import sys +import textwrap + +def ensure_empty_dir(dpath): + todelete = [] + try: + todelete = os.listdir(dpath) + except OSError, e: + # Don't delete a _file_ that's in the way. + # Don't try to create parent directories that are missing. + if e.errno != errno.ENOENT: + raise + os.mkdir(dpath) + return + for f in todelete: + p = os.path.join(dpath, f) + try: + os.remove(p) + except OSError, e: + if e.errno != errno.EISDIR and e.errno != errno.EPERM: + raise + shutil.rmtree(p) + +def ensure_parent_directories(path): + try: + os.makedirs(os.path.dirname(path)) + except OSError, e: + if e.errno != errno.EEXIST: + raise + +def word_salad(f, words, seed, maxlen): + rng = random.Random(seed) + salad = [] + slen = 0 + while slen < maxlen - 1: + nl = rng.randint(1, min((maxlen - 1) - slen, len(words))) - 1 + w = rng.choice(words[nl]) + salad.append(w) + slen += len(w) + 1 + salad = textwrap.fill(" ".join(salad), 78) + while len(salad) < maxlen-1: + salad += '.' + salad += '\n' + f.write(salad) + +def load_words(): + words = [ [] for _ in xrange(15) ] + for w in open('/usr/share/dict/words'): + w = w.strip() + if w.endswith("'s"): continue + if len(w) > 15 or len(w) < 2: continue + words[len(w)-1].append(w) + # special case words[0] as dictfiles often have every single single letter + words[0].extend(('a','I')) + return words + +FILE_PREFIX = '<!doctype html>\n<title>{0}</title>\n<p>\n' +FILE_SUFFIX = '</p>\n' + +def create_one(parent, ctr, digits, words, filesize, seed, resume, progress): + label = format(ctr, '0'+str(digits)+'d') + fname = os.path.join(parent, *label) + '.html' + ensure_parent_directories(fname) + + if os.path.exists(fname): + if not resume: raise RuntimeError('{0} already exists'.format(fname)) + return + + prefix = FILE_PREFIX.format(label) + suffix = FILE_SUFFIX + limit = filesize - (len(prefix) + len(suffix)) + if limit <= 0: + raise TypeError("{0} bytes is too small to generate (minimum {1})" + .format(filesize, len(prefix)+len(suffix))) + + if progress: + sys.stderr.write(fname + '\n') + + f = open(fname, "w") + f.write(prefix) + word_salad(f, words, ctr+seed, limit) + f.write(suffix) + +def bounded_pareto(rng, alpha, L, H): + while True: + U = rng.random() + if U < 1: break + Ha = H**alpha + La = L**alpha + return int(round((-(U*Ha - U*La - Ha)/(Ha * La)) ** (-1/alpha))) + +if __name__ == '__main__': + + default_filesize = 3584 + default_filecount = 10000 # 0/0/0/0.html through 9/9/9/9.html + + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('directory', + help='directory to populate with files') + parser.add_argument('-c', '--count', type=int, default=default_filecount, + help='number of files to generate') + sg = parser.add_mutually_exclusive_group() + sg.add_argument('-s', '--size', type=int, default=default_filesize, + help='all files will be exactly SIZE bytes long') + sg.add_argument('-p', '--pareto', type=float, + metavar='ALPHA', + help='file sizes will follow a bounded Pareto distribution' + ' with parameter ALPHA') + parser.add_argument('-m', '--minsize', type=int, default=512, + help='minimum file size (only useful with -p)') + parser.add_argument('-M', '--maxsize', type=int, default=2*1024*1024, + help='maximum file size (only useful with -p)') + parser.add_argument('-S', '--seed', type=int, default=719, + help='seed for random number generator') + parser.add_argument('--resume', action='store_true', + help='resume an interrupted run where it left off') + parser.add_argument('--progress', action='store_true', + help='report progress') + + args = parser.parse_args() + digits = len(str(args.count - 1)) + rng = random.Random(args.seed) + + words = load_words() + if not args.resume: + ensure_empty_dir(args.directory) + + size = args.size + for i in xrange(args.count): + if args.pareto is not None: + size = bounded_pareto(rng, args.pareto, args.minsize, args.maxsize) + create_one(args.directory, i, digits, words, size, args.seed, + args.resume, args.progress) diff --git a/scripts/bm-mcurl.c b/scripts/bm-mcurl.c new file mode 100644 index 0000000..ac24f3a --- /dev/null +++ b/scripts/bm-mcurl.c @@ -0,0 +1,196 @@ +/* Use libcurl to retrieve many URLs, according to a wildcard pattern, + starting new connections at a constant rate until we hit a limit. + + Command line arguments -- all are required, but 'proxy' may be an + empty string if you want direct connections: + + bm-mcurl [-v] rate limit proxy url-pattern [url-pattern ...] + + There is no output; it is assumed that you are monitoring traffic + externally. Passing -v turns on CURLOPT_VERBOSE debugging spew. + */ + +#define _XOPEN_SOURCE 600 + +#include <stdbool.h> +#include <stddef.h> + +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <unistd.h> + +#include <curl/curl.h> +#include "tool_urlglob.h" + +#define NORETURN __attribute__((noreturn)) + +static bool verbose = false; + +static size_t +discard_data(char *ptr, size_t size, size_t nmemb, void *userdata) +{ + return size * nmemb; +} + +static size_t +read_abort(void *ptr, size_t size, size_t nmemb, void *userdata) +{ + /* we don't do anything that should require this to be called, + so if it does get called, something is wrong */ + return CURL_READFUNC_ABORT; +} + +static CURL * +setup_curl_easy_handle(char *proxy) +{ + CURL *h = curl_easy_init(); + if (!h) abort(); + +#define SET_OR_CRASH(h, opt, param) \ + do { if (curl_easy_setopt(h, opt, param)) abort(); } while (0) + + SET_OR_CRASH(h, CURLOPT_VERBOSE, (unsigned long)verbose); + SET_OR_CRASH(h, CURLOPT_NOPROGRESS, 1L); + SET_OR_CRASH(h, CURLOPT_FAILONERROR, 1L); + SET_OR_CRASH(h, CURLOPT_USERAGENT, "bm-mcurl/0.1"); + SET_OR_CRASH(h, CURLOPT_ACCEPT_ENCODING, ""); + SET_OR_CRASH(h, CURLOPT_AUTOREFERER, 1L); + SET_OR_CRASH(h, CURLOPT_FOLLOWLOCATION, 1L); + SET_OR_CRASH(h, CURLOPT_MAXREDIRS, 30L); + + SET_OR_CRASH(h, CURLOPT_WRITEFUNCTION, discard_data); + SET_OR_CRASH(h, CURLOPT_WRITEDATA, NULL); + SET_OR_CRASH(h, CURLOPT_READFUNCTION, read_abort); + SET_OR_CRASH(h, CURLOPT_READDATA, NULL); + + if (proxy && proxy[0]) { + SET_OR_CRASH(h, CURLOPT_PROXY, proxy); + SET_OR_CRASH(h, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5_HOSTNAME); + } +#undef SET_OR_CRASH +} + +static bool +process_events_once(CURLM *multi, unsigned long timeout_max) +{ + struct timeval tv; + int rc; /* select() return code */ + + fd_set fdread; + fd_set fdwrite; + fd_set fdexcept; + int maxfd = -1; + + unsigned long timeout = 1000000; /* one second - ultimate default */ + long curl_tout_ms = -1; + + /* get fd sets for all pending transfers */ + FD_ZERO(&fdread); + FD_ZERO(&fdwrite); + FD_ZERO(&fdexcept); + curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcept, &maxfd); + + /* timeout */ + if (timeout_max > 0 && timeout_max < timeout) + timeout = timeout_max; + + curl_multi_timeout(multi_handle, &curl_tout_ms); + + if (curl_tout_ms >= 0) { + unsigned long curl_tout_us = ((unsigned long)curl_tout_ms) * 1000; + if (timeout > curl_tout_us) + timeout = curl_tout_us; + } + + tv.tv_sec = timeout / 1000000; + if(tv.tv_sec >= 1) + tv.tv_sec = 1; + else + tv.tv_usec = timeout % 1000000; + + do { + rc = select(maxfd+1, &fdread, &fdwrite, &fdexcept, &tv); + } while (rc == -1 && errno == EINTR); + + if (rc > 0) { + int still_running; + curl_multi_perform(multi_handle, &still_running); + return !!still_running; + } else + abort(); +} + +/* Note: this function must not return until we are ready to start + another connection. */ +static void +queue_one(CURLM *multi, unsigned long rate, unsigned long limit, + char *proxy, char *url) +{ + +} + +static void +run(unsigned long rate, unsigned long limit, char *proxy, char **urls) +{ + CURLM *multi; + curl_global_init(); + multi = curl_multi_init(); + if (!multi) abort(); + + for (char **upat = urls; *upat; url++) { + URLGlob *uglob; + int *n; + if (glob_url(&uglob, *upat, &n, stderr)) + continue; + do { + char *url; + if (glob_next_url(&url, uglob)) abort(); + queue_one(multi, rate, limit, proxy, url); /* takes ownership */ + } while (--n); + glob_cleanup(uglob); + } + + /* spin the event loop until all outstanding transfers complete */ + while (process_events_once(multi, 0)); + + curl_multi_cleanup(multi); +} + +static NORETURN +usage(const char *av0, const char *complaint) +{ + fprintf(stderr, + "%s\nusage: %s [-v] rate limit proxy url [url...]\n", + complaint, av0); + exit(2); +} + +int +main(int argc, char **argv) +{ + unsigned long rate; + unsigned long limit; + char *endp; + + if (argv[1] && (!strcmp(argv[1], "-v") || !strcmp(argv[1], "--verbose"))) { + verbose = true; + argc--; + argv[1] = argv[0]; + argv++; + } + + if (argc < 5) + usage("not enough arguments"); + + rate = strtoul(argv[1], &endp, 10); + if (endp == argv[1] || *endp) + usage("rate must be a positive integer (connections per second)"); + + limit = strtoul(argv[2], &endp, 10); + if (endp == argv[2] || *endp) + usage("limit must be a positive integer (max outstanding requests)"); + + run(rate, limit, argv[3], argv+4); + return 0; +} diff --git a/scripts/tool_urlglob.c b/scripts/tool_urlglob.c new file mode 100644 index 0000000..d714971 --- /dev/null +++ b/scripts/tool_urlglob.c @@ -0,0 +1,516 @@ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2011, Daniel Stenberg, <daniel(a)haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +#include <stdbool.h> +#include <stdlib.h> +#include <string.h> +#include <curl/curl.h> + +#include "tool_urlglob.h" + +typedef enum { + GLOB_OK, + GLOB_NO_MEM, + GLOB_ERROR +} GlobCode; + +/* + * glob_word() + * + * Input a full globbed string, set the forth argument to the amount of + * strings we get out of this. Return GlobCode. + */ +static GlobCode glob_word(URLGlob *, /* object anchor */ + char *, /* globbed string */ + size_t, /* position */ + int *); /* returned number of strings */ + +static GlobCode glob_set(URLGlob *glob, char *pattern, + size_t pos, int *amount) +{ + /* processes a set expression with the point behind the opening '{' + ','-separated elements are collected until the next closing '}' + */ + URLPattern *pat; + GlobCode res; + bool done = false; + char* buf = glob->glob_buffer; + + pat = &glob->pattern[glob->size / 2]; + /* patterns 0,1,2,... correspond to size=1,3,5,... */ + pat->type = UPTSet; + pat->content.Set.size = 0; + pat->content.Set.ptr_s = 0; + pat->content.Set.elements = NULL; + + ++glob->size; + + while(!done) { + switch (*pattern) { + case '\0': /* URL ended while set was still open */ + snprintf(glob->errormsg, sizeof(glob->errormsg), + "unmatched brace at pos %zu\n", pos); + return GLOB_ERROR; + + case '{': + case '[': /* no nested expressions at this time */ + snprintf(glob->errormsg, sizeof(glob->errormsg), + "nested braces not supported at pos %zu\n", pos); + return GLOB_ERROR; + + case ',': + case '}': /* set element completed */ + *buf = '\0'; + if(pat->content.Set.elements) { + char **new_arr = realloc(pat->content.Set.elements, + (pat->content.Set.size + 1) * sizeof(char*)); + if(!new_arr) { + short elem; + for(elem = 0; elem < pat->content.Set.size; elem++) + Curl_safefree(pat->content.Set.elements[elem]); + Curl_safefree(pat->content.Set.elements); + pat->content.Set.ptr_s = 0; + pat->content.Set.size = 0; + } + pat->content.Set.elements = new_arr; + } + else + pat->content.Set.elements = malloc(sizeof(char*)); + if(!pat->content.Set.elements) { + snprintf(glob->errormsg, sizeof(glob->errormsg), "out of memory\n"); + return GLOB_NO_MEM; + } + pat->content.Set.elements[pat->content.Set.size] = + strdup(glob->glob_buffer); + if(!pat->content.Set.elements[pat->content.Set.size]) { + short elem; + for(elem = 0; elem < pat->content.Set.size; elem++) + Curl_safefree(pat->content.Set.elements[elem]); + Curl_safefree(pat->content.Set.elements); + pat->content.Set.ptr_s = 0; + pat->content.Set.size = 0; + snprintf(glob->errormsg, sizeof(glob->errormsg), "out of memory\n"); + return GLOB_NO_MEM; + } + ++pat->content.Set.size; + + if(*pattern == '}') { + /* entire set pattern completed */ + int wordamount; + + /* always check for a literal (may be "") between patterns */ + res = glob_word(glob, ++pattern, ++pos, &wordamount); + if(res) { + short elem; + for(elem = 0; elem < pat->content.Set.size; elem++) + Curl_safefree(pat->content.Set.elements[elem]); + Curl_safefree(pat->content.Set.elements); + pat->content.Set.ptr_s = 0; + pat->content.Set.size = 0; + return res; + } + + *amount = pat->content.Set.size * wordamount; + + done = true; + continue; + } + + buf = glob->glob_buffer; + ++pattern; + ++pos; + break; + + case ']': /* illegal closing bracket */ + snprintf(glob->errormsg, sizeof(glob->errormsg), + "illegal pattern at pos %zu\n", pos); + return GLOB_ERROR; + + case '\\': /* escaped character, skip '\' */ + if(pattern[1]) { + ++pattern; + ++pos; + } + /* intentional fallthrough */ + default: + *buf++ = *pattern++; /* copy character to set element */ + ++pos; + } + } + return GLOB_OK; +} + +static GlobCode glob_range(URLGlob *glob, char *pattern, + size_t pos, int *amount) +{ + /* processes a range expression with the point behind the opening '[' + - char range: e.g. "a-z]", "B-Q]" + - num range: e.g. "0-9]", "17-2000]" + - num range with leading zeros: e.g. "001-999]" + expression is checked for well-formedness and collected until the next ']' + */ + URLPattern *pat; + char *c; + char sep; + char sep2; + int step; + int rc; + GlobCode res; + int wordamount = 1; + + pat = &glob->pattern[glob->size / 2]; + /* patterns 0,1,2,... correspond to size=1,3,5,... */ + ++glob->size; + + if(ISALPHA(*pattern)) { + /* character range detected */ + char min_c; + char max_c; + + pat->type = UPTCharRange; + + rc = sscanf(pattern, "%c-%c%c%d%c", &min_c, &max_c, &sep, &step, &sep2); + + if((rc < 3) || (min_c >= max_c) || ((max_c - min_c) > ('z' - 'a'))) { + /* the pattern is not well-formed */ + snprintf(glob->errormsg, sizeof(glob->errormsg), + "error: bad range specification after pos %zu\n", pos); + return GLOB_ERROR; + } + + /* check the (first) separating character */ + if((sep != ']') && (sep != ':')) { + snprintf(glob->errormsg, sizeof(glob->errormsg), + "error: unsupported character (%c) after range at pos %zu\n", + sep, pos); + return GLOB_ERROR; + } + + /* if there was a ":[num]" thing, use that as step or else use 1 */ + pat->content.CharRange.step = + ((sep == ':') && (rc == 5) && (sep2 == ']')) ? step : 1; + + pat->content.CharRange.ptr_c = pat->content.CharRange.min_c = min_c; + pat->content.CharRange.max_c = max_c; + } + else if(ISDIGIT(*pattern)) { + /* numeric range detected */ + int min_n; + int max_n; + + pat->type = UPTNumRange; + pat->content.NumRange.padlength = 0; + + rc = sscanf(pattern, "%d-%d%c%d%c", &min_n, &max_n, &sep, &step, &sep2); + + if((rc < 2) || (min_n > max_n)) { + /* the pattern is not well-formed */ + snprintf(glob->errormsg, sizeof(glob->errormsg), + "error: bad range specification after pos %zu\n", pos); + return GLOB_ERROR; + } + pat->content.NumRange.ptr_n = pat->content.NumRange.min_n = min_n; + pat->content.NumRange.max_n = max_n; + + /* if there was a ":[num]" thing, use that as step or else use 1 */ + pat->content.NumRange.step = + ((sep == ':') && (rc == 5) && (sep2 == ']')) ? step : 1; + + if(*pattern == '0') { + /* leading zero specified */ + c = pattern; + while(ISDIGIT(*c)) { + c++; + ++pat->content.NumRange.padlength; /* padding length is set for all + instances of this pattern */ + } + } + } + else { + snprintf(glob->errormsg, sizeof(glob->errormsg), + "illegal character in range specification at pos %zu\n", pos); + return GLOB_ERROR; + } + + c = (char*)strchr(pattern, ']'); /* continue after next ']' */ + if(c) + c++; + else { + snprintf(glob->errormsg, sizeof(glob->errormsg), "missing ']'"); + return GLOB_ERROR; /* missing ']' */ + } + + /* always check for a literal (may be "") between patterns */ + + res = glob_word(glob, c, pos + (c - pattern), &wordamount); + if(res == GLOB_ERROR) { + wordamount = 1; + res = GLOB_OK; + } + + if(!res) { + if(pat->type == UPTCharRange) + *amount = wordamount * (pat->content.CharRange.max_c - + pat->content.CharRange.min_c + 1); + else + *amount = wordamount * (pat->content.NumRange.max_n - + pat->content.NumRange.min_n + 1); + } + + return res; /* GLOB_OK or GLOB_NO_MEM */ +} + +static GlobCode glob_word(URLGlob *glob, char *pattern, + size_t pos, int *amount) +{ + /* processes a literal string component of a URL + special characters '{' and '[' branch to set/range processing functions + */ + char* buf = glob->glob_buffer; + size_t litindex; + GlobCode res = GLOB_OK; + + *amount = 1; /* default is one single string */ + + while(*pattern != '\0' && *pattern != '{' && *pattern != '[') { + if(*pattern == '}' || *pattern == ']') { + snprintf(glob->errormsg, sizeof(glob->errormsg), + "unmatched close brace/bracket at pos %zu\n", pos); + return GLOB_ERROR; + } + + /* only allow \ to escape known "special letters" */ + if(*pattern == '\\' && + (*(pattern+1) == '{' || *(pattern+1) == '[' || + *(pattern+1) == '}' || *(pattern+1) == ']') ) { + + /* escape character, skip '\' */ + ++pattern; + ++pos; + } + *buf++ = *pattern++; /* copy character to literal */ + ++pos; + } + *buf = '\0'; + litindex = glob->size / 2; + /* literals 0,1,2,... correspond to size=0,2,4,... */ + glob->literal[litindex] = strdup(glob->glob_buffer); + if(!glob->literal[litindex]) { + snprintf(glob->errormsg, sizeof(glob->errormsg), "out of memory\n"); + return GLOB_NO_MEM; + } + ++glob->size; + + switch (*pattern) { + case '\0': + /* singular URL processed */ + break; + + case '{': + /* process set pattern */ + res = glob_set(glob, ++pattern, ++pos, amount); + break; + + case '[': + /* process range pattern */ + res = glob_range(glob, ++pattern, ++pos, amount); + break; + } + + if(res) + Curl_safefree(glob->literal[litindex]); + + return res; +} + +int glob_url(URLGlob** glob, char* url, int *urlnum, FILE *error) +{ + /* + * We can deal with any-size, just make a buffer with the same length + * as the specified URL! + */ + URLGlob *glob_expand; + int amount; + char *glob_buffer; + GlobCode res; + + *glob = NULL; + + glob_buffer = malloc(strlen(url) + 1); + if(!glob_buffer) + return CURLE_OUT_OF_MEMORY; + + glob_expand = calloc(1, sizeof(URLGlob)); + if(!glob_expand) { + Curl_safefree(glob_buffer); + return CURLE_OUT_OF_MEMORY; + } + glob_expand->size = 0; + glob_expand->urllen = strlen(url); + glob_expand->glob_buffer = glob_buffer; + glob_expand->beenhere = 0; + + res = glob_word(glob_expand, url, 1, &amount); + if(!res) + *urlnum = amount; + else { + if(error && glob_expand->errormsg[0]) { + /* send error description to the error-stream */ + fprintf(error, "curl: (%d) [globbing] %s", + (res == GLOB_NO_MEM) ? CURLE_OUT_OF_MEMORY : CURLE_URL_MALFORMAT, + glob_expand->errormsg); + } + /* it failed, we cleanup */ + Curl_safefree(glob_buffer); + Curl_safefree(glob_expand); + *urlnum = 1; + return (res == GLOB_NO_MEM) ? CURLE_OUT_OF_MEMORY : CURLE_URL_MALFORMAT; + } + + *glob = glob_expand; + return CURLE_OK; +} + +void glob_cleanup(URLGlob* glob) +{ + size_t i; + int elem; + + for(i = glob->size - 1; i < glob->size; --i) { + if(!(i & 1)) { /* even indexes contain literals */ + Curl_safefree(glob->literal[i/2]); + } + else { /* odd indexes contain sets or ranges */ + if((glob->pattern[i/2].type == UPTSet) && + (glob->pattern[i/2].content.Set.elements)) { + for(elem = glob->pattern[i/2].content.Set.size - 1; + elem >= 0; + --elem) { + Curl_safefree(glob->pattern[i/2].content.Set.elements[elem]); + } + Curl_safefree(glob->pattern[i/2].content.Set.elements); + } + } + } + Curl_safefree(glob->glob_buffer); + Curl_safefree(glob); +} + +int glob_next_url(char **globbed, URLGlob *glob) +{ + URLPattern *pat; + char *lit; + size_t i; + size_t j; + size_t len; + size_t buflen = glob->urllen + 1; + char *buf = glob->glob_buffer; + + *globbed = NULL; + + if(!glob->beenhere) + glob->beenhere = 1; + else { + bool carry = true; + + /* implement a counter over the index ranges of all patterns, + starting with the rightmost pattern */ + for(i = glob->size / 2 - 1; carry && (i < glob->size); --i) { + carry = false; + pat = &glob->pattern[i]; + switch (pat->type) { + case UPTSet: + if((pat->content.Set.elements) && + (++pat->content.Set.ptr_s == pat->content.Set.size)) { + pat->content.Set.ptr_s = 0; + carry = true; + } + break; + case UPTCharRange: + pat->content.CharRange.ptr_c = (char)(pat->content.CharRange.step + + (int)((unsigned char)pat->content.CharRange.ptr_c)); + if(pat->content.CharRange.ptr_c > pat->content.CharRange.max_c) { + pat->content.CharRange.ptr_c = pat->content.CharRange.min_c; + carry = true; + } + break; + case UPTNumRange: + pat->content.NumRange.ptr_n += pat->content.NumRange.step; + if(pat->content.NumRange.ptr_n > pat->content.NumRange.max_n) { + pat->content.NumRange.ptr_n = pat->content.NumRange.min_n; + carry = true; + } + break; + default: + printf("internal error: invalid pattern type (%d)\n", (int)pat->type); + return CURLE_FAILED_INIT; + } + } + if(carry) { /* first pattern ptr has run into overflow, done! */ + /* TODO: verify if this should actally return CURLE_OK. */ + return CURLE_OK; /* CURLE_OK to match previous behavior */ + } + } + + for(j = 0; j < glob->size; ++j) { + if(!(j&1)) { /* every other term (j even) is a literal */ + lit = glob->literal[j/2]; + len = snprintf(buf, buflen, "%s", lit); + buf += len; + buflen -= len; + } + else { /* the rest (i odd) are patterns */ + pat = &glob->pattern[j/2]; + switch(pat->type) { + case UPTSet: + if(pat->content.Set.elements) { + len = strlen(pat->content.Set.elements[pat->content.Set.ptr_s]); + snprintf(buf, buflen, "%s", + pat->content.Set.elements[pat->content.Set.ptr_s]); + buf += len; + buflen -= len; + } + break; + case UPTCharRange: + *buf++ = pat->content.CharRange.ptr_c; + break; + case UPTNumRange: + len = snprintf(buf, buflen, "%0*d", + pat->content.NumRange.padlength, + pat->content.NumRange.ptr_n); + buf += len; + buflen -= len; + break; + default: + printf("internal error: invalid pattern type (%d)\n", (int)pat->type); + return CURLE_FAILED_INIT; + } + } + } + *buf = '\0'; + + *globbed = strdup(glob->glob_buffer); + if(!*globbed) + return CURLE_OUT_OF_MEMORY; + + return CURLE_OK; +} diff --git a/scripts/tool_urlglob.h b/scripts/tool_urlglob.h new file mode 100644 index 0000000..562b08e --- /dev/null +++ b/scripts/tool_urlglob.h @@ -0,0 +1,69 @@ +#ifndef HEADER_CURL_TOOL_URLGLOB_H +#define HEADER_CURL_TOOL_URLGLOB_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2011, Daniel Stenberg, <daniel(a)haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +typedef enum { + UPTSet = 1, + UPTCharRange, + UPTNumRange +} URLPatternType; + +typedef struct { + URLPatternType type; + union { + struct { + char **elements; + short size; + short ptr_s; + } Set; + struct { + char min_c; + char max_c; + char ptr_c; + int step; + } CharRange; + struct { + int min_n; + int max_n; + short padlength; + int ptr_n; + int step; + } NumRange ; + } content; +} URLPattern; + +typedef struct { + char *literal[10]; + URLPattern pattern[9]; + size_t size; + size_t urllen; + char *glob_buffer; + char beenhere; + char errormsg[80]; /* error message buffer */ +} URLGlob; + +int glob_url(URLGlob**, char*, int *, FILE *); +int glob_next_url(char **, URLGlob *); +void glob_cleanup(URLGlob* glob); + +#endif /* HEADER_CURL_TOOL_URLGLOB_H */

1 0

[stegotorus/master] Finish bm-mcurl
by zwol＠torproject.org 20 Jul '12

20 Jul '12

commit 3ca55ce2498c8b946fcdcdd52539f2dd90c08ee1 Author: Zack Weinberg <zackw(a)cmu.edu> Date: Mon Feb 13 16:32:08 2012 -0800 Finish bm-mcurl --- scripts/bm-mcurl.c | 266 ++++++++++++++++++++++++++++++++++-------------- scripts/tool_urlglob.c | 46 ++++++--- 2 files changed, 219 insertions(+), 93 deletions(-) diff --git a/scripts/bm-mcurl.c b/scripts/bm-mcurl.c index ac24f3a..890b9e4 100644 --- a/scripts/bm-mcurl.c +++ b/scripts/bm-mcurl.c @@ -15,6 +15,8 @@ #include <stdbool.h> #include <stddef.h> +#include <errno.h> +#include <math.h> #include <stdlib.h> #include <string.h> #include <time.h> @@ -23,18 +25,73 @@ #include <curl/curl.h> #include "tool_urlglob.h" -#define NORETURN __attribute__((noreturn)) +#define NORETURN void __attribute__((noreturn)) +#define UNUSED __attribute__((unused)) static bool verbose = false; +static inline double +timevaldiff(const struct timeval *start, const struct timeval *finish) +{ + double s = finish->tv_sec - start->tv_sec; + s += ((double)(finish->tv_usec - start->tv_usec)) / 1.0e6; + return s; +} + +struct url_iter +{ + char **upats; + URLGlob *uglob; + int nglob; +}; + +static inline struct url_iter +url_prep(char **upats) +{ + struct url_iter it; + it.upats = upats; + it.uglob = NULL; + it.nglob = -1; + return it; +} + +static char * +url_next(struct url_iter *it) +{ + char *url; + + if (!it->uglob) { + for (;;) { + if (!*it->upats) + return 0; + if (!glob_url(&it->uglob, *it->upats, &it->nglob, stderr)) { + if (verbose) + fprintf(stderr, "# %s\n", *it->upats); + break; + } + it->upats++; + } + } + + if (glob_next_url(&url, it->uglob)) + abort(); + if (--it->nglob == 0) { + glob_cleanup(it->uglob); + it->uglob = 0; + it->upats++; + } + return url; +} + static size_t -discard_data(char *ptr, size_t size, size_t nmemb, void *userdata) +discard_data(char *ptr UNUSED, size_t size, size_t nmemb, void *userdata UNUSED) { return size * nmemb; } static size_t -read_abort(void *ptr, size_t size, size_t nmemb, void *userdata) +read_abort(void *ptr UNUSED, size_t size UNUSED, size_t nmemb UNUSED, + void *userdata UNUSED) { /* we don't do anything that should require this to be called, so if it does get called, something is wrong */ @@ -50,11 +107,11 @@ setup_curl_easy_handle(char *proxy) #define SET_OR_CRASH(h, opt, param) \ do { if (curl_easy_setopt(h, opt, param)) abort(); } while (0) - SET_OR_CRASH(h, CURLOPT_VERBOSE, (unsigned long)verbose); + /*SET_OR_CRASH(h, CURLOPT_VERBOSE, (unsigned long)verbose);*/ SET_OR_CRASH(h, CURLOPT_NOPROGRESS, 1L); SET_OR_CRASH(h, CURLOPT_FAILONERROR, 1L); SET_OR_CRASH(h, CURLOPT_USERAGENT, "bm-mcurl/0.1"); - SET_OR_CRASH(h, CURLOPT_ACCEPT_ENCODING, ""); + SET_OR_CRASH(h, CURLOPT_ENCODING, ""); SET_OR_CRASH(h, CURLOPT_AUTOREFERER, 1L); SET_OR_CRASH(h, CURLOPT_FOLLOWLOCATION, 1L); SET_OR_CRASH(h, CURLOPT_MAXREDIRS, 30L); @@ -69,99 +126,148 @@ setup_curl_easy_handle(char *proxy) SET_OR_CRASH(h, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5_HOSTNAME); } #undef SET_OR_CRASH + + return h; } -static bool -process_events_once(CURLM *multi, unsigned long timeout_max) +static void +process_urls(struct url_iter *it, CURLM *multi, CURL **handles, + unsigned long limit, double interval) { - struct timeval tv; - int rc; /* select() return code */ - + struct timeval last, now, timeout; fd_set fdread; fd_set fdwrite; fd_set fdexcept; int maxfd = -1; - - unsigned long timeout = 1000000; /* one second - ultimate default */ - long curl_tout_ms = -1; - - /* get fd sets for all pending transfers */ - FD_ZERO(&fdread); - FD_ZERO(&fdwrite); - FD_ZERO(&fdexcept); - curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcept, &maxfd); - - /* timeout */ - if (timeout_max > 0 && timeout_max < timeout) - timeout = timeout_max; - - curl_multi_timeout(multi_handle, &curl_tout_ms); - - if (curl_tout_ms >= 0) { - unsigned long curl_tout_us = ((unsigned long)curl_tout_ms) * 1000; - if (timeout > curl_tout_us) - timeout = curl_tout_us; + int rc; + int still_running; + int dummy; + unsigned long maxh = 0; + unsigned long i; + CURLMsg *msg; + CURL *h; + char *url; + double d_timeout; + long curl_timeout; + bool no_more_urls = false; + + last.tv_sec = 0; + last.tv_usec = 0; + + for (;;) { + /* possibly queue another URL for download */ + if (!no_more_urls) { + gettimeofday(&now, 0); + if (timevaldiff(&last, &now) >= interval && maxh < limit) { + last = now; + url = url_next(it); + if (url) { + if (curl_easy_setopt(handles[maxh], CURLOPT_URL, url) != CURLM_OK) + abort(); + if (curl_multi_add_handle(multi, handles[maxh]) != CURLM_OK) + abort(); + maxh++; + free(url); /* curl takes a copy */ + } else + no_more_urls = true; + } + } + + /* call curl_multi_perform as many times as it wants */ + again: + switch (curl_multi_perform(multi, &still_running)) { + case CURLM_OK: break; + case CURLM_CALL_MULTI_PERFORM: goto again; + default: + abort(); + } + if (no_more_urls && still_running == 0) + break; + + /* clean up finished downloads */ + while ((msg = curl_multi_info_read(multi, &dummy))) { + if (msg->msg != CURLMSG_DONE) + abort(); /* no other messages are defined as of Feb 2012 */ + h = msg->easy_handle; + if (verbose) { + double rqtime = 0.0; + char *url = "<?>"; + curl_easy_getinfo(h, CURLINFO_EFFECTIVE_URL, &url); + curl_easy_getinfo(h, CURLINFO_TOTAL_TIME, &rqtime); + fprintf(stderr, "%f %s\n", rqtime, url); + } + if (curl_multi_remove_handle(multi, h) != CURLM_OK) + abort(); + for (i = 0; i < maxh; i++) { + if (handles[i] == h) + goto found; + } + abort(); + found: + /* shuffle 'h' to the beginning of the set of handles not + currently in use */ + handles[i] = handles[--maxh]; + handles[maxh] = h; + } + + /* wait for external event or timeout */ + FD_ZERO(&fdread); + FD_ZERO(&fdwrite); + FD_ZERO(&fdexcept); + curl_multi_fdset(multi, &fdread, &fdwrite, &fdexcept, &maxfd); + + curl_multi_timeout(multi, &curl_timeout); + if (curl_timeout >= 0) + d_timeout = ((double)curl_timeout) / 1000.0; + else + d_timeout = 1; + if (d_timeout > interval) + d_timeout = interval; + + timeout.tv_sec = floor(d_timeout); + timeout.tv_usec = lrint((d_timeout - timeout.tv_sec) * 1e6); + + do + rc = select(maxfd+1, &fdread, &fdwrite, &fdexcept, &timeout); + while (rc == -1 && errno == EINTR); + if (rc == -1) + abort(); } - - tv.tv_sec = timeout / 1000000; - if(tv.tv_sec >= 1) - tv.tv_sec = 1; - else - tv.tv_usec = timeout % 1000000; - - do { - rc = select(maxfd+1, &fdread, &fdwrite, &fdexcept, &tv); - } while (rc == -1 && errno == EINTR); - - if (rc > 0) { - int still_running; - curl_multi_perform(multi_handle, &still_running); - return !!still_running; - } else - abort(); } -/* Note: this function must not return until we are ready to start - another connection. */ static void -queue_one(CURLM *multi, unsigned long rate, unsigned long limit, - char *proxy, char *url) -{ - -} - -static void -run(unsigned long rate, unsigned long limit, char *proxy, char **urls) +run(double interval, unsigned long limit, char *proxy, char **upats) { + struct url_iter it; CURLM *multi; - curl_global_init(); + CURL **handles; + unsigned long n; + + curl_global_init(CURL_GLOBAL_ALL); multi = curl_multi_init(); if (!multi) abort(); - for (char **upat = urls; *upat; url++) { - URLGlob *uglob; - int *n; - if (glob_url(&uglob, *upat, &n, stderr)) - continue; - do { - char *url; - if (glob_next_url(&url, uglob)) abort(); - queue_one(multi, rate, limit, proxy, url); /* takes ownership */ - } while (--n); - glob_cleanup(uglob); + handles = calloc(limit, sizeof(CURL *)); + for (n = 0; n < limit; n++) { + handles[n] = setup_curl_easy_handle(proxy); } - /* spin the event loop until all outstanding transfers complete */ - while (process_events_once(multi, 0)); + it = url_prep(upats); + process_urls(&it, multi, handles, limit, interval); + for (n = 0; n < limit; n++) { + curl_easy_cleanup(handles[n]); + } + free(handles); curl_multi_cleanup(multi); + curl_global_cleanup(); } static NORETURN usage(const char *av0, const char *complaint) { fprintf(stderr, - "%s\nusage: %s [-v] rate limit proxy url [url...]\n", + "%s\nusage: %s [-v] cps limit proxy url [url...]\n", complaint, av0); exit(2); } @@ -169,7 +275,7 @@ usage(const char *av0, const char *complaint) int main(int argc, char **argv) { - unsigned long rate; + unsigned long cps; unsigned long limit; char *endp; @@ -181,16 +287,20 @@ main(int argc, char **argv) } if (argc < 5) - usage("not enough arguments"); + usage(argv[0], "not enough arguments"); - rate = strtoul(argv[1], &endp, 10); + cps = strtoul(argv[1], &endp, 10); if (endp == argv[1] || *endp) - usage("rate must be a positive integer (connections per second)"); + usage(argv[0], "cps must be a positive integer (connections per second)"); limit = strtoul(argv[2], &endp, 10); if (endp == argv[2] || *endp) - usage("limit must be a positive integer (max outstanding requests)"); + usage(argv[0], + "limit must be a positive integer (max outstanding requests)"); + + if (limit == 0) + usage(argv[0], "minimum number of outstanding requests is 1"); - run(rate, limit, argv[3], argv+4); + run(1./(double)cps, limit, argv[3], argv+4); return 0; } diff --git a/scripts/tool_urlglob.c b/scripts/tool_urlglob.c index d714971..b160188 100644 --- a/scripts/tool_urlglob.c +++ b/scripts/tool_urlglob.c @@ -20,6 +20,8 @@ * ***************************************************************************/ +#define _XOPEN_SOURCE 600 /* strdup */ + #include <stdbool.h> #include <stdlib.h> #include <string.h> @@ -33,6 +35,20 @@ typedef enum { GLOB_ERROR } GlobCode; +/* assumes ASCII */ +static inline bool +ISALPHA(unsigned char c) +{ + return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); +} + +/* C90 guarantees '0' through '9' are consecutive */ +static inline bool +ISDIGIT(unsigned char c) +{ + return c >= '0' && c <= '9'; +} + /* * glob_word() * @@ -86,8 +102,8 @@ static GlobCode glob_set(URLGlob *glob, char *pattern, if(!new_arr) { short elem; for(elem = 0; elem < pat->content.Set.size; elem++) - Curl_safefree(pat->content.Set.elements[elem]); - Curl_safefree(pat->content.Set.elements); + free(pat->content.Set.elements[elem]); + free(pat->content.Set.elements); pat->content.Set.ptr_s = 0; pat->content.Set.size = 0; } @@ -104,8 +120,8 @@ static GlobCode glob_set(URLGlob *glob, char *pattern, if(!pat->content.Set.elements[pat->content.Set.size]) { short elem; for(elem = 0; elem < pat->content.Set.size; elem++) - Curl_safefree(pat->content.Set.elements[elem]); - Curl_safefree(pat->content.Set.elements); + free(pat->content.Set.elements[elem]); + free(pat->content.Set.elements); pat->content.Set.ptr_s = 0; pat->content.Set.size = 0; snprintf(glob->errormsg, sizeof(glob->errormsg), "out of memory\n"); @@ -122,8 +138,8 @@ static GlobCode glob_set(URLGlob *glob, char *pattern, if(res) { short elem; for(elem = 0; elem < pat->content.Set.size; elem++) - Curl_safefree(pat->content.Set.elements[elem]); - Curl_safefree(pat->content.Set.elements); + free(pat->content.Set.elements[elem]); + free(pat->content.Set.elements); pat->content.Set.ptr_s = 0; pat->content.Set.size = 0; return res; @@ -337,7 +353,7 @@ static GlobCode glob_word(URLGlob *glob, char *pattern, } if(res) - Curl_safefree(glob->literal[litindex]); + free(glob->literal[litindex]); return res; } @@ -361,7 +377,7 @@ int glob_url(URLGlob** glob, char* url, int *urlnum, FILE *error) glob_expand = calloc(1, sizeof(URLGlob)); if(!glob_expand) { - Curl_safefree(glob_buffer); + free(glob_buffer); return CURLE_OUT_OF_MEMORY; } glob_expand->size = 0; @@ -380,8 +396,8 @@ int glob_url(URLGlob** glob, char* url, int *urlnum, FILE *error) glob_expand->errormsg); } /* it failed, we cleanup */ - Curl_safefree(glob_buffer); - Curl_safefree(glob_expand); + free(glob_buffer); + free(glob_expand); *urlnum = 1; return (res == GLOB_NO_MEM) ? CURLE_OUT_OF_MEMORY : CURLE_URL_MALFORMAT; } @@ -397,7 +413,7 @@ void glob_cleanup(URLGlob* glob) for(i = glob->size - 1; i < glob->size; --i) { if(!(i & 1)) { /* even indexes contain literals */ - Curl_safefree(glob->literal[i/2]); + free(glob->literal[i/2]); } else { /* odd indexes contain sets or ranges */ if((glob->pattern[i/2].type == UPTSet) && @@ -405,14 +421,14 @@ void glob_cleanup(URLGlob* glob) for(elem = glob->pattern[i/2].content.Set.size - 1; elem >= 0; --elem) { - Curl_safefree(glob->pattern[i/2].content.Set.elements[elem]); + free(glob->pattern[i/2].content.Set.elements[elem]); } - Curl_safefree(glob->pattern[i/2].content.Set.elements); + free(glob->pattern[i/2].content.Set.elements); } } } - Curl_safefree(glob->glob_buffer); - Curl_safefree(glob); + free(glob->glob_buffer); + free(glob); } int glob_next_url(char **globbed, URLGlob *glob)

1 0

[stegotorus/master] Merge branch 'master' of ssh://sandbox03.sv.cmu.edu/usr/home/zack/stegotorus
by zwol＠torproject.org 20 Jul '12

20 Jul '12

commit 33dd34526ccf5c5b87875838857251f7bd4f8963 Merge: 704c053 572cfd2 Author: Zack Weinberg <zackw(a)cmu.edu> Date: Mon Feb 13 10:55:06 2012 -0800 Merge branch 'master' of ssh://sandbox03.sv.cmu.edu/usr/home/zack/stegotorus src/main.cc | 2 +- src/socks.cc | 1 + src/test/unittest_socks.cc | 1 + src/util.cc | 4 ++-- 4 files changed, 5 insertions(+), 3 deletions(-)

1 0

[stegotorus/master] Another subtle bug involving races near circuit teardown.
by zwol＠torproject.org 20 Jul '12

20 Jul '12

commit 4ae14e9e21d9b2848d089ba3ee150582e20744b9 Author: Zack Weinberg <zackw(a)panix.com> Date: Wed Feb 1 21:19:18 2012 -0800 Another subtle bug involving races near circuit teardown. --- src/network.cc | 3 ++- src/protocol/chop.cc | 32 +++++++++++++++++++------------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/network.cc b/src/network.cc index a11cd7d..a55e539 100644 --- a/src/network.cc +++ b/src/network.cc @@ -423,7 +423,8 @@ upstream_flush_cb(struct bufferevent *bev, void *arg) ckt->connected ? "" : " (not connected)", ckt->flushing ? "" : " (not flushing)"); - if (remain == 0 && ckt->flushing && ckt->connected) { + if (remain == 0 && ckt->flushing && ckt->connected + && (!ckt->flush_timer || !evtimer_pending(ckt->flush_timer, NULL))) { bufferevent_disable(bev, EV_WRITE); if (bufferevent_get_enabled(bev) || evbuffer_get_length(bufferevent_get_input(bev)) > 0) { diff --git a/src/protocol/chop.cc b/src/protocol/chop.cc index 5acc010..d7da97f 100644 --- a/src/protocol/chop.cc +++ b/src/protocol/chop.cc @@ -784,8 +784,6 @@ chop_push_to_upstream(circuit_t *c) static int chop_find_or_make_circuit(conn_t *conn, uint64_t circuit_id) { - log_assert(conn->cfg->mode == LSN_SIMPLE_SERVER); - chop_config_t *cfg = static_cast<chop_config_t *>(conn->cfg); chop_circuit_table::value_type in(circuit_id, 0); std::pair<chop_circuit_table::iterator, bool> out = cfg->circuits.insert(in); @@ -954,6 +952,11 @@ chop_config_t::circuit_create(size_t) ckt->recv_crypt = decryptor::create(s2c_key, 16); while (!ckt->circuit_id) rng_bytes((uint8_t *)&ckt->circuit_id, sizeof(uint64_t)); + + chop_circuit_table::value_type in(ckt->circuit_id, 0); + std::pair<chop_circuit_table::iterator, bool> out = circuits.insert(in); + log_assert(out.second); + out.first->second = ckt; } return ckt; } @@ -996,17 +999,20 @@ chop_circuit_t::~chop_circuit_t() free(p); } - if (this->cfg->mode == LSN_SIMPLE_SERVER) { - /* The IDs for old circuits are preserved for a while (at present, - indefinitely; FIXME: purge them on a timer) against the - possibility that we'll get a junk connection for one of them - right after we close it (same deal as the TIME_WAIT state in TCP). */ - chop_config_t *cfg = static_cast<chop_config_t *>(this->cfg); - out = cfg->circuits.find(this->circuit_id); - log_assert(out != cfg->circuits.end()); - log_assert(out->second == this); - out->second = NULL; - } + /* The IDs for old circuits are preserved for a while (at present, + indefinitely; FIXME: purge them on a timer) against the + possibility that we'll get a junk connection for one of them + right after we close it (same deal as the TIME_WAIT state in + TCP). Note that we can hit this case for the *client* if the + cover protocol includes a mandatory reply to every client + message and the hidden channel closed s->c before c->s: the + circuit will get destroyed on the client side after the c->s FIN, + and the mandatory reply will be to a stale circuit. */ + chop_config_t *cfg = static_cast<chop_config_t *>(this->cfg); + out = cfg->circuits.find(this->circuit_id); + log_assert(out != cfg->circuits.end()); + log_assert(out->second == this); + out->second = NULL; } void

1 0

[stegotorus/master] Yet another missed shutdown case.
by zwol＠torproject.org 20 Jul '12

20 Jul '12

commit e06cb9a45c52f0190f37be0a9109f25decda126e Author: Zack Weinberg <zackw(a)panix.com> Date: Thu Feb 2 08:43:18 2012 -0800 Yet another missed shutdown case. --- src/protocol/chop.cc | 19 +++++++++++++++++++ 1 files changed, 19 insertions(+), 0 deletions(-) diff --git a/src/protocol/chop.cc b/src/protocol/chop.cc index d7da97f..89a9ccc 100644 --- a/src/protocol/chop.cc +++ b/src/protocol/chop.cc @@ -1282,6 +1282,25 @@ chop_conn_t::recv() if (evbuffer_get_length(bufferevent_get_input(c->up_buffer))) c->send(); + /* If we're at EOF, close all connections (sending first if + necessary). If we're the client we have to keep trying to talk + as long as we haven't both sent and received a FIN, or we might + deadlock. */ + else if (ckt->sent_fin && ckt->received_fin) { + circuit_disarm_flush_timer(ckt); + for (unordered_set<conn_t *>::iterator i = ckt->downstreams.begin(); + i != ckt->downstreams.end(); i++) { + chop_conn_t *conn = static_cast<chop_conn_t*>(*i); + if (conn->must_transmit_timer && + evtimer_pending(conn->must_transmit_timer, NULL)) + must_transmit_timer_cb(-1, 0, conn); + conn_send_eof(conn); + } + } else { + if (ckt->cfg->mode != LSN_SIMPLE_SERVER) + circuit_arm_flush_timer(ckt, ckt->flush_interval()); + } + return 0; }

1 0

[stegotorus/master] Monkeyin' with the test suite.
by zwol＠torproject.org 20 Jul '12

20 Jul '12

commit dda19c5a05404677b14ca8460b7cebf7d330b91a Author: Zack Weinberg <zackw(a)panix.com> Date: Thu Feb 2 09:51:58 2012 -0800 Monkeyin' with the test suite. --- src/test/test_tl.py | 28 +++++++++++++--------------- 1 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/test/test_tl.py b/src/test/test_tl.py index a91beca..6d632f8 100644 --- a/src/test/test_tl.py +++ b/src/test/test_tl.py @@ -70,21 +70,19 @@ class TimelineTest(object): "127.0.0.1:5010","nosteg_rr", )) - # def test_chop_nosteg_rr2(self): - # self.doTest("chop", - # ("chop", "server", "127.0.0.1:5001", - # "127.0.0.1:5010","nosteg_rr","127.0.0.1:5011","nosteg_rr", - # "chop", "client", "127.0.0.1:4999", - # "127.0.0.1:5010","nosteg_rr","127.0.0.1:5011","nosteg_rr", - # )) - - # def test_chop_http(self): - # self.doTest("chop", - # ("chop", "server", "127.0.0.1:5001", - # "127.0.0.1:5010","http","127.0.0.1:5011","http", - # "chop", "client", "127.0.0.1:4999", - # "127.0.0.1:5010","http","127.0.0.1:5011","http", - # )) + def test_chop_nosteg_rr2(self): + self.doTest("chop", + ("chop", "server", "127.0.0.1:5001", + "127.0.0.1:5010","nosteg_rr","127.0.0.1:5011","nosteg_rr", + "chop", "client", "127.0.0.1:4999", + "127.0.0.1:5010","nosteg_rr","127.0.0.1:5011","nosteg_rr", + )) + + # NOTE: 'embed' steg presently cannot be tested using this system + # because it runs out of trace data before any of the tests complete. + + # NOTE: 'http' steg presently cannot be tested using this system + # because the trace pools are process-global rather than per-listener. # Synthesize TimelineTest+TestCase subclasses for every 'tl_*' file in # the test directory.

1 0

[stegotorus/master] first version of stegotorus... binary name still obfsproxy
by zwol＠torproject.org 20 Jul '12

20 Jul '12

commit 2e5825fee2123ad6fcac1fd5aff7b85ea65b5b06 Author: Vinod Yegneswaran <vinod(a)csl.sri.com> Date: Mon Oct 31 22:33:46 2011 +0000 first version of stegotorus... binary name still obfsproxy js + flash steg enabled. git-svn-id: svn+ssh://spartan.csl.sri.com/svn/private/DEFIANCE@107 a58ff0ac-194c-e011-a152-003048836090 --- Makefile.am | 10 +- NOTES | 20 + run-autogen.csh | 3 + src/steg/cookies.c | 230 ++++++++ src/steg/cookies.h | 17 + src/steg/crc32.c | 82 +++ src/steg/crc32.h | 18 + src/steg/jsSteg.c | 1200 ++++++++++++++++++++++++++++++++++++++ src/steg/jsSteg.h | 65 +++ src/steg/payloads.c | 1486 ++++++++++++++++++++++++++++++++++++++++++++++++ src/steg/payloads.h | 159 ++++++ src/steg/pdfSteg.c | 618 ++++++++++++++++++++ src/steg/pdfSteg.h | 29 + src/steg/swfSteg.c | 282 +++++++++ src/steg/swfSteg.c.old | 264 +++++++++ src/steg/swfSteg.h | 42 ++ src/steg/x_http.c.old | 337 +++++++++++ src/steg/x_http2.c | 700 +++++++++++++++++++++++ src/steg/zpack.c | 408 +++++++++++++ src/steg/zpack.h | 14 + start-client.csh | 8 + start-server.csh | 6 + torrc | 12 + 23 files changed, 6009 insertions(+), 1 deletions(-) diff --git a/Makefile.am b/Makefile.am index bcb6e88..e8b68f5 100644 --- a/Makefile.am +++ b/Makefile.am @@ -22,7 +22,15 @@ PROTOCOLS = \ src/protocols/x_rr.c STEGANOGRAPHERS = \ - src/steg/x_http.c + src/steg/x_http.c \ + src/steg/x_http2.c \ + src/steg/payloads.c \ + src/steg/cookies.c \ + src/steg/jsSteg.c \ + src/steg/swfSteg.c \ + src/steg/zpack.c \ + src/steg/crc32.c \ + src/steg/pdfSteg.c libobfsproxy_a_SOURCES = \ src/connections.c \ diff --git a/NOTES b/NOTES new file mode 100644 index 0000000..5b5f23c --- /dev/null +++ b/NOTES @@ -0,0 +1,20 @@ +To Test ObfsProxy: + +start server (start-server.csh) +start client (start-client.csh) +start Tor (copy torrc.sample to etc/tor) + + +Now test tor: +curl --socks4a 127.0.0.1:9060 -o - http://check.torproject.org + + +To add new steg modules: + +1. Add a file below src/steg which implements a steg module; note that +the STEG_DEFINE_MODULE boilerplate macro is mandatory. The name of +the file should be the same as the name of the module (as set by +STEG_DEFINE_MODULE) plus the .c extension. +2. Add the file to the STEGANOGRAPHERS list in Makefile.am. + +That should be all that is necessary. diff --git a/run-autogen.csh b/run-autogen.csh new file mode 100644 index 0000000..9fe8c44 --- /dev/null +++ b/run-autogen.csh @@ -0,0 +1,3 @@ +#!/bin/csh +setenv PATH /usr/local/bin:`echo $PATH` +./autogen.sh diff --git a/src/steg/cookies.c b/src/steg/cookies.c new file mode 100644 index 0000000..e8d43b9 --- /dev/null +++ b/src/steg/cookies.c @@ -0,0 +1,230 @@ + +#include "cookies.h" + +int unwrap_cookie(unsigned char* inbuf, unsigned char* outbuf, int buflen) { + int i,j; + j = 0; + + for (i=0; i < buflen; i++) { + char c = inbuf[i]; + + if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')) + outbuf[j++] = c; + } + + return j; + +} + + + + + + +/* valid cookie characters: anything between between 33 and 126, with the exception of "=" and ';'*/ +/* writes a line of the form XXXX=YYYYY of length cookielen to outbuf, outbuf length >= cookielen */ +/* returns data consumed if success. datalen assumed to be greater than cookielen*/ + + +int gen_one_cookie(unsigned char* outbuf, int cookielen, unsigned char* data, int datalen) { + int sofar = 0; + unsigned char c; + int namelen, vlen; + int data_consumed = 0; + + if (cookielen < 4) + return -1; + + + + if (cookielen > 13) + namelen = rand() % 10 + 1; + else + namelen = rand() % (cookielen - 3) + 1; + + vlen = cookielen - namelen; + + + + while (sofar < namelen) { + c = rand() % (127 - 33) + 33; + if (c == '=' || c == ';' || c == '`' || c == '\'' || c == '%') + continue; + + if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') || (rand () % 4 != 0)) { + if (data_consumed < datalen) + outbuf[sofar++] = data[data_consumed++]; + } + else + outbuf[sofar++] = c; + } + + + outbuf[sofar++] = '='; + + + while (sofar < cookielen) { + c = rand() % (127 - 33) + 33; + if (c == '=' || c == ';' || c == '`' || c == '\'' || c == '%') + continue; + + if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') || (rand() % 4 != 0)) { + if (data_consumed < datalen) + outbuf[sofar++] = data[data_consumed++]; + } + else + outbuf[sofar++] = c; + } + + + + return data_consumed; + +} + + + + +/* dummy version for testing */ +int gen_one_cookie2(unsigned char* outbuf, int cookielen, unsigned char* data, int datalen) { + int i; + + if (cookielen < 4) + return -1; + + if (datalen >= cookielen) { + memcpy(outbuf, data, cookielen); + return cookielen; + } + + memcpy(outbuf, data, datalen); + for (i=datalen; i < cookielen; i++) + outbuf[i] = '+'; + + return datalen; + +} + +/* returns data consumed */ +int gen_cookie_field(unsigned char* outbuf, int total_cookie_len, unsigned char* data, int datalen) { + int rem_cookie_len = total_cookie_len; + int consumed = 0; + + + if (total_cookie_len < 4) { + fprintf(stderr, "error: cookie length too small\n"); + return -1; + } + + while (rem_cookie_len > 4) { + int cookielen = 4 + rand() % (rem_cookie_len - 3); + + int cnt = gen_one_cookie(outbuf, cookielen, data + consumed, datalen - consumed); + + if (cnt < 0) { + fprintf(stderr, "error: couldn't create cookie %d\n", cnt); + return cnt; + } + + + + consumed += cnt; + // fprintf(stderr, "cnt = %d %d %d; consumed = %d\n", cnt, rem_cookie_len, cookielen, consumed); + rem_cookie_len = rem_cookie_len - cookielen; + outbuf += cookielen; + + + + if (rem_cookie_len == 0) { + break; + } + else if (rem_cookie_len <= 5) { + int i = 0; + if ((consumed < datalen) && (consumed % 2 == 1)) { + outbuf[0] = data[consumed]; + consumed++; + outbuf++; + rem_cookie_len--; + } + + for (i=0; i < rem_cookie_len; i++) + outbuf[i] = "ghijklmnopqrstuvwxyzGHIJKLMNOPQRSTUVWXYZ"[rand() % 40]; + + return consumed; + } + + + outbuf[0] = ';'; + outbuf++; + rem_cookie_len--; + } + + + if (consumed % 2 == 1) { + if ((outbuf[-1] >= '0' && outbuf[-1] <= '9') || (outbuf[-1] >= 'a' && outbuf[-1] <= 'f')) { + outbuf[-1] = '*'; + consumed--; + } + else { + outbuf[-1] = data[consumed]; + consumed++; + } + } + + + return consumed; +} + + +/* dummy version for testing */ +int gen_cookie_field2(unsigned char* outbuf, int total_cookie_len, unsigned char* data, int datalen) { + int i; + if (datalen >= total_cookie_len) { + memcpy(outbuf, data, total_cookie_len); + return total_cookie_len; + } + + memcpy(outbuf, data, datalen); + for (i=datalen; i < total_cookie_len; i++) + outbuf[i] = '*'; + + return datalen; +} + + + + +/* + +int main () { + char outbuf[200]; + char data[52] = "1a239023820389023802380389abc2322132321932847203aedf"; + char data2[200]; + // srand(time(NULL)); + srand (20); + + + + int i=0; + + for (i=0; i < 1000000; i++) { + int cookielen = rand()%50 + 5; + bzero(outbuf, sizeof(outbuf)); + int len = gen_cookie_field(outbuf, cookielen, data, sizeof(data)); + // printf("len = %d cookie = %s %d\n", len, outbuf, cookielen); + bzero(data2, sizeof(data2)); + int len2 = unwrap_cookie(outbuf, data2, cookielen); + // printf("unwrapped datalen = %d data = %s\n", len, data2); + + if (len != len2) + printf("hello %d\n", i); + } + + + + +} + +*/ + + diff --git a/src/steg/cookies.h b/src/steg/cookies.h new file mode 100644 index 0000000..4970776 --- /dev/null +++ b/src/steg/cookies.h @@ -0,0 +1,17 @@ +#ifndef _COOKIES_H +#define _COOKIES_H + + + +#include <stdio.h> +#include <strings.h> +#include <stdlib.h> + +int unwrap_cookie(unsigned char* inbuf, unsigned char* outbuf, int buflen); +int gen_cookie_field(unsigned char* outbuf, int total_cookie_len, unsigned char* data, int datalen); +int gen_one_cookie(unsigned char* outbuf, int cookielen, unsigned char* data, int datalen); +int gen_one_cookie2(unsigned char* outbuf, int cookielen, unsigned char* data, int datalen); +int gen_cookie_field2(unsigned char* outbuf, int total_cookie_len, unsigned char* data, int datalen); + + +#endif diff --git a/src/steg/crc32.c b/src/steg/crc32.c new file mode 100644 index 0000000..7fdc847 --- /dev/null +++ b/src/steg/crc32.c @@ -0,0 +1,82 @@ +#include "crc32.h" + +#define CRC32C(c,d) (c=(c>>8)^crc_c[(c^(d))&0xFF]) + +static const unsigned int crc_c[256] = { + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, + 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, + 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, + 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, + 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, + 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, + 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, + 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, + 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, + 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, + 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, + 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, + 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, + 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, + 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, + 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, + 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, + 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, + 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, + 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, + 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, + 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, + 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, + 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, + 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, + 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, + 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, + 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, + 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, + 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, + 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, + 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, + 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, + 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, + 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, + 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, + 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, + 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, + 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, + 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, + 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, + 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, + 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, + 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, + 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, + 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, + 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, + 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, + 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, + 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, + 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, + 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, + 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, + 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, + 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, + 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, + 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d, +}; + + +unsigned int generate_crc32c(char *buffer, size_t length) { + size_t i; + unsigned int crc32 = ~0L; + + for (i = 0; i < length; i++){ + CRC32C(crc32, (unsigned char)buffer[i]); + } + return ~crc32; +} + diff --git a/src/steg/crc32.h b/src/steg/crc32.h new file mode 100644 index 0000000..780e7bd --- /dev/null +++ b/src/steg/crc32.h @@ -0,0 +1,18 @@ +#ifndef __crc32cr_table_h__ +#define __crc32cr_table_h__ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include <sys/types.h> + +#if defined HAVE_STDINT_H +# include <stdint.h> +#elif defined HAVE_INTTYPES_H +# include <inttypes.h> +#endif + +unsigned int generate_crc32c(char *string, size_t length); + +#endif diff --git a/src/steg/jsSteg.c b/src/steg/jsSteg.c new file mode 100644 index 0000000..5946062 --- /dev/null +++ b/src/steg/jsSteg.c @@ -0,0 +1,1200 @@ +#include "payloads.h" +#include "jsSteg.h" +#include "cookies.h" + + +/* + * jsSteg: A Javascript-based steganography module + * + */ + + +/* + * int isxString(char *str) + * + * description: + * return 1 if all char in str are hexadecimal + * return 0 otherwise + * + */ +int isxString(char *str) { + unsigned int i; + char *dp = str; + for (i=0; i<strlen(str); i++) { + if (! isxdigit(*dp) ) { + return 0; + } + } + return 1; +} + + +/* + * int encode(char *data, char *jTemplate, char *jData, + * unsigned int dlen, unsigned int jtlen, unsigned int jdlen) + * + * description: + * embed hex-encoded data (data) in the input Javascript (jTemplate) + * and put the result in jData + * function returns the number of characters in data successfully + * embedded in jData, or returns one of the error codes + * + * approach: + * replaces characters in jTemplate that are hexadecimal (i.e., {0-9,a-f,A-F}) + * with those in data, and leave the non-hex char in place + * + * input: + * - data[] : hex data to hide + * - dlen : size of data + * - jTemplate[] : Javascript + * - jlen : size of jTemplate + * - jdlen : size of jData, output buffer + * + * output: + * - jData : result of encoding data in jTemplate + * + * assumptions: + * - data is hex-encoded + * + * exceptions: + * - if (jdlen < jtlen) return INVALID_BUF_SIZE + * - if (data contains non-hex char) return INVALID_DATA_CHAR + * + * example: + * data = "0123456789ABCDEF" + * jTemplate = "dfp_ord=Math.random()*10000000000000000; dfp_tile = 1;" + * encode() returns 16 + * jData = "01p_or2=M3th.r4n5om()*6789ABCDEF0000000; dfp_tile = 1;" + * + */ +int encode(char *data, char *jTemplate, char *jData, + unsigned int dlen, unsigned int jtlen, unsigned int jdlen ) +{ + unsigned int encCnt = 0; /* num of data encoded in jData */ + char *dp, *jtp, *jdp; /* current pointers for data, jTemplate, and jData */ + + unsigned int j; + + /* + * insanity checks + */ + if (jdlen < jtlen) { return INVALID_BUF_SIZE; } + + dp = data; jtp = jTemplate; jdp = jData; + + if (! isxString(dp) ) { return INVALID_DATA_CHAR; } + + /* handling boundary case: dlen == 0 */ + if (dlen < 1) { return 0; } + + + for (j=0; j<jtlen; j++) { + /* found a hex char in jTemplate that can be used for encoding data */ + if ( isxdigit(*jtp) ) { + *jdp = *dp; + dp++; + encCnt++; + if (encCnt == dlen) { + jtp++; jdp++; + break; + } + } else { + *jdp = *jtp; + } + jtp++; jdp++; + } + + + /* copying the rest of jTemplate to jdata */ + while (jtp < (jTemplate+jtlen)) { + *jdp++ = *jtp++; + } + + return encCnt; +} + + +#define startScriptTypeJS "<script type=\"text/javascript\">" +#define endScriptTypeJS "</script>" +// #define JS_DELIMITER "?" +// #define JS_DELIMITER_REPLACEMENT "." + + +/* + * similar to encode(), but uses offset2Hex to look for usable hex char + * in JS for encoding. See offset2Hex for what hex char are considered + * usable. encode() also converts JS_DELIMITER that appears in the + * the JS to JS_DELIMITER_REPLACEMENT, before all the data is encoded. + * + * Output: + * fin - signal the caller whether all data has been encoded and + * a JS_DELIMITER has been added + */ +int encode2(char *data, char *jTemplate, char *jData, + unsigned int dlen, unsigned int jtlen, + unsigned int jdlen, int *fin) +{ + unsigned int encCnt = 0; /* num of data encoded in jData */ + char *dp, *jtp, *jdp; /* current pointers for data, jTemplate, and jData */ + int i,j; + + /* + * insanity checks + */ + if (jdlen < jtlen) { return INVALID_BUF_SIZE; } + + dp = data; jtp = jTemplate; jdp = jData; + + if (! isxString(dp) ) { return INVALID_DATA_CHAR; } + + /* handling boundary case: dlen == 0 */ + if (dlen < 1) { return 0; } + + + i = offset2Hex(jtp, (jTemplate+jtlen)-jtp, 0); + while (encCnt < dlen && i != -1) { + // copy next i char from jtp to jdp, + // except that if *jtp==JS_DELIMITER, copy + // JS_DELIMITER_REPLACEMENT to jdp instead + j = 0; + while (j < i) { + if (*jtp == JS_DELIMITER) { + *jdp = JS_DELIMITER_REPLACEMENT; + } else { + *jdp = *jtp; + } + jtp = jtp + 1; jdp = jdp + 1; j++; + } + + *jdp = *dp; + encCnt++; + dp = dp + 1; jtp = jtp + 1; jdp = jdp + 1; + + i = offset2Hex(jtp, (jTemplate+jtlen)-jtp, 1); + } + + + + // copy the rest of jTemplate to jdata + // if we've encoded all data, replace the first + // char in jTemplate by JS_DELIMITER, if needed, + // to signal the end of data encoding + +#ifdef DEBUG2 + printf("encode2: encCnt = %d; dlen = %d\n", encCnt, dlen); +#endif + + *fin = 0; + if (encCnt == dlen) { + // replace the next char in jTemplate by JS_DELIMITER + if (jtp < (jTemplate+jtlen)) { + *jdp = JS_DELIMITER; + } + jdp = jdp+1; jtp = jtp+1; + *fin = 1; + } + + while (jtp < (jTemplate+jtlen)) { + if (*jtp == JS_DELIMITER) { + if (encCnt < dlen) { + *jdp = JS_DELIMITER_REPLACEMENT; + } else { + *jdp = *jtp; + } + // else if (isxdigit(*jtp)) { + // if (encCnt < dlen && *fin == 0) { + // *jdp = JS_DELIMITER; + // *fin = 1; + // } else { + // *jdp = *jtp; + // } + // } + } else { + *jdp = *jtp; + } + jdp = jdp+1; jtp = jtp+1; + } + +#ifdef DEBUG2 + printf("encode2: encCnt = %d; dlen = %d\n", encCnt, dlen); + printf("encode2: fin= %d\n", *fin); +#endif + + return encCnt; + +} + + + +int encodeHTTPBody(char *data, char *jTemplate, char *jData, + unsigned int dlen, unsigned int jtlen, + unsigned int jdlen, int mode) +{ + char *dp, *jtp, *jdp; // current pointers for data, jTemplate, and jData + unsigned int encCnt = 0; // num of data encoded in jData + int n; // tmp for updating encCnt + char *jsStart, *jsEnd; + int skip; + int scriptLen; + int fin; + unsigned int dlen2 = dlen; + dp = data; + jtp = jTemplate; + jdp = jData; + + + if (mode == CONTENT_JAVASCRIPT) { + // assumption: the javascript pertaining to jTemplate has enough capacity + // to encode jData. thus, we only invoke encode() once here. + encCnt = encode2(dp, jtp, jdp, dlen, jtlen, jdlen, &fin); + // ensure that all dlen char from data have been encoded in jData +#ifdef DEBUG + if (encCnt != dlen || fin == 0) { + printf("Problem encoding all data to the JS\n"); + } +#endif + return encCnt; + + } + + else if (mode == CONTENT_HTML_JAVASCRIPT) { + while (encCnt < dlen2) { + jsStart = strstr(jtp, startScriptTypeJS); + if (jsStart == NULL) { +#ifdef DEBUG + printf("lack of usable JS; can't find startScriptType\n"); +#endif + return encCnt; + } + skip = strlen(startScriptTypeJS)+jsStart-jtp; +#ifdef DEBUG2 + printf("copying %d (skip) char from jtp to jdp\n", skip); +#endif + memcpy(jdp, jtp, skip); + jtp = jtp+skip; jdp = jdp+skip; + jsEnd = strstr(jtp, endScriptTypeJS); + if (jsEnd == NULL) { +#ifdef DEBUG + printf("lack of usable JS; can't find endScriptType\n"); +#endif + return encCnt; + } + + // the JS for encoding data is between jsStart and jsEnd + scriptLen = jsEnd - jtp; + // n = encode2(dp, jtp, jdp, dlen, jtlen, jdlen, &fin); + n = encode2(dp, jtp, jdp, dlen, scriptLen, jdlen, &fin); + // update encCnt, dp, and dlen based on n + if (n > 0) { + encCnt = encCnt+n; dp = dp+n; dlen = dlen-n; + } + // update jtp, jdp, jdlen + skip = jsEnd-jtp; + jtp = jtp+skip; jdp = jdp+skip; jdlen = jdlen-skip; + skip = strlen(endScriptTypeJS); + memcpy(jdp, jtp, skip); + jtp = jtp+skip; jdp = jdp+skip; jdlen = jdlen-skip; + } + + // copy the rest of jTemplate to jdp + skip = jTemplate+jtlen-jtp; + + // handling the boundary case in which JS_DELIMITER hasn't been + // added by encode() + if (fin == 0 && dlen == 0) { + if (skip > 0) { + *jtp = JS_DELIMITER; + jtp = jtp+1; jdp = jdp+1; + skip--; + } + } + memcpy(jdp, jtp, skip); + return encCnt; + + } else { + log_warn("Unknown mode (%d) for encode2()", mode); + return 0; + } + + +} + +/* + * int decode(char *jData, char *dataBuf, + * unsigned int jdlen, unsigned int dlen, unsigned int dataBufSize) + * + * description: + * extract hex char from Javascript embedded with data (jData) + * and put the result in dataBuf + * function returns the number of hex char extracted from jData + * to dataBuf, or returns one of the error codes + * + * input: + * - jData[]: Javascript embedded with hex-encoded data + * - jdlen : size of jData + * - dlen : size of data to recover + * - dataBufSize : size of output data buffer (dataBuf) + * + * output: + * - dataBuf[] : output buffer for recovered data + * + * assumptions: + * - data is hex-encoded + * + * exceptions: + * - if (dlen > dataBufSize) return INVALID_BUF_SIZE + * + * example: + * jData = "01p_or2=M3th.r4n5om()*6789ABCDEF0000000; dfp_tile = 1;" + * jdlen = 54 + * dlen = 16 + * dataBufSize = 1000 + * decode() returns 16 + * dataBuf= "0123456789ABCDEF" + * + */ +int decode (char *jData, char *dataBuf, unsigned int jdlen, + unsigned int dlen, unsigned int dataBufSize ) +{ + unsigned int decCnt = 0; /* num of data decoded */ + char *dp, *jdp; /* current pointers for dataBuf and jData */ + unsigned int j; + + if (dlen > dataBufSize) { return INVALID_BUF_SIZE; } + + dp = dataBuf; jdp = jData; + for (j=0; j<jdlen; j++) { + if ( isxdigit(*jdp) ) { + if (decCnt < dlen) { + decCnt++; + *dp++ = *jdp++; + } else { + break; + } + } else { + jdp++; + } + } + return decCnt; +} + + +/* + * decode2() is similar to decode(), but uses offset2Hex to look for + * applicable hex char in JS for decoding. Also, the decoding process + * stops when JS_DELIMITER is encountered. + */ +int decode2 (char *jData, char *dataBuf, unsigned int jdlen, + unsigned int dataBufSize, int *fin ) +{ + unsigned int decCnt = 0; /* num of data decoded */ + char *dp, *jdp; /* current pointers for dataBuf and jData */ + int i,j; + int cjdlen = jdlen; + + *fin = 0; + dp = dataBuf; jdp = jData; + + i = offset2Hex(jdp, cjdlen, 0); + while (i != -1) { + // return if JS_DELIMITER exists between jdp and jdp+i + for (j=0; j<i; j++) { + if (*jdp == JS_DELIMITER) { + *fin = 1; + return decCnt; + } + jdp = jdp+1; cjdlen--; + } + // copy hex data from jdp to dp + if (dataBufSize <= 0) { + return decCnt; + } + *dp = *jdp; + jdp = jdp+1; cjdlen--; + dp = dp+1; dataBufSize--; + decCnt++; + + // find the next hex char + i = offset2Hex(jdp, cjdlen, 1); + } + + // look for JS_DELIMITER between jdp to jData+jdlen + while (jdp < jData+jdlen) { + if (*jdp == JS_DELIMITER) { + *fin = 1; + break; + } + jdp = jdp+1; + } + + return decCnt; +} + + +int decodeHTTPBody (char *jData, char *dataBuf, unsigned int jdlen, + unsigned int dataBufSize, int *fin, int mode ) +{ + char *jsStart, *jsEnd; + char *dp, *jdp; // current pointers for data and jData + int scriptLen; + int decCnt = 0; + int n; + int dlen = dataBufSize; + dp = dataBuf; jdp = jData; + + if (mode == CONTENT_JAVASCRIPT) { + decCnt = decode2(jData, dataBuf, jdlen, dataBufSize, fin); + if (*fin == 0) { + log_warn("Unable to find JS_DELIMITER"); + } + } + else if (mode == CONTENT_HTML_JAVASCRIPT) { + *fin = 0; + while (*fin == 0) { + jsStart = strstr(jdp, startScriptTypeJS); + if (jsStart == NULL) { +#ifdef DEBUG + printf("Can't find startScriptType for decoding data inside script type JS\n"); +#endif + return decCnt; + } + jdp = jsStart+strlen(startScriptTypeJS); + jsEnd = strstr(jdp, endScriptTypeJS); + if (jsEnd == NULL) { +#ifdef DEBUG + printf("Can't find endScriptType for decoding data inside script type JS\n"); +#endif + return decCnt; + } + + // the JS for decoding data is between jsStart and jsEnd + scriptLen = jsEnd - jdp; + n = decode2(jdp, dp, scriptLen, dlen, fin); + if (n > 0) { + decCnt = decCnt+n; dlen=dlen-n; dp=dp+n; + } + jdp = jsEnd+strlen(endScriptTypeJS); + } // while (*fin==0) + } else { + log_warn("Unknown mode (%d) for encode2()", mode); + return 0; + } + + return decCnt; +} + + + + + +void printerr(int errno) { + if (errno == INVALID_BUF_SIZE) { + printf ("Error: Output buffer too small\n"); + } + else if (errno == INVALID_DATA_CHAR) { + printf ("Error: Non-hex char in data\n"); + } + else { + printf ("Unknown error: %i\n", errno); + } +} + + +int testEncode(char *data, char *js, char *outBuf, unsigned int dlen, unsigned int jslen, + unsigned int outBufLen, int testNum) { + int r; + + printf ("***** Start of testEncode (%i) *****\n", testNum); + printf ("Input:\n"); + printf ("data = %s\n", data); + printf ("data len = %i\n", dlen); + printf ("js = %s\n", js); + printf ("js len = %i\n", jslen); + r = encode (data, js, outBuf, dlen, jslen, outBufLen); + if (r < 0) { + printerr(r); + } else { + printf ("\nOutput:\n"); + printf ("%i char of data embedded in outBuf\n", r); + outBuf[jslen] = '\0'; + printf ("outBuf = %s\n", outBuf); + } + printf ("***** End of testEncode (%i) *****\n", testNum); + return r; +} + +int testDecode(char *inBuf, char *outBuf, unsigned int inBufSize, unsigned int dlen, + unsigned int outBufSize, int testNum) { + + int r; + + printf ("***** Start of testDecode (%i) *****\n", testNum); + printf ("Input:\n"); + printf ("inBuf = %s\n", inBuf); + printf ("inBuf size = %i\n", inBufSize); + printf ("data len = %i\n", dlen); + printf ("outBuf size = %i\n", outBufSize); + r = decode(inBuf, outBuf, inBufSize, dlen, outBufSize); + if (r < 0) { + printerr(r); + } else { + printf ("\nOutput:\n"); + printf ("%i char of data recovered from inBuf (to outBuf)\n", r); + outBuf[r] = '\0'; + printf ("outBuf = %s\n", outBuf); + } + printf ("***** End of testDecode (%i) *****\n", testNum); + return r; +} + + +int testEncode2(char *data, char *js, char *outBuf, + unsigned int dlen, unsigned int jslen, unsigned int outBufLen, + int mode, int testNum) { + int r; + // int fin; + + printf ("***** Start of testEncode2 (%i) *****\n", testNum); + printf ("Input:\n"); + printf ("data = %s\n", data); + printf ("data len = %i\n", dlen); + printf ("js = %s\n", js); + printf ("js len = %i\n", jslen); + // r = encode2(data, js, outBuf, dlen, jslen, outBufLen, &fin); + r = encodeHTTPBody(data, js, outBuf, dlen, jslen, outBufLen, mode); + + if (r < 0) { + printerr(r); + } + else { + printf ("\nOutput:\n"); + printf ("%i char of data embedded in outBuf\n", r); + // printf ("fin = %d\n", fin); + outBuf[jslen] = '\0'; + printf ("outBuf = %s\n", outBuf); + + if ((unsigned int) r < dlen) { + printf ("Incomplete data encoding\n"); + } + } + printf ("***** End of testEncode (%i) *****\n", testNum); + return r; +} + + + + +int testDecode2(char *inBuf, char *outBuf, + unsigned int inBufSize, unsigned int outBufSize, + int mode, int testNum) { + int r; + int fin; + + printf ("***** Start of testDecode2 (%i) *****\n", testNum); + printf ("Input:\n"); + printf ("inBuf = %s\n", inBuf); + printf ("inBuf size = %i\n", inBufSize); + printf ("outBuf size = %i\n", outBufSize); + r = decodeHTTPBody(inBuf, outBuf, inBufSize, outBufSize, &fin, mode); + if (r < 0) { + printerr(r); + } else { + printf ("\nOutput:\n"); + printf ("%i char of data recovered from inBuf (to outBuf)\n", r); + outBuf[r] = '\0'; + printf ("outBuf = %s\n", outBuf); + } + printf ("***** End of testDecode2 (%i) *****\n", testNum); + return r; +} + + +int +x_http2_server_JS_transmit (steg_t* s, struct evbuffer *source, conn_t *conn) { + + struct evbuffer_iovec *iv; + int nv; + struct evbuffer *dest = conn_get_outbound(conn); + size_t sbuflen = evbuffer_get_length(source); + char outbuf[HTTP_MSG_BUF_SIZE]; + char data[(int) sbuflen*2]; + unsigned int datalen; + + size_t sofar = 0; + unsigned int cnt = 0; + int r; + + + + // by dynamically configuring the size returned by x_http2_transmit_room(), + // we can ensure that the size of data (source) is less than the max + // data length the JavaScript (jsTemplate) can encode + + + + // do { + int i; + unsigned int jsLen; + int mode; + char *hend; + unsigned int hLen; + unsigned int mjs; + + char *jsTemplate = NULL; + int jsTemplateSize = 0; + + + + + /* int hdrLen; + int content_len; + int fin2; + + char* tmp; + char data2[20000]; + int decCnt; + */ + + datalen = 0; + + + log_debug("sbuflen = %d sofar = %d\n", (int) sbuflen, (int) sofar); + + // log_debug("SERVER: dumping data with length %d:", (int) sbuflen); + // evbuffer_dump(source, stderr); + // Convert data in 'source' to hexadecimal and write it to data + + nv = evbuffer_peek(source, sbuflen - sofar, NULL, NULL, 0); + iv = xzalloc(sizeof(struct evbuffer_iovec) * nv); + + if (evbuffer_peek(source, sbuflen - sofar, NULL, iv, nv) != nv) { + free(iv); + return -1; + } + + // jsTemplate should be init already, by x_http2_new or the previous invocation + // of this function + + mjs = get_max_JS_capacity(); + + if (mjs <= 0) { + log_debug("SERVER ERROR: (server_transmit) No JavaScript found in jsTemplate\n"); + return -1; + } + + if (sbuflen > (size_t) mjs) { + log_debug("SERVER ERROR: (server_transmit) jsTemplate cannot accommodate data %d %dn", + (int) sbuflen, (int) mjs); + return -1; + } + + + cnt = 0; + + for (i = 0; i < nv; i++) { + const unsigned char *p = iv[i].iov_base; + const unsigned char *limit = p + iv[i].iov_len; + char c; + + while (p < limit && cnt < sbuflen-sofar) { + c = *p++; + data[datalen] = "0123456789abcdef"[(c & 0xF0) >> 4]; + data[datalen+1] = "0123456789abcdef"[(c & 0x0F) >> 0]; + datalen += 2; + cnt++; + } + } + + log_debug("SERVER encoded data in hex string (len %d):", datalen); + // buf_dump((unsigned char*)data, datalen, stderr); + + free(iv); + + + + if (get_payload(HTTP_CONTENT_JAVASCRIPT, datalen, &jsTemplate, &jsTemplateSize) == 1) { + log_debug("SERVER found the next HTTP response template with size %d", jsTemplateSize); + } else { + log_debug("SERVER couldn't find the next HTTP response template; reusing the previous one"); + } + + log_debug("MJS %d %d", datalen, mjs); + + + + if (jsTemplate == NULL) { + fprintf(stderr, "NO suitable payload found %d %d\n", datalen, mjs); + exit(-1); + } + + + // use encodeHTTPBody to embed data in the JS jsTemplate + // assumption: jsTemplate is null-terminated + + log_debug("strlen(jsTemplate) = %d", (int)strlen(jsTemplate)); + log_debug("jsTemplateSize = %d", jsTemplateSize); + // unsigned int jsLen = strlen(jsTemplate); + jsLen = jsTemplateSize; + + + mode = has_eligible_HTTP_content (jsTemplate, jsLen, HTTP_CONTENT_JAVASCRIPT); + hend = strstr(jsTemplate, "\r\n\r\n"); + if (hend == NULL) { + log_warn("Unable to find end of header in the HTTP template"); + return -1; + } + + log_debug("SERVER: using HTTP resp template of length = %d\n", jsLen); + // log_debug("HTTP resp tempmlate:"); + // buf_dump((unsigned char*)jsTemplate, jsLen, stderr); + // fprintf(stderr, "==========================\n"); + + hLen = hend+4-jsTemplate; + r = encodeHTTPBody(data, hend+4, outbuf, datalen, jsLen-hLen, HTTP_MSG_BUF_SIZE, mode); + + + + + /// NEW STUFF + + +/* hdrLen = strstr(jsTemplate, "\r\n\r\n") - jsTemplate + 4; + tmp = strstr(jsTemplate, "Content-Length: ") + strlen("Content-Length: "); + + content_len = atoi(tmp); + + + decCnt = decodeHTTPBody(jsTemplate + hdrLen, data2, content_len, HTTP_MSG_BUF_SIZE, &fin2, mode); + + + if (decCnt == (int) datalen) + fprintf(stderr, "cnts match\n"); + else + fprintf(stderr, "cnts don't match %d %d\n", decCnt, datalen); + +*/ + + + if (r < 0 || ((unsigned int) r < datalen)) { + fprintf(stderr, "incomplete data encoding\n"); + exit(-1); + log_debug("SERVER ERROR: Incomplete data encoding"); + return -1; + } + + // note: the transformation is length-preserving for now + log_debug("SERVER: HTTP body with encoded data:"); + // buf_dump((unsigned char*)outbuf, jsLen-hLen, stderr); + // fprintf(stderr, "==========================\n"); + + if (evbuffer_add(dest, jsTemplate, hLen)) { + log_debug("SERVER ERROR: x_http2_server_transmit: evbuffer_add() fails for jsTemplate"); + return -1; + } + + // fprintf(stderr, "HELLO ==========================\n"); + + if (evbuffer_add(dest, outbuf, jsLen-hLen)) { + log_debug("SERVER ERROR: x_http2_server_transmit: evbuffer_add() fails for outbuf"); + return -1; + } + + sofar += datalen/2; + evbuffer_drain(source, datalen/2); + // } while (sbuflen > sofar); + + + + // fprintf(stderr, "SERVER TRANSMITTED payload of size %d\n", (int) sbuflen); + + // obtain a usable HTTP response template for the next data, and + // modify jsTemplateCapacity + + + log_debug("SERVER finding the next HTTP response template"); + + + + + + // conn_cease_transmission(conn); + conn_close_after_transmit(conn); + // downcast_steg(s)->have_transmitted = 1; + return 0; +} + + + + + + +int +x_http2_handle_client_JS_receive(steg_t *s, conn_t *conn, struct evbuffer *dest, struct evbuffer* source) { + struct evbuffer_ptr s2; + unsigned int response_len = 0; + unsigned int content_len = 0; + unsigned int hdrLen; + char buf[10]; + char respMsg[HTTP_MSG_BUF_SIZE]; + char data[HTTP_MSG_BUF_SIZE]; + + unsigned char *field; + unsigned char *fieldStart; + unsigned char* fieldEnd; + unsigned char *fieldValStart; + char *httpBody; + + ev_ssize_t r; + int mode, decCnt, fin; + struct evbuffer * scratch; + int i,j,k; + char c; + + + s2 = evbuffer_search(source, "\r\n\r\n", sizeof ("\r\n\r\n") -1 , NULL); + if (s2.pos == -1) { + log_debug("CLIENT Did not find end of HTTP header %d", (int) evbuffer_get_length(source)); + // evbuffer_dump(source, stderr); + return RECV_INCOMPLETE; + } + + log_debug("CLIENT received response header with len %d", (int)s2.pos); + + response_len = 0; + hdrLen = s2.pos + strlen("\r\n\r\n"); + response_len += hdrLen; + + // get content length, e.g., Content-Length: 22417 + field = evbuffer_pullup(source, s2.pos); + if (field == NULL) { + log_debug("CLIENT unable to pullup the complete HTTP header"); + return RECV_BAD; + } + + fieldStart = (unsigned char*) strstr((char*) field, "Content-Length: "); + if (fieldStart == NULL) { + log_debug("CLIENT unable to find Content-Length in the header"); + return RECV_BAD; + } + + fieldEnd = (unsigned char*) strstr((char *)fieldStart, "\r\n"); + if (fieldEnd == NULL) { + log_debug("CLIENT unable to find end of line for Content-Length"); + return RECV_BAD; + } + + fieldValStart = fieldStart+strlen("Content-Length: "); + if ((unsigned int) (fieldEnd-fieldValStart) > (sizeof(buf)-1)) { + log_debug("CLIENT: Value of Content-Length too large"); + return RECV_BAD; + } + memcpy(buf, fieldValStart, fieldEnd-fieldValStart); + buf[fieldEnd-fieldValStart] = 0; + + content_len = atoi(buf); + log_debug("CLIENT received Content-Length = %d\n", content_len); + + response_len += content_len; + + if (response_len > evbuffer_get_length(source)) + return RECV_INCOMPLETE; + + + + // read the entire HTTP resp + if (response_len < HTTP_MSG_BUF_SIZE) { + r = evbuffer_copyout(source, respMsg, response_len); + log_debug("CLIENT %d char copied from source to respMsg (expected %d)", (int)r, response_len); + if (r < 0) { + log_debug("CLIENT ERROR: evbuffer_copyout fails"); + return RECV_INCOMPLETE; + } + if (r < response_len) { + log_debug("CLIENT: evbuffer_copyout incomplete; got %d instead of %d", (int)r, response_len); + return RECV_INCOMPLETE; + } + respMsg[response_len] = 0; + } else { + log_debug("CLIENT: HTTP response too large to handle"); + return RECV_BAD; + } + + log_debug("CLIENT received HTTP response with length %d\n", response_len); + log_debug("HTTP response:"); + // buf_dump((unsigned char*)respMsg, response_len, stderr); + // fprintf(stderr, "==========================\n"); + + httpBody = respMsg + hdrLen; + + log_debug("CLIENT Before has_eligible HTTP content\n"); + // find out if the resp is a JavaScript or JS embedded in HTML doc + mode = has_eligible_HTTP_content (respMsg, response_len, HTTP_CONTENT_JAVASCRIPT); + if (mode != 1 && mode != 2) { + log_debug("CLIENT ERROR: HTTP response not useful for jsSteg (mode %d)", mode); + return RECV_BAD; + } + + log_debug("CLIENT Before decodeHTTPBody; mode: %d\n", mode); + + // call decodeHTTPBody + decCnt = decodeHTTPBody(httpBody, data, response_len-hdrLen, HTTP_MSG_BUF_SIZE, &fin, mode); + data[decCnt] = 0; + + log_debug("After decodeHTTPBody; decCnt: %d\n", decCnt); + + // decCnt is an odd number or data is not a hex string + if (decCnt % 2) { + log_debug("CLIENT ERROR: An odd number of hex characters received"); + // buf_dump((unsigned char*)data, decCnt, stderr); + return RECV_BAD; + } + + if (! isxString(data)) { + log_debug("CLIENT ERROR: Data received not hex"); + // buf_dump((unsigned char*)data, decCnt, stderr); + return RECV_BAD; + } + + log_debug("Hex data received:"); + // buf_dump ((unsigned char*)data, decCnt, stderr); + + // get a scratch buffer + scratch = evbuffer_new(); + if (!scratch) return RECV_BAD; + + if (evbuffer_expand(scratch, decCnt/2)) { + log_debug("CLIENT ERROR: Evbuffer expand failed \n"); + evbuffer_free(scratch); + return RECV_BAD; + } + + // convert hex data back to binary + for (i=0, j=0; i< decCnt; i=i+2, ++j) { + sscanf(&data[i], "%2x", (unsigned int*) &k); + c = (char)k; + evbuffer_add(scratch, &c, 1); + } + + log_debug("CLIENT Done converting hex data to binary:\n"); + // evbuffer_dump(scratch, stderr); + + + // fprintf(stderr, "CLIENT RECEIVED payload of size %d\n", (int) evbuffer_get_length(scratch)); + // add the scratch buffer (which contains the data) to dest + + if (evbuffer_add_buffer(dest, scratch)) { + evbuffer_free(scratch); + log_debug("CLIENT ERROR: Failed to transfer buffer"); + return RECV_BAD; + } + log_debug("Added scratch (buffer) to dest\n"); + + evbuffer_free(scratch); + + + if (response_len <= evbuffer_get_length(source)) { + if (evbuffer_drain(source, response_len) == -1) { + log_debug("CLIENT ERROR: Added scratch (buffer) to dest\n"); + return RECV_BAD; + } + } + else { + log_warn("response_len > buffer size... can't drain"); + exit(-1); + } + + + log_debug("Drained source for %d char\n", response_len); + + // downcast_steg(s)->have_received = 1; + conn_expect_close(conn); + + return RECV_GOOD; +} + + + +//int +//x_http2_handle_server_JS_receive(steg_t *s, conn_t *conn, struct evbuffer *dest, struct evbuffer* source) { +// +// int cnt = 0; +// +// do { +// struct evbuffer_ptr s2 = evbuffer_search(source, "\r\n\r\n", sizeof ("\r\n\r\n") -1 , NULL); +// unsigned char* data; +// unsigned char* limit; +// unsigned char *p; +// int unwrapped_cookie_len; +// struct evbuffer *scratch; +// unsigned char c, h, secondhalf; +// unsigned char buf[evbuffer_get_length(source)]; +// +// +// if (s2.pos == -1) { +// log_debug("Did not find end of request %d", (int) evbuffer_get_length(source)); +// // evbuffer_dump(source, stderr); +// return RECV_INCOMPLETE; +// } +// +// log_debug("SERVER received request header of length %d", (int)s2.pos); +// +// data = evbuffer_pullup(source, s2.pos); +// if (data == NULL) { +// log_debug("SERVER evbuffer_pullup fails"); +// return RECV_BAD; +// } +// +// limit = data + s2.pos; +// +// data = (unsigned char*) strstr((char*) data, "Cookie:"); +// +// if (data == NULL || memcmp(data, "Cookie:", sizeof "Cookie:"-1)) { +// log_debug("Unexpected HTTP verb: %.*s", 5, data); +// return RECV_BAD; +// } +// +// p = data + sizeof "Cookie: "-1; +// unwrapped_cookie_len = unwrap_cookie(p, buf, (int) (limit - p)); +// +// log_debug("SERVER: received cookie of length = %d %d\n", unwrapped_cookie_len, (int) (limit-p)); +// // buf_dump(buf, unwrapped_cookie_len, stderr); +// // fprintf(stderr, "==========================\n"); +// // buf_dump(p, (int) (limit-p), stderr); +// +// +// // log_debug("hello SERVER received %d cnt = %d\n", (int) (limit - p), cnt); +// // buf_dump(p, (int) (limit-p), stderr); +// +// /* We need a scratch buffer here because the contract is that if +// we hit a decode error we *don't* write anything to 'dest'. */ +// scratch = evbuffer_new(); +// +// if (!scratch) return RECV_BAD; +// +// +// if (evbuffer_expand(scratch, unwrapped_cookie_len/2)) { +// log_debug("Evbuffer expand failed \n"); +// evbuffer_free(scratch); +// return RECV_BAD; +// } +// p = buf; +// +// +// secondhalf = 0; +// while ((int) (p - buf) < unwrapped_cookie_len) { +// if (!secondhalf) c = 0; +// if ('0' <= *p && *p <= '9') h = *p - '0'; +// else if ('a' <= *p && *p <= 'f') h = *p - 'a' + 10; +// else if ('A' <= *p && *p <= 'F') h = *p - 'A' + 10; +// else if (*p == '=' && !secondhalf) { +// p++; +// continue; +// } else { +// evbuffer_free(scratch); +// log_debug("Decode error: unexpected URI characterasdfaf %d", *p); +// return RECV_BAD; +// } +// +// c = (c << 4) + h; +// if (secondhalf) { +// evbuffer_add(scratch, &c, 1); +// // log_debug("adding to scratch"); +// cnt++; +// } +// secondhalf = !secondhalf; +// p++; +// } +// +// +// +// if (evbuffer_add_buffer(dest, scratch)) { +// evbuffer_free(scratch); +// log_debug("Failed to transfer buffer"); +// return RECV_BAD; +// } +// evbuffer_drain(source, s2.pos + sizeof("\r\n\r\n") - 1); +// evbuffer_free(scratch); +// } while (evbuffer_get_length(source)); +// +// +// log_debug("SERVER RECEIVED payload %d\n", cnt); +// // downcast_steg(s)->have_received = 1; +// conn_transmit_soon(conn, 100); +// return RECV_GOOD; +//} + + + + +/***** + int + main() { + int jDataSize = 1000; + char jData[jDataSize]; + int outDataBufSize = 1000; + char outDataBuf[outDataBufSize]; + + int r; + // test case 1: data embedded in javascript + r = testEncode2(data1, js1, jData, strlen(data1), strlen(js1), jDataSize, + CONTENT_JAVASCRIPT, 1); + if (r > 0) { testDecode2(jData, outDataBuf, strlen(js1), outDataBufSize, CONTENT_JAVASCRIPT, 1); } + + // test case 4: data embedded in one script type javascript + r = testEncode2(data1, js4, jData, strlen(data1), strlen(js4), jDataSize, + CONTENT_HTML_JAVASCRIPT, 4); + if (r > 0) { testDecode2(jData, outDataBuf, strlen(js4), outDataBufSize, CONTENT_HTML_JAVASCRIPT, 4); } + + // test case 5: data embedded in one script type javascript + r = testEncode2(data1, js5, jData, strlen(data1), strlen(js5), jDataSize, + CONTENT_HTML_JAVASCRIPT, 5); + if (r > 0) { testDecode2(jData, outDataBuf, strlen(js5), outDataBufSize, CONTENT_HTML_JAVASCRIPT, 5); } + + + return 0; + } +*****/ + +/***** + int + main() { + int jDataSize = 1000; + char jData[jDataSize]; + int jDataSmallSize = 5; + char jDataSmall[jDataSmallSize]; + + int outDataBufSize = 1000; + char outDataBuf[outDataBufSize]; + int outDataSmallSize = 5; + char outDataSmall[outDataSmallSize]; + + int r; + + // test case 1: data embedded in javascript + r = testEncode(data1, js1, jData, strlen(data1), strlen(js1), jDataSize, 1); + if (r > 0) { testDecode(jData, outDataBuf, strlen(js1), r, outDataBufSize, 1); } + + // test case 2: data embedded in javascript + r = testEncode(data1, js2, jData, strlen(data1), strlen(js2), jDataSize, 2); + if (r > 0) { testDecode(jData, outDataBuf, strlen(js2), r, outDataBufSize, 2); } + + // test case 3: data partially embedded in javascript; num of hex char in js < data len + r = testEncode(data1, js3, jData, strlen(data1), strlen(js3), jDataSize, 3); + if (r > 0) { testDecode(jData, outDataBuf, strlen(js3), r, outDataBufSize, 3); } + + // test case 4: data embedded in javascript; larger data + r = testEncode(data2, js1, jData, strlen(data2), strlen(js1), jDataSize, 4); + if (r > 0) { testDecode(jData, outDataBuf, strlen(js1), r, outDataBufSize, 4); } + + // test case 5 (for encode): err for non-hex data + testEncode(nonhexstr, js1, jData, strlen(nonhexstr), strlen(js1), jDataSize, 5); + + // test case 6 (for encode): err for small output buf + testEncode(data1, js1, jDataSmall, strlen(data1), strlen(js1), jDataSmallSize, 6); + + // test case 7 (for decode): err for small output buf + r = testEncode(data1, js1, jData, strlen(data1), strlen(js1), jDataSize, 7); + if (r > 0) { testDecode(jData, outDataSmall, strlen(js1), r, outDataSmallSize, 7); } + } +*****/ + diff --git a/src/steg/jsSteg.h b/src/steg/jsSteg.h new file mode 100644 index 0000000..3c5f6ae --- /dev/null +++ b/src/steg/jsSteg.h @@ -0,0 +1,65 @@ +#ifndef _JSSTEG_H +#define _JSSTEG_H + + +#include <stdio.h> +#include <string.h> +#include <ctype.h> +#include "util.h" +#include "connections.h" +#include "steg.h" +#include <event2/buffer.h> + + +/* error codes */ +#define INVALID_BUF_SIZE -1 +#define INVALID_DATA_CHAR -2 + + +int encodeHTTPBody(char *data, char *jTemplate, char *jData,unsigned int dlen, + unsigned int jtlen, unsigned int jdlen, int mode); +int isxString(char *str); +int decodeHTTPBody (char *jData, char *dataBuf, unsigned int jdlen, + unsigned int dataBufSize, int *fin, int mode); + +int encode(char *data, char *jTemplate, char *jData, + unsigned int dlen, unsigned int jtlen, unsigned int jdlen ); + +int encode2(char *data, char *jTemplate, char *jData, + unsigned int dlen, unsigned int jtlen, + unsigned int jdlen, int *fin); + +int decode (char *jData, char *dataBuf, unsigned int jdlen, + unsigned int dlen, unsigned int dataBufSize ); + +int decode2 (char *jData, char *dataBuf, unsigned int jdlen, + unsigned int dataBufSize, int *fin ); + +void printerr(int errno); + +int testEncode(char *data, char *js, char *outBuf, unsigned int dlen, unsigned int jslen, + unsigned int outBufLen, int testNum); + +int testDecode(char *inBuf, char *outBuf, unsigned int inBufSize, unsigned int dlen, + unsigned int outBufSize, int testNum); + +int testEncode2(char *data, char *js, char *outBuf, + unsigned int dlen, unsigned int jslen, unsigned int outBufLen, + int mode, int testNum); + +int testDecode2(char *inBuf, char *outBuf, + unsigned int inBufSize, unsigned int outBufSize, + int mode, int testNum); + + +int +x_http2_server_JS_transmit (steg_t* s, struct evbuffer *source, conn_t *conn); + +int +x_http2_handle_client_JS_receive(steg_t *s, conn_t *conn, struct evbuffer *dest, struct evbuffer* source); + + + +#endif + + diff --git a/src/steg/payloads.c b/src/steg/payloads.c new file mode 100644 index 0000000..2782aa6 --- /dev/null +++ b/src/steg/payloads.c @@ -0,0 +1,1486 @@ +#include "util.h" +#include "payloads.h" +#include "swfSteg.h" + + +/* These variables below are write-once, hence they should be race-safe */ + +static int initTypePayload[MAX_CONTENT_TYPE]; +static int typePayloadCount[MAX_CONTENT_TYPE]; +static int typePayload[MAX_CONTENT_TYPE][MAX_PAYLOADS]; +static int typePayloadCap[MAX_CONTENT_TYPE][MAX_PAYLOADS]; + + +static unsigned int max_JS_capacity = 0; +static unsigned int max_PDF_capacity = 0; + + + +pentry_header payload_hdrs[MAX_PAYLOADS]; +char* payloads[MAX_PAYLOADS]; +int payload_count = 0; + + +unsigned int get_max_JS_capacity() { + return max_JS_capacity; +} + +unsigned int get_max_PDF_capacity() { + return max_PDF_capacity; +} + + + +/* + * fixContentLen corrects the Content-Length for an HTTP msg that + * has been ungzipped, and removes the "Content-Encoding: gzip" + * field from the header. + * + * The function returns -1 if no change to the HTTP msg has been made, + * when the msg wasn't gzipped or an error has been encountered + * If fixContentLen changes the msg header, it will put the new HTTP + * msg in buf and returns the length of the new msg + * + * Input: + * payload - pointer to the (input) HTTP msg + * payloadLen - length of the (input) HTTP msg + * + * Ouptut: + * buf - pointer to the buffer containing the new HTTP msg + * bufLen - length of buf + * + */ +int fixContentLen (char* payload, int payloadLen, char *buf, int bufLen) { + + int gzipFlag=0, clFlag=0, clZeroFlag=0; + char* ptr = payload; + char* clPtr = payload; + char* gzipPtr = payload; + char* end; + + + char *cp, *clEndPtr; + int hdrLen, bodyLen, r, len; + + + + + + // note that the ordering between the Content-Length and the Content-Encoding + // in an HTTP msg may be different for different msg + + // if payloadLen is larger than the size of our buffer, + // stop and return -1 + if (payloadLen > bufLen) { return -1; } + + while (1) { + end = strstr(ptr, "\r\n"); + if (end == NULL) { + // log_debug("invalid header %d %d %s \n", payloadLen, (int) (ptr - payload), payload); + return -1; + } + + if (!strncmp(ptr, "Content-Encoding: gzip\r\n", 24)) { + gzipFlag = 1; + gzipPtr = ptr; + } else if (!strncmp(ptr, "Content-Length: 0", 17)) { + clZeroFlag = 1; + } else if (!strncmp(ptr, "Content-Length:", 15)) { + clFlag = 1; + clPtr = ptr; + } + + if (!strncmp(end, "\r\n\r\n", 4)){ + break; + } + ptr = end+2; + } + + // stop if zero Content-Length or Content-Length not found + if (clZeroFlag || ! clFlag) return -1; + + // end now points to the end of the header, before "\r\n\r\n" + cp=buf; + bodyLen = (int)(payloadLen - (end+4-payload)); + + clEndPtr = strstr(clPtr, "\r\n"); + if (clEndPtr == NULL) { + log_debug("unable to find end of line for Content-Length"); + return -1; + } + if (gzipFlag && clFlag) { + if (gzipPtr < clPtr) { // Content-Encoding appears before Content-Length + + // copy the part of the header before Content-Encoding + len = (int)(gzipPtr-payload); + memcpy(cp, payload, len); + cp = cp+len; + + // copy the part of the header between Content-Encoding and Content-Length + // skip 24 char, the len of "Content-Encoding: gzip\r\n" + // *** this is temporary; we'll remove this after the obfsproxy can perform gzip + len = (int)(clPtr-(gzipPtr+24)); + memcpy(cp, gzipPtr+24, len); + cp = cp+len; + + // put the new Content-Length + memcpy(cp, "Content-Length: ", 16); + cp = cp+16; + r = sprintf(cp, "%d\r\n", bodyLen); + if (r < 0) { + log_debug("sprintf fails"); + return -1; + } + cp = cp+r; + + // copy the part of the header after Content-Length, if any + if (clEndPtr != end) { // there is header info after Content-Length + len = (int)(end-(clEndPtr+2)); + memcpy(cp, clEndPtr+2, len); + cp = cp+len; + memcpy(cp, "\r\n\r\n", 4); + cp = cp+4; + } else { // Content-Length is the last hdr field + memcpy(cp, "\r\n", 2); + cp = cp+2; + } + + hdrLen = cp-buf; + +/**** +log_debug("orig: hdrLen = %d, bodyLen = %d, payloadLen = %d", (int)(end+4-payload), bodyLen, payloadLen); +log_debug("new: hdrLen = %d, bodyLen = %d, payloadLen = %d", hdrLen, bodyLen, hdrLen+bodyLen); + ****/ + + // copy the HTTP body + memcpy(cp, end+4, bodyLen); + return (hdrLen+bodyLen); + + } else { // Content-Length before Content-Encoding + // copy the part of the header before Content-Length + len = (int)(clPtr-payload); + memcpy(cp, payload, len); + cp = cp+len; + + // put the new Content-Length + memcpy(cp, "Content-Length: ", 16); + cp = cp+16; + r = sprintf(cp, "%d\r\n", bodyLen); + if (r < 0) { + log_debug("sprintf fails"); + return -1; + } + cp = cp+r; + + // copy the part of the header between Content-Length and Content-Encoding + len = (int)(gzipPtr-(clEndPtr+2)); + memcpy(cp, clEndPtr+2, len); + cp = cp+len; + + // copy the part of the header after Content-Encoding + // skip 24 char, the len of "Content-Encoding: gzip\r\n" + // *** this is temporary; we'll remove this after the obfsproxy can perform gzip + if (end > (gzipPtr+24)) { // there is header info after Content-Encoding + len = (int)(end-(gzipPtr+24)); + memcpy(cp, gzipPtr+24, len); + cp = cp+len; + memcpy(cp, "\r\n\r\n", 4); + cp = cp+4; + } else { // Content-Encoding is the last field in the hdr + memcpy(cp, "\r\n", 2); + cp = cp+2; + } + hdrLen = cp-buf; + +/**** +log_debug("orig: hdrLen = %d, bodyLen = %d, payloadLen = %d", (int)(end+4-payload), bodyLen, payloadLen); +log_debug("new: hdrLen = %d, bodyLen = %d, payloadLen = %d", hdrLen, bodyLen, hdrLen+bodyLen); + ****/ + + // copy the HTTP body + memcpy(cp, end+4, bodyLen); + return (hdrLen+bodyLen); + } + } + return -1; +} + +void load_payloads(const char* fname) { + FILE* f; + char buf[HTTP_MSG_BUF_SIZE]; + char buf2[HTTP_MSG_BUF_SIZE]; + pentry_header pentry; + int pentryLen; + int r; + + if (payload_count != 0) + return; + + srand(time(NULL)); + f = fopen(fname, "r"); + if (f == NULL) { + fprintf(stderr, "Cannot open trace file %s. Exiting\n", fname); + exit(1); + } + + bzero(payload_hdrs, sizeof(payload_hdrs)); + + while (payload_count < MAX_PAYLOADS) { + + if (fread(&pentry, 1, sizeof(pentry_header), f) < sizeof(pentry_header)) { + break; + } + + pentryLen = ntohl(pentry.length); + if((unsigned int) pentryLen > sizeof(buf)) { +#ifdef DEBUG + // fprintf(stderr, "pentry too big %d %d\n", pentry.length, ntohl(pentry.length)); + fprintf(stderr, "pentry too big %d\n", pentryLen); +#endif + // skip to the next pentry + if (fseek(f, pentryLen, SEEK_CUR)) { + fprintf(stderr, "skipping to next pentry fails\n"); + } + continue; + // exit(0); + } + + pentry.length = pentryLen; + pentry.ptype = ntohs(pentry.ptype); + + if (fread(buf, 1, pentry.length, f) < (unsigned int) pentry.length) + break; + + // todo: + // fixed content length for gzip'd HTTP msg + // fixContentLen returns -1, if no change to the msg + // otherwise, it put the new HTTP msg (with hdr changed) in buf2 + // and returns the size of the new msg + + r = -1; + if (pentry.ptype == TYPE_HTTP_RESPONSE) { + r = fixContentLen (buf, pentry.length, buf2, HTTP_MSG_BUF_SIZE); + // log_debug("for payload_count %d, fixContentLen returns %d", payload_count, r); + } + // else { + // log_debug("for payload_count %d, pentry.ptype = %d", payload_count, pentry.ptype); + // } + + if (r < 0) { + payloads[payload_count] = malloc(pentry.length + 1); + memcpy(payloads[payload_count], buf, pentry.length); + } else { + pentry.length = r; + payloads[payload_count] = malloc(pentry.length + 1); + memcpy(payloads[payload_count], buf2, pentry.length); + } + payload_hdrs[payload_count] = pentry; + payloads[payload_count][pentry.length] = 0; + payload_count++; + } // while + +#ifdef DEBUG + printf("loading payload count = %d\n", payload_count); +#endif + + fclose(f); +} + + + + + +void gen_rfc_1123_date(char* buf, int buf_size) { + time_t t = time(NULL); + struct tm *my_tm = gmtime(&t); + strftime(buf, buf_size, "Date: %a, %d %b %Y %H:%M:%S GMT\r\n", my_tm); +} + + + +/* sample response header */ +/* + +HTTP/1.1 200 OK +Date: Fri, 21 Oct 2011 19:09:49 GMT +Server: Apache +Content-Length: 1441 +* Content-Encoding: gzip +Content-Type: text/html;charset=UTF-8 +* Keep-Alive: timeout=15, max=93 +Connection: close + +*/ + + +int gen_response_header(char* content_type, int gzip, int length, char* buf, int buflen) { + char* ptr; + + // conservative assumption here.... + if (buflen < 256) { + fprintf(stderr, "gen_response_header: buflen too small\n"); + return -1; + } + + sprintf(buf, "HTTP/1.1 200 OK\r\n"); + ptr = buf + strlen("HTTP/1.1 200 OK\r\n"); + gen_rfc_1123_date(ptr, buflen - (ptr - buf)); + ptr = ptr + strlen(ptr); + + if (gzip) + sprintf(ptr, "Server: Apache\r\nContent-Length: %d\r\nContent-Encoding: gzip\r\nContent-Type: %s\r\nConnection: close\r\n\r\n", length, content_type); + else + sprintf(ptr, "Server: Apache\r\nContent-Length: %d\r\nContent-Type: %s\r\nConnection: close\r\n\r\n", length, content_type); + + ptr += strlen(ptr); + + return ptr - buf; +} + + + + + + +int parse_client_headers(char* inbuf, char* outbuf, int len) { + // client-side + // remove Host: field + // remove referrer fields? + + char* ptr = inbuf; + int outlen = 0; + + while (1) { + // char* end = strstr(ptr, "\r\n", len - (ptr - inbuf)); + char* end = strstr(ptr, "\r\n"); + if (end == NULL) { + fprintf(stderr, "invalid client header %d %d %s \n PTR = %s\n", len, (int) (len - (ptr - inbuf)), inbuf, ptr); + // fprintf(stderr, "HERE %s\n", ptr); + break; + } + + if (!strncmp(ptr, "Host:", 5) || + !strncmp(ptr, "Referer:", 8) || + !strncmp(ptr, "Cookie:", 7)) { + goto next; + } + + memcpy(outbuf + outlen, ptr, end - ptr + 2); + outlen += end - ptr + 2; + + next: + if (!strncmp(end, "\r\n\r\n", 4)){ + break; + } + ptr = end+2; + } + + return outlen; + + // server-side + // fix date fields + // fix content-length + + + +} + + + + +/* first line is of the form.... + GET /XX/XXXX.swf[?YYYY] HTTP/1.1\r\n +*/ + + +int +find_uri_type(char* buf) { + + char* uri; + int uri_len; + char* ext; + + if (strncmp(buf, "GET", 3) != 0 && strncmp(buf, "POST", 4) != 0) + return -1; + + buf = strchr(buf, ' ') + 1; + uri_len = strchr(buf, ' ') - buf; + uri = malloc(uri_len + 1); + + strncpy(uri, buf, uri_len); + uri[uri_len] = 0; + + if (strchr(uri, '?')) + ext = strchr(uri, '?') - 4; + else + ext = uri + uri_len - 4; + + + if (!strncmp(ext, ".pdf", 4) || !strncmp(ext, ".PDF", 4)) + return HTTP_CONTENT_PDF; + + if (!strncmp(ext, ".swf", 4) || !strncmp(ext, ".SWF", 4)) + return HTTP_CONTENT_SWF; + + // if (!strncmp(ext, ".js", 3) || !strncmp(ext, ".JS", 3)) + return HTTP_CONTENT_JAVASCRIPT; + + if (!strncmp(ext-1, "html", 4) || !strncmp(ext, "htm", 3) || strchr(ext-1, '.') == NULL) + return HTTP_CONTENT_HTML; + + return -1; + +} + + + + + + + + + + + +unsigned int find_client_payload(char* buf, int len, int type) { + int r = rand() % payload_count; + int cnt = 0; + char* inbuf; + +#ifdef DEBUG + fprintf(stderr, "TRYING payload %d \n", r); +#endif + while (1) { + pentry_header* p = &payload_hdrs[r]; + if (p->ptype == type) { + inbuf = payloads[r]; + if (find_uri_type(inbuf) != HTTP_CONTENT_SWF && + find_uri_type(inbuf) != HTTP_CONTENT_JAVASCRIPT) + + goto next; + + if (p->length > len) { + fprintf(stderr, "BUFFER TOO SMALL... \n"); + goto next; + } + else + len = p->length; + break; + } + next: + r = (r+1) % payload_count; + + + // no matching payloads... + if (cnt++ == payload_count) { + fprintf(stderr, "NO MATCHING PAYLOADS... \n"); + return 0; + } + } + + inbuf[len] = 0; + + // clean up the buffer... + return parse_client_headers(inbuf, buf, len); + +} + + +/* + * skipJSPattern returns the number of characters to skip when + * the input pointer matches the start of a common JavaScript + * keyword + * + * todo: + * Use a more efficient algo (e.g., Aho-Corasick) in the next iteration + */ +int skipJSPattern (char *cp, int len) { + + // log_debug("Turning off skipJSPattern for debugging"); + return 0; + + if (len < 1) return 0; + + if (len > 8) { + // "function " and "function(" + if (cp[0] == 'f' && + !strncmp(cp+1, "un", 2) && + isxdigit(cp[3]) && + !strncmp(cp+4, "tion", 4) && + (cp[8] == ' ' || cp[8] == '(')) + return 9; + } + + if (len > 6) { + // "return " + if (cp[0] == 'r' && + isxdigit(cp[1]) && + !strncmp(cp+2, "turn ", 5)) + return 7; + // "switch " + if (cp[0] == 's' && + !strncmp(cp+1, "wit", 3) && + isxdigit(cp[4]) && + !strncmp(cp+5, "h ", 2)) + return 7; + } + + if (len > 5) { + // "while " and "while(" + if (cp[0] == 'w' && + !strncmp(cp+1, "hil", 3) && + isxdigit(cp[4]) && + (cp[5] == ' ' || cp[5] == '(')) + return 6; + } + + if (len > 4) { + // "else " and "else{" + if (cp[0] == 'e' && + !strncmp(cp, "ls", 2) && + isxdigit(cp[3]) && + (cp[4] == ' ' || cp[4] == '{')) + return 5; + } + + if (len > 3) { + // "var " + if (cp[0] == 'v' && + isxdigit(cp[1]) && + cp[2] == 'r' && + cp[3] == ' ') + return 4; + } + + return 0; +} + + +int isalnum_ (char c) { + if (isalnum(c) || c == '_') return 1; + else return 0; +} + +int offset2Alnum_ (char *p, int range) { + char *cp = p; + + while ((cp < (p+range)) && !isalnum_(*cp)) { + cp++; + } + + if (cp < (p+range)) { + return (cp-p); + } else { + return -1; + } +} + +/* + * offset2Hex returns the offset to the next usable hex char. + * usable here refer to char that our steg module can use to encode + * data. in particular, words that correspond to common JavaScript keywords + * are not used for data encoding (see skipJSPattern). Also, because + * JS var name must start with an underscore or a letter (but not a digit) + * we don't use the first char of a word for encoding data + * + * e.g., the JS statement "var a;" won't be used for encoding data + * because "var" is a common JS keyword and "a" is the first char of a word + * + * Input: + * p - ptr to the starting pos + * range - max number of char to look + * isLastCharHex - is the char pointed to by (p-1) a hex char + * + * Output: + * offset2Hex returns the offset to the next usable hex char + * between p and (p+range), if it exists; + * otherwise, it returns -1 + * + */ +int offset2Hex (char *p, int range, int isLastCharHex) { + char *cp = p; + int i,j; + int isFirstWordChar = 1; + + if (range < 1) return -1; + + // case 1: last char is hexadecimal + if (isLastCharHex) { + if (isxdigit(*cp)) return 0; // base case + else { + cp++; + i = offset2Alnum_(cp, p+range-cp); + while (i == 0) { + if (isxdigit(*cp)) return (cp-p); + cp++; + i = offset2Alnum_(cp, p+range-cp); + } + if (i == -1) return -1; + // if (i > 0), fallthru and handle case 2 + } + } + + // case 2: + // find the next word that starts with alnum or underscore, + // which could be a variable, keyword, or literal inside a string + + i = offset2Alnum_(cp, p+range-cp); + if (i == -1) return -1; + // isFirstWordChar = 1; + + while (cp < (p+range) && i != -1) { + + if (i == 0) { + j = skipJSPattern(cp, p+range-cp); + if (j > 0) { + // cp points a word that should not be used (e.g., JS keyword) + cp = cp+j; isFirstWordChar = 1; + } else { // j == 0 + if (isFirstWordChar) { + cp++; isFirstWordChar = 0; // skip the 1st char of a word + } + else { // we are in the middle of a word; no need to invoke skipJSPattern + if (isxdigit(*cp)) return (cp-p); + else cp++; + } + } + } else { // i > 0 + cp += i; isFirstWordChar = 1; + } + i = offset2Alnum_(cp, p+range-cp); + + } // while + + // cannot find next usable hex char + return -1; + +} + +/* + * capacityJS returns the number of usable hex char + * in the input HTTP message for data encoding + * + * Input: + * buf - pointer to HTTP message + * len - sizeof buf + * mode - + * CONTENT_JAVASCRIPT (case 1); HTTP message body is a JavaScript + * (e.g., Content-Type: {text/javascript, application/x-javascript, + * application/javascript}) + * CONTENT_HTML_JAVASCRIPT (case 2) + * Content-Type: text/html and HTTP message body contains + * <script type="text/javascript"> ... </script> + * + * Output: + * capacityJS returns the number of usable hex char that can be embedded + * in the HTTP message + * + * Note: + * This is a prototype for the simple version (all hex char are usable) + * will refine this to skip JavaScript keywords in the next iteration + * + */ + + + +unsigned int capacityJS (char* buf, int len, int mode) { + char *hEnd, *bp, *jsStart, *jsEnd; + int cnt=0; + + // jump to the beginning of the body of the HTTP message + hEnd = strstr(buf, "\r\n\r\n"); + if (hEnd == NULL) { + // cannot find the separator between HTTP header and HTTP body + return 0; + } + bp = hEnd + 4; + + if (mode == CONTENT_JAVASCRIPT) { + while (bp < (buf+len)) { + if (isxdigit(*bp)) { + cnt++; +#ifdef DEBUG + printf("%c", *bp); +#endif + } + bp++; + } +#ifdef DEBUG + printf("\n"); +#endif + return cnt; + } else if (mode == CONTENT_HTML_JAVASCRIPT) { + while (bp < (buf+len)) { + jsStart = strstr(bp, "<script type=\"text/javascript\">"); + if (jsStart == NULL) break; + bp = jsStart+31; + jsEnd = strstr(bp, "</script>"); + if (jsEnd == NULL) break; + // count the number of usable hex char between jsStart+31 and jsEnd + while (bp < jsEnd) { + if (isxdigit(*bp)) { + cnt++; +#ifdef DEBUG + printf("%c", *bp); +#endif + } + bp++; + } + bp += 9; + } +#ifdef DEBUG + printf("\n"); +#endif + return cnt; + } else { + fprintf(stderr, "Unknown mode (%d) for capacityJS() ... \n", mode); + return 0; + } + +} + +/* + * capacityJS3 is the next iteration for capacityJS + */ +unsigned int capacityJS3 (char* buf, int len, int mode) { + char *hEnd, *bp, *jsStart, *jsEnd; + int cnt=0; + int j; + + // jump to the beginning of the body of the HTTP message + hEnd = strstr(buf, "\r\n\r\n"); + if (hEnd == NULL) { + // cannot find the separator between HTTP header and HTTP body + return 0; + } + bp = hEnd + 4; + + + if (mode == CONTENT_JAVASCRIPT) { + j = offset2Hex(bp, (buf+len)-bp, 0); + while (j != -1) { + cnt++; + if (j == 0) { + bp = bp+1; + } else { + bp = bp+j+1; + } +// #ifdef DEBUG +// printf("got |%c|\n", *(bp-1)); +// #endif + j = offset2Hex(bp, (buf+len)-bp, 1); + } // while + return cnt; + } else if (mode == CONTENT_HTML_JAVASCRIPT) { + while (bp < (buf+len)) { + jsStart = strstr(bp, "<script type=\"text/javascript\">"); + if (jsStart == NULL) break; + bp = jsStart+31; + jsEnd = strstr(bp, "</script>"); + if (jsEnd == NULL) break; + // count the number of usable hex char between jsStart+31 and jsEnd + + j = offset2Hex(bp, jsEnd-bp, 0); + while (j != -1) { + cnt++; + if (j == 0) { + bp = bp+1; + } else { + bp = bp+j+1; + } +#ifdef DEBUG +printf("got |%c|\n", *(bp-1)); +#endif + j = offset2Hex(bp, jsEnd-bp, 1); + } // while (j != -1) + + bp += 9; + } // while (bp < (buf+len)) + return cnt; + } else { + fprintf(stderr, "Unknown mode (%d) for capacityJS() ... \n", mode); + return 0; + } +} + + +/* + * strInBinary looks for char array pattern of length patternLen in a char array + * blob of length blobLen + * + * return a pointer for the first occurrence of pattern in blob, if found + * otherwise, return NULL + * + */ +char * +strInBinary (const char *pattern, unsigned int patternLen, + const char *blob, unsigned int blobLen) { + int found = 0; + char *cp = (char *)blob; + + while (1) { + if (blob+blobLen-cp < patternLen) break; + if (*cp == pattern[0]) { + if (memcmp(cp, pattern, patternLen) == 0) { + found = 1; + break; + } + } + cp++; + } + if (found) return cp; + else return NULL; +} + + +/* + * has_eligible_HTTP_content() identifies if the input HTTP message + * contains a specified type of content, used by a steg module to + * select candidate HTTP message as cover traffic + */ + +// for JavaScript, there are two cases: +// 1) If Content-Type: has one of the following values +// text/javascript +// application/x-javascript +// application/javascript +// 2) Content-Type: text/html and +// HTTP body contains <script type="text/javascript"> ... </script> +// #define CONTENT_JAVASCRIPT 1 (for case 1) +// #define CONTENT_HTML_JAVASCRIPT 2 (for case 2) +// +// for pdf, we look for the msgs whose Content-Type: has one of the +// following values +// 1) application/pdf +// 2) application/x-pdf +// + +int has_eligible_HTTP_content (char* buf, int len, int type) { + char* ptr = buf; + char* matchptr; + int tjFlag=0, thFlag=0, ceFlag=0, teFlag=0, http304Flag=0, clZeroFlag=0, pdfFlag=0, swfFlag=0; + char* end, *cp; + +#ifdef DEBUG + fprintf(stderr, "TESTING availabilty of js in payload ... \n"); +#endif + + if (type != HTTP_CONTENT_JAVASCRIPT && + type != HTTP_CONTENT_PDF && type != HTTP_CONTENT_SWF) + return 0; + + + // assumption: buf is null-terminated + if (!strstr(buf, "\r\n\r\n")) + return 0; + + + while (1) { + end = strstr(ptr, "\r\n"); + if (end == NULL) { + break; + } + + if (!strncmp(ptr, "Content-Type:", 13)) { + + if (!strncmp(ptr+14, "text/javascript", 15) || + !strncmp(ptr+14, "application/javascript", 22) || + !strncmp(ptr+14, "application/x-javascript", 24)) { + tjFlag = 1; + } + if (!strncmp(ptr+14, "text/html", 9)) { + thFlag = 1; + } + if (!strncmp(ptr+14, "application/pdf", 15) || + !strncmp(ptr+14, "application/x-pdf", 17)) { + pdfFlag = 1; + } + if (!strncmp(ptr+14, "application/x-shockwave-flash", strlen("application/x-shockwave-flash"))) { + swfFlag = 1; + } + + } else if (!strncmp(ptr, "Content-Encoding:", 17)) { + ceFlag = 1; + } else if (!strncmp(ptr, "Transfer-Encoding:", 18)) { + teFlag = 1; + } else if (!strncmp(ptr, "HTTP/1.1 304 ", 13)) { + http304Flag = 1; + } else if (!strncmp(ptr, "Content-Length: 0", 17)) { + clZeroFlag = 1; + } + + if (!strncmp(end, "\r\n\r\n", 4)){ + break; + } + ptr = end+2; + } + +#ifdef DEBUG + printf("tjFlag=%d; thFlag=%d; ceFlag=%d; teFlag=%d; http304Flag=%d; clZeroFlag=%d\n", + tjFlag, thFlag, ceFlag, teFlag, http304Flag, clZeroFlag); +#endif + + if (type == HTTP_CONTENT_JAVASCRIPT) { + // empty body if it's HTTP not modified (304) or zero Content-Length + if (http304Flag || clZeroFlag) return 0; + + // for now, we're not dealing with Transfer-Encoding (e.g., chunked) + // if (teFlag) return 0; + // if (ceFlag) return 0; // for now, we remove "Content-Encoding: gzip" in fixContentLen + if (teFlag || ceFlag) return 0; +/***** + if (teFlag) return 0; + *****/ + + if (tjFlag && ceFlag && end != NULL) { + log_debug("(JS) gzip flag detected with hdr len %d", (int)(end-buf+4)); + } else if (thFlag && ceFlag && end != NULL) { + log_debug("(HTML) gzip flag detected with hdr len %d", (int)(end-buf+4)); + } + + // case 1 + if (tjFlag) return 1; + + // case 2: check if HTTP body contains <script type="text/javascript"> + if (thFlag) { + matchptr = strstr(ptr, "<script type=\"text/javascript\">"); + if (matchptr != NULL) { + return 2; + } + } + } + + if (type == HTTP_CONTENT_PDF && pdfFlag) { + // reject msg with empty body: HTTP not modified (304) or zero Content-Length + if (http304Flag || clZeroFlag) return 0; + + // for now, we're not dealing with Transfer-Encoding (e.g., chunked) + // or Content-Encoding (e.g., gzip) + // if (teFlag) return 0; + // if (ceFlag) return 0; + if (teFlag || ceFlag) return 0; + + // check if HTTP body contains "endstream"; + // strlen("endstream") == 9 + + cp = strInBinary("endstream", 9, ptr, buf+len-ptr); + if (cp != NULL) { + // log_debug("Matched endstream!"); + return 1; + } + } + + if (type == HTTP_CONTENT_SWF && swfFlag == 1 && + ((len + buf - end) > SWF_SAVE_FOOTER_LEN + SWF_SAVE_HEADER_LEN + 8)) + return 1; + + return 0; +} + + + +unsigned int capacityPDF (char* buf, int len) { + char *hEnd, *bp, *streamStart, *streamEnd; + int cnt=0; + int size; + + // jump to the beginning of the body of the HTTP message + hEnd = strstr(buf, "\r\n\r\n"); + if (hEnd == NULL) { + // cannot find the separator between HTTP header and HTTP body + return 0; + } + bp = hEnd + 4; + + while (bp < (buf+len)) { + streamStart = strInBinary("stream", 6, bp, (buf+len)-bp); + // streamStart = strstr(bp, "stream"); + if (streamStart == NULL) break; + bp = streamStart+6; + streamEnd = strInBinary("endstream", 9, bp, (buf+len)-bp); + // streamEnd = strstr(bp, "endstream"); + if (streamEnd == NULL) break; + // count the number of char between streamStart+6 and streamEnd + size = streamEnd - (streamStart+6) - 2; // 2 for \r\n before streamEnd + if (size > 0) { + cnt = cnt + size; + log_debug("capacity of pdf increase by %d", size); + } + bp += 9; + } + return cnt; +} + + + + + + + + + +/* + * init_payload_pool initializes the arrays pertaining to + * message payloads for the specified content type + * + * Specifically, it populates the following arrays + * static int initTypePayload[MAX_CONTENT_TYPE]; + * static int typePayloadCount[MAX_CONTENT_TYPE]; + * static int typePayload[MAX_CONTENT_TYPE][MAX_PAYLOADS]; + * static int typePayloadCap[MAX_CONTENT_TYPE][MAX_PAYLOADS]; + * + * Input: + * len - max length of payload + * type - ptype field value in pentry_header + * contentType - (e.g, HTTP_CONTENT_JAVASCRIPT for JavaScript content) + */ + + + + +int init_JS_payload_pool(int len, int type, int minCapacity) { + + // stat for usable payload + int minPayloadSize = 0, maxPayloadSize = 0; + int sumPayloadSize = 0; + int minPayloadCap = 0, maxPayloadCap = 0; + int sumPayloadCap = 0; + + unsigned int contentType = HTTP_CONTENT_JAVASCRIPT; + + int cnt = 0; + int r; + pentry_header* p; + char* msgbuf; + int cap; + int mode; + + + + if (payload_count == 0) { + log_debug("payload_count == 0; forgot to run load_payloads()?\n"); + return 0; + } + + if (initTypePayload[contentType] != 0) return 1; // init is done already + + + for (r = 0; r < payload_count; r++) { + p = &payload_hdrs[r]; + if (p->ptype != type || p->length > len) { + continue; + } + + msgbuf = payloads[r]; + + mode = has_eligible_HTTP_content(msgbuf, p->length, HTTP_CONTENT_JAVASCRIPT); + if (mode > 0) { + + cap = capacityJS3(msgbuf, p->length, mode); + if (cap < JS_DELIMITER_SIZE) + continue; + + cap = (cap - JS_DELIMITER_SIZE)/2; + + if (cap > minCapacity) { + typePayloadCap[contentType][cnt] = cap; // (cap-JS_DELIMITER_SIZE)/2; + // because we use 2 hex char to encode every data byte, the available + // capacity for encoding data is divided by 2 + typePayload[contentType][cnt] = r; + cnt++; + + // update stat + if (cnt == 1) { + minPayloadSize = p->length; maxPayloadSize = p->length; + minPayloadCap = cap; maxPayloadCap = cap; + } + else { + if (minPayloadSize > p->length) minPayloadSize = p->length; + if (maxPayloadSize < p->length) maxPayloadSize = p->length; + if (minPayloadCap > cap) minPayloadCap = cap; + if (maxPayloadCap < cap) { + maxPayloadCap = cap; + } + + } + sumPayloadSize += p->length; sumPayloadCap += cap; + } + } + } + + + max_JS_capacity = maxPayloadCap; + + + initTypePayload[contentType] = 1; + typePayloadCount[contentType] = cnt; + log_debug("init_payload_pool: typePayloadCount for contentType %d = %d", + contentType, typePayloadCount[contentType]); + log_debug("minPayloadSize = %d", minPayloadSize); + log_debug("maxPayloadSize = %d", maxPayloadSize); + log_debug("avgPayloadSize = %f", (float)sumPayloadSize/(float)cnt); + log_debug("minPayloadCap = %d", minPayloadCap); + log_debug("maxPayloadCap = %d", maxPayloadCap); + log_debug("avgPayloadCap = %f", (float)sumPayloadCap/(float)cnt); + return 1; +} + + + + + + + + + +int init_PDF_payload_pool(int len, int type, int minCapacity) { + + // stat for usable payload + int minPayloadSize = 0, maxPayloadSize = 0; + int sumPayloadSize = 0; + int minPayloadCap = 0, maxPayloadCap = 0; + int sumPayloadCap = 0; + + int cnt = 0; + int r; + pentry_header* p; + char* msgbuf; + int cap; + int mode; + unsigned int contentType = HTTP_CONTENT_PDF; + + + if (payload_count == 0) { + fprintf(stderr, "payload_count == 0; forgot to run load_payloads()?\n"); + return 0; + } + + if (initTypePayload[contentType] != 0) return 1; // init is done already + + + for (r = 0; r < payload_count; r++) { + p = &payload_hdrs[r]; + if (p->ptype != type || p->length > len) { + continue; + } + + msgbuf = payloads[r]; + + mode = has_eligible_HTTP_content(msgbuf, p->length, HTTP_CONTENT_PDF); + if (mode > 0) { + // use capacityPDF() to find out the amount of data that we + // can encode in the pdf doc + // cap = minCapacity+1; + cap = capacityPDF(msgbuf, p->length); + log_debug("got pdf (index %d) with capacity %d", r, cap); + if (cap > minCapacity) { + log_debug("pdf (index %d) greater than mincapacity %d", cnt, minCapacity); + typePayloadCap[contentType][cnt] = (cap-PDF_DELIMITER_SIZE)/2; + typePayload[contentType][cnt] = r; + cnt++; + + // update stat + if (cnt == 1) { + minPayloadSize = p->length; maxPayloadSize = p->length; + minPayloadCap = cap; maxPayloadCap = cap; + } + else { + if (minPayloadSize > p->length) minPayloadSize = p->length; + if (maxPayloadSize < p->length) maxPayloadSize = p->length; + if (minPayloadCap > cap) minPayloadCap = cap; + if (maxPayloadCap < cap) maxPayloadCap = cap; + } + sumPayloadSize += p->length; sumPayloadCap += cap; + } + } + } + + max_PDF_capacity = maxPayloadCap; + initTypePayload[contentType] = 1; + typePayloadCount[contentType] = cnt; + log_debug("init_payload_pool: typePayloadCount for contentType %d = %d", + contentType, typePayloadCount[contentType]); + log_debug("minPayloadSize = %d", minPayloadSize); + log_debug("maxPayloadSize = %d", maxPayloadSize); + log_debug("avgPayloadSize = %f", (float)sumPayloadSize/(float)cnt); + log_debug("minPayloadCap = %d", minPayloadCap); + log_debug("maxPayloadCap = %d", maxPayloadCap); + log_debug("avgPayloadCap = %f", (float)sumPayloadCap/(float)cnt); + return 1; +} + + + + + +int init_SWF_payload_pool(int len, int type, int minCapacity) { + + // stat for usable payload + int minPayloadSize = 0, maxPayloadSize = 0; + int sumPayloadSize = 0; + + int cnt = 0; + int r; + pentry_header* p; + char* msgbuf; + int mode; + unsigned int contentType = HTTP_CONTENT_SWF; + + + if (payload_count == 0) { + fprintf(stderr, "payload_count == 0; forgot to run load_payloads()?\n"); + return 0; + } + + if (initTypePayload[contentType] != 0) return 1; // init is done already + + + for (r = 0; r < payload_count; r++) { + p = &payload_hdrs[r]; + if (p->ptype != type || p->length > len) { + continue; + } + + msgbuf = payloads[r]; + // found a payload corr to the specified contentType + + mode = has_eligible_HTTP_content(msgbuf, p->length, HTTP_CONTENT_SWF); + if (mode > 0) { + typePayload[contentType][cnt] = r; + cnt++; + // update stat + if (cnt == 1) { + minPayloadSize = p->length; + maxPayloadSize = p->length; + } + else { + if (minPayloadSize > p->length) + minPayloadSize = p->length; + if (maxPayloadSize < p->length) + maxPayloadSize = p->length; + } + sumPayloadSize += p->length; + } + } + + initTypePayload[contentType] = 1; + typePayloadCount[contentType] = cnt; + log_debug("init_payload_pool: typePayloadCount for contentType %d = %d", + contentType, typePayloadCount[contentType]); + log_debug("minPayloadSize = %d", minPayloadSize); + log_debug("maxPayloadSize = %d", maxPayloadSize); + log_debug("avgPayloadSize = %f", (float)sumPayloadSize/(float)cnt); + return 1; +} + + + + + + + + + +int get_next_payload (int contentType, char** buf, int* size, int* cap) { + int r; + + log_debug("get_next_payload: contentType = %d, initTypePayload = %d, typePayloadCount = %d", + contentType, initTypePayload[contentType], typePayloadCount[contentType]); + + + if (contentType <= 0 || + contentType >= MAX_CONTENT_TYPE || + initTypePayload[contentType] == 0 || + typePayloadCount[contentType] == 0) + return 0; + + r = rand() % typePayloadCount[contentType]; +// int r = 1; +// log_debug("SERVER: *** always choose the same payload ***"); + + log_debug("SERVER: picked payload with index %d", r); + *buf = payloads[typePayload[contentType][r]]; + *size = payload_hdrs[typePayload[contentType][r]].length; + *cap = typePayloadCap[contentType][r]; + return 1; +} + + + + + + + + +int get_payload (int contentType, int cap, char** buf, int* size) { + int r; + unsigned int i = 0; + unsigned int cnt = 0; + + log_debug("get_payload: contentType = %d, initTypePayload = %d, typePayloadCount = %d", + contentType, initTypePayload[contentType], typePayloadCount[contentType]); + + + if (contentType <= 0 || + contentType >= MAX_CONTENT_TYPE || + initTypePayload[contentType] == 0 || + typePayloadCount[contentType] == 0) + return 0; + + + cnt = typePayloadCount[contentType]; + r = rand() % cnt; + + for (i=0; i < cnt; i++) { + + if (typePayloadCap[contentType][(r+i) % cnt] <= cap) + continue; + + *buf = payloads[typePayload[contentType][(r+i)%cnt]]; + *size = payload_hdrs[typePayload[contentType][(r+i)%cnt]].length; + return 1; + } + + + + return 0; + +} + + + + +int +find_content_length (char *hdr, int hlen) { + char *clStart; + char* clEnd; + char *clValStart; + int valLen; + int contentLen; + char buf[10]; + + clStart = strstr(hdr, "Content-Length: "); + if (clStart == NULL) { + log_debug("Unable to find Content-Length in the header"); + return -1; + } + + clEnd = strstr((char *)clStart, "\r\n"); + if (clEnd == NULL) { + log_debug("Unable to find end of line for Content-Length"); + return -1; + } + + // clValStart = clStart+strlen("Content-Length: "); + clValStart = clStart+16; + + valLen = clEnd-clValStart; + if (valLen > 9) return -1; + memcpy(buf, clValStart, valLen); + buf[valLen] = 0; + contentLen = atoi(buf); + return contentLen; +} + + + + + + +/* + +void testOffset2Alnum_skipJSPattern () { + char s1[] = "for (i=0; i<10; i++) { print i; }"; + + char s2[] = "***abcde*****"; + int d, i; + + printf("s1 = %s\n", s1); + printf("s2 = %s\n", s2); + + + d = offset2Alnum_(s1, strlen(s1)); + printf ("offset2Alnum_ for s1 = %d\n", d); + d = offset2Alnum_(s2, strlen(s2)); + printf ("offset2Alnum_ for s2 = %d\n", d); + + i = skipJSPattern (s1, strlen(s1)); + printf ("skipJSPattern for s1 = %d\n", i); + i = skipJSPattern (s2, strlen(s2)); + printf ("skipJSPattern for s2 = %d\n", i); +} + + + + +void testOffset2Hex () { + int d; + char s3[] = "for (bc=0; bc<10; bc++) { ad=2*bc+ad; }"; + printf("len(s3)=%d; s3 = |%s|\n", (int)strlen(s3), s3); + + d = offset2Alnum_(s3, strlen(s3)); + printf ("offset2Alnum_ for s3 = %d\n", d); + d = offset2Hex(s3, strlen(s3), 0); + printf ("offset2Hex for s3 = %d\n", d); +} + + +void testCapacityJS () { + int d; + char s4[] = "\r\n\r\n abc = abc + 1;"; + char s6[] = "\r\n\r\n <script type=\"text/javascript\">abc = abc + 1;</script>"; + + printf("\nTest for CONTENT_JAVASCRIPT:\n"); + printf("len(s4)=%d; s4 = |%s|\n", (int)strlen(s4), s4); + + d = offset2Alnum_(s4, strlen(s4)); + printf ("offset2Alnum_ for s4 = %d\n", d); + d = offset2Hex(s4, strlen(s4), 0); + printf ("offset2Hex for s4 = %d\n", d); + + printf("capacityJS (JS) returns %d\n", capacityJS(s4, strlen(s4), CONTENT_JAVASCRIPT)); + printf("capacityJS3 (JS) returns %d\n", capacityJS3(s4, strlen(s4), CONTENT_JAVASCRIPT)); + + printf("\nTest for CONTENT_HTML_JAVASCRIPT:\n"); + printf("len(s6)=%d; s6 = |%s|\n", (int)strlen(s6), s6); + + d = offset2Alnum_(s6, strlen(s6)); + printf ("offset2Alnum_ for s6 = %d\n", d); + d = offset2Hex(s6, strlen(s6), 0); + printf ("offset2Hex for s6 = %d\n", d); + + printf("capacityJS (HTML) returns %d\n", capacityJS(s6, strlen(s6), CONTENT_HTML_JAVASCRIPT)); + printf("capacityJS3 (HTML) returns %d\n", capacityJS3(s6, strlen(s6), CONTENT_HTML_JAVASCRIPT)); +} +*/ + + +/***** +int main() { + char buf[HTTP_MSG_BUF_SIZE]; + bzero(buf, sizeof(buf)); + // test for TYPE_HTTP_REQUEST + // load_payloads("../../traces/client.out"); + // int len = find_client_payload(buf, 10000, TYPE_HTTP_REQUEST); + // printf("%s\n", buf); + + // test for TYPE_HTTP_RESPONSE + // load_payloads("../../traces/server-cnn-nogzip.out"); + // load_payloads("../../traces/server-portals.out"); // ptype==1? + + // testOffset2Alnum_skipJSPattern(); + // testOffset2Hex(); + // testCapacityJS(); + + load_payloads("../../traces/server.out"); + // int r; + // r = find_server_payload(&buf, sizeof(buf), TYPE_HTTP_RESPONSE, HTTP_CONTENT_JAVASCRIPT); + // if (r > 0) { + // printf("Available payload capablity %d\n", r); + // } + // return r; + + return 0; +} + *****/ + diff --git a/src/steg/payloads.h b/src/steg/payloads.h new file mode 100644 index 0000000..b3fcc9d --- /dev/null +++ b/src/steg/payloads.h @@ -0,0 +1,159 @@ +#ifndef _PAYLOADS_H +#define _PAYLOADS_H + +#include <stdio.h> +#include <stdlib.h> +#include <time.h> +#include <string.h> +#include <sys/types.h> +#include <arpa/inet.h> +#include <ctype.h> + + +/* three files: + server_data, client data, protocol data +*/ + + + +#define RECV_GOOD 0 +#define RECV_INCOMPLETE 0 +#define RECV_BAD -1 + + + +#define CONN_DATA_REQUEST 1 /* payload packet sent by client */ +#define CONN_DATA_REPLY 2 /* payload packet sent by server */ + +#define NO_NEXT_STATE -1 + +#define MAX_PAYLOADS 10000 +// #define HTTP_MSG_BUF_SIZE 100000 + +// jsSteg-specific defines +#define JS_DELIMITER '?' +// a JavaScript delimiter is used to signal the end of encoding +// to facilitate the decoding process +#define JS_DELIMITER_REPLACEMENT '.' +// JS_DELIMITER that exists in the JavaScript before the end of +// data encoding will be replaced by JS_DELIMITER_REPLACEMENT +#define JS_DELIMITER_SIZE 1 + +#define JS_MIN_AVAIL_SIZE 2050 +// JS_MIN_AVAIL_SIZE should reflect the min number of data bytes +// a JavaScript may encapsulate +// Using hex-based encoding, it takes 2 hex char in JS +// to encode 1 data byte. Thus the size of data that can be encoded +// is about half this value + + +#define PDF_DELIMITER_SIZE 2 +#define PDF_MIN_AVAIL_SIZE 10240 +// PDF_MIN_AVAIL_SIZE should reflect the min number of data bytes +// a pdf doc can encode + +// specifying the type of contents as an input argument +// for has_eligible_HTTP_content() +#define HTTP_CONTENT_JAVASCRIPT 1 +#define HTTP_CONTENT_PDF 2 +#define HTTP_CONTENT_SWF 3 +#define HTTP_CONTENT_ENCRYPTEDZIP 4 +#define HTTP_CONTENT_HTML 5 + +// used by the JavaScript steg module to distinguish two cases in which +// JS may appear in the HTTP msg +// 1) CONTENT-TYPE in HTTP header specifies that the HTTP body is a JS +// 2) CONTENT-TYPE corresponds to HTML, and the HTTP body contains JS +// denoted by script type for JS +#define CONTENT_JAVASCRIPT 1 +#define CONTENT_HTML_JAVASCRIPT 2 + + +// payloads for specific content type +// +// MAX_CONTENT_TYPE specifies the maximum number of supported content types +// (e.g. HTTP_CONTENT_JAVASCRIPT is a content type) +// +// initTypePayload[x] specifies whether the arrays typePayloadCount and +// typePayloads for content type x +// +// typePayloadCount[x] specifies the number of available payloads for +// content type x +// +// typePayload[x][] contains references to the corresponding entries in +// payload_hdrs[] and payloads[] +// +// typePayloadCap[x][] specifies the capacity for typePayload[x][] + +#define MAX_CONTENT_TYPE 11 + + + +typedef int SID; +typedef short PacketType; +typedef short StateFlag; + +#define TYPE_SERVICE_DATA 0x1 +#define TYPE_HTTP_REQUEST 0x2 +#define TYPE_HTTP_RESPONSE 0x4 +#define BEGIN_STATE_FLG 0x1 +#define END_STATE_FLG 0x2 + + +/* struct for reading in the payload_gen dump file */ +typedef struct { + PacketType ptype; + int length; + ushort port; /* network format */ +}pentry_header; + + + + +typedef struct service_state { + SID id; + PacketType data_type; + SID next_state; + // double* probabilities; + StateFlag flg; + int dir; +}state; + + +#define HTTP_MSG_BUF_SIZE 100000 +void load_payloads(const char* fname); +unsigned int find_client_payload(char* buf, int len, int type); +unsigned int find_server_payload(char** buf, int len, int type, int contentType); + +int init_JS_payload_pool(int len, int type, int minCapacity); +int init_SWF_payload_pool(int len, int type, int minCapacity); +int init_PDF_payload_pool(int len, int type,int minCapacity); + + +int get_next_payload (int contentType, char** buf, int* size, int* cap); + +int get_payload (int contentType, int cap, char** buf, int* size); + +int has_eligible_HTTP_content (char* buf, int len, int type); +int fixContentLen (char* payload, int payloadLen, char *buf, int bufLen); +void gen_rfc_1123_date(char* buf, int buf_size); +int parse_client_headers(char* inbuf, char* outbuf, int len); +int skipJSPattern (char *cp, int len); +int isalnum_ (char c); +int offset2Alnum_ (char *p, int range); +int offset2Hex (char *p, int range, int isLastCharHex); +unsigned int capacityJS (char* buf, int len, int mode); +unsigned int capacityJS3 (char* buf, int len, int mode); +unsigned int get_max_JS_capacity(void); + +char * strInBinary (const char *pattern, unsigned int patternLen, const char *blob, unsigned int blobLen); + + +unsigned int capacityPDF (char* buf, int len); +unsigned int get_max_PDF_capacity(void); +int find_content_length (char *hdr, int hlen); +int find_uri_type(char* buf); + +int gen_response_header(char* content_type, int gzip, int length, char* buf, int buflen); + +#endif diff --git a/src/steg/pdfSteg.c b/src/steg/pdfSteg.c new file mode 100644 index 0000000..58147fa --- /dev/null +++ b/src/steg/pdfSteg.c @@ -0,0 +1,618 @@ +#include "payloads.h" +#include "pdfSteg.h" + +void buf_dump(unsigned char* buf, int len, FILE *out); + +#define STREAM_BEGIN ">>stream" +#define STREAM_BEGIN_SIZE 8 +#define STREAM_END "endstream" +#define STREAM_END_SIZE 9 + +#define DEBUG + + +/* + * pdfSteg: A PDF-based steganography module + * + */ + + +/* + * addDelimiter processes the input buffer (inbuf) of length inbuflen, + * copies it to output buffer (outbuf) of size outbufsize, + * and adds a two-char-long, end-of-data pattern at the end of outbuf + * based on delimiter1 and delimiter2. + * + * The end-of-data pattern consists of delimiter1 followed by a char + * that is not delimiter1. Thus, delimiter1 and delimiter2 must be + * different. + * + * If delimiter1 appears in the input buffer, addDelimiter puts two + * delimiter1 char in output buffer (to enable removeDelimiter to perform + * the back transformation) + * + * addDelimiter returns the length of the data written to outbuf, including + * the end-of-data pattern, if the transformation succeeds; + * otherwise, it returns -1 + * + */ +int +addDelimiter(char *inbuf, int inbuflen, char *outbuf, int outbuflen, + const char delimiter1, const char delimiter2) +{ + int cnt; + char *ibp, ic, rc; + + if (delimiter1 == delimiter2) return -1; + + cnt = 0; + ibp = inbuf; + while ((ibp-inbuf)<inbuflen && cnt<(outbuflen-2)) { + ic = *(ibp++); + if (ic != delimiter1) { + outbuf[cnt++] = ic; + } else { + outbuf[cnt++] = delimiter1; + outbuf[cnt++] = delimiter1; + } + } + + // error if outbuf is no large enough for storing the resulting data + if (cnt >= (outbuflen-2)) return -1; + + // put delimiter1 and a char that is not a delimiter1 + // as the end-of-data pattern at the end of outbuf + outbuf[cnt++] = delimiter1; + // try to get a random char (that is not delimiter1) + rc = (char) (rand() % 256); + if (rc != delimiter1) { + outbuf[cnt++] = rc; + } else { // unable to get a rand char != delimiter1, use delimiter2 + outbuf[cnt++] = delimiter2; + } + return cnt; +} + + +/* + * removeDelimiter performs the reverse transformation of addDelimiter. + * + * returns the length of data written to outbuf, if succeed; + * otherwise, it returns -1 + * endFlag indicates whether the end-of-encoding byte pattern (i.e., + * delimiter1 followed by non-delimiter1) is detected + */ +int +removeDelimiter(char *inbuf, int inbuflen, char *outbuf, int outbuflen, + const char delimiter1, int *endFlag, int *escape) +{ + int cnt; + char *ibp, ic1, ic2; + + cnt = 0; + *endFlag = 0; + ibp = inbuf; + + if (inbuflen <= 0) return -1; + + // special case: 2-char, end-of-data pattern could be in two buffers + // if *escape == true, we need to see if + // 1) (*ibp == delimiter1) -> put delimiter1 in outbuf + // 2) (*ibp != delimiter1) -> end-of-data detected + if (*escape) { + ic1 = *ibp; + if (ic1 == delimiter1) { + outbuf[cnt++] = ic1; ibp++; + } else { + *endFlag = 1; + return 0; + } + } + + *escape = 0; + while ((ibp-inbuf+1)<inbuflen && cnt<outbuflen) { + ic1 = *(ibp++); + if (ic1 != delimiter1) { + // *escape = 0; + outbuf[cnt++] = ic1; + } else { + // *escape = 1; + // lookahead 1 char + ic2 = *ibp; + // if the next char is delimiter1 + if (ic2 == delimiter1) { + outbuf[cnt++] = delimiter1; ibp++; + // *escape = 0; + } else { // end-of-data pattern detected + *endFlag = 1; + break; + } + } + } + + // if (*escape) { + // *escape = 0; + // return cnt; + // } + if (ibp-inbuf == inbuflen) return cnt; + + // handling the last char in inbuf, if needed + ic1 = *ibp; + if (ic1 != delimiter1) { + outbuf[cnt++] = ic1; + } else { + // look at the next stream obj to handle the special cases + *escape = 1; + } + + return cnt; +} + + + +/* + * pdfWrap embeds data of length dlen inside the stream objects of the PDF + * document (length plen) that appears in the body of a HTTP msg, and + * stores the result in the output buffer of size outsize + * + * pdfWrap returns the length of the pdf document with the data embedded + * inside, if succeed; otherwise, it returns -1 to indicate an error + * + */ +int +pdfWrap (char *data, unsigned int dlen, + char *pdfTemplate, unsigned int plen, + char *outbuf, unsigned int outbufsize) +{ + char data2[dlen*2+2]; + char *tp, *dp, *op, *streamStart, *streamEnd, *plimit; + int data2len, cnt, size, size2; + + // assumption: pdfWrap is length-preserving + if (outbufsize < plen) return -1; + + data2len = addDelimiter(data, dlen, data2, HTTP_MSG_BUF_SIZE, PDF_DELIMITER, PDF_DELIMITER2); + if (data2len < 1) return -1; + + + op = outbuf; // current pointer for output buffer + tp = pdfTemplate; // current pointer for http msg template + dp = data2; // current pointer for data2 + cnt = 0; // number of data char encoded + plimit = pdfTemplate+plen; + + while (tp < plimit) { + // find the next stream obj + streamStart = strInBinary(STREAM_BEGIN, STREAM_BEGIN_SIZE, tp, plimit-tp); + if (streamStart == NULL) { + log_warn("Cannot find stream in pdf"); + return -1; + } + + // copy everything between tp and "stream" (inclusive) to outbuf + size = streamStart - tp + STREAM_BEGIN_SIZE; + memcpy(op, tp, size); + op += size; + tp = streamStart + STREAM_BEGIN_SIZE; + + streamEnd = strInBinary(STREAM_END, STREAM_END_SIZE, tp, plimit-tp); + if (streamEnd == NULL) { + log_warn("Cannot find endstream in pdf"); + return -1; + } + + // count the number of usable char between tp and streamEnd + size = streamEnd-tp; + + // encoding data in the stream obj + if (size > 0) { + size2 = data2len - cnt; + if (size < size2) { + memcpy(op, dp, size); + op += size; tp += size; dp += size; + memcpy(op, tp, STREAM_END_SIZE); + op += STREAM_END_SIZE; tp += STREAM_END_SIZE; + cnt += size; + } else { // done encoding data + memcpy(op, dp, size2); + op += size2; tp += size2; dp += size2; + cnt += size2; + printf("Encoded %d char in pdf. Done encoding\n", size2); + break; + } + log_debug("Encoded %d char in pdf", size); + } else { // empty stream + memcpy(op, tp, STREAM_END_SIZE); + op += STREAM_END_SIZE; tp += STREAM_END_SIZE; + } + + if (cnt >= data2len) break; // this shouldn't happen ... + } + + // copy the rest of pdfTemplate to outbuf + size = plimit-tp; + log_debug("copying the rest of pdfTemplate to outbuf (size %d)", size); + memcpy(op, tp, size); + op += size; + return (op-outbuf); +} + + + + +/* + * pdfUnwrap is the inverse operation of pdfWrap + */ +int +pdfUnwrap (char *data, unsigned int dlen, + char *outbuf, unsigned int outbufsize) +{ + char *dp, *op, *streamStart, *streamEnd, *dlimit, *olimit; + int cnt, size, size2, endFlag; + int escape = 0; + + dp = data; // current pointer for data + op = outbuf; // current pointer for outbuf + cnt = 0; // number of char decoded + dlimit = data+dlen; + olimit = outbuf+outbufsize; + + while (dp < dlimit) { + // find the next stream obj + streamStart = strInBinary(STREAM_BEGIN, STREAM_BEGIN_SIZE, dp, dlimit-dp); + if (streamStart == NULL) { + log_warn("Cannot find stream in pdf"); + return -1; + } + + dp = streamStart + STREAM_BEGIN_SIZE; + streamEnd = strInBinary(STREAM_END, STREAM_END_SIZE, dp, dlimit-dp); + if (streamEnd == NULL) { + log_warn("Cannot find endstream in pdf"); + return -1; + } + + // count the number of usable char between tp and streamEnd + size = streamEnd-dp; + + if (size > 0) { + size2 = removeDelimiter(dp, size, op, olimit-op, PDF_DELIMITER, &endFlag, &escape); + if (size2 < 0) { + return -1; + } + cnt += size2; + if (endFlag) { // Done decoding + break; + } else { // Continue decoding + op += size2; + dp = streamEnd + STREAM_END_SIZE; + } + } else { // empty stream obj + dp = streamEnd + STREAM_END_SIZE; + } + } + + return cnt; +} + + + + + +int x_http2_server_PDF_transmit (steg_t* s, struct evbuffer *source, conn_t *conn) { + + struct evbuffer *dest = conn_get_outbound(conn); + size_t sbuflen = evbuffer_get_length(source); + unsigned int mpdf; + char *pdfTemplate = NULL, *hend; + int pdfTemplateSize = 0; + // char data1[HTTP_MSG_BUF_SIZE]; + char data1[(int) sbuflen]; + char outbuf[HTTP_MSG_BUF_SIZE]; + int cnt, hLen, outbuflen, i; + + struct evbuffer_iovec *iv; + int nv; + + // for debugging pdfWrap and pdfUnwrap + // char data2[(int) sbuflen]; + // int data2len; + + log_debug("Entering SERVER PDF transmit with sbuflen %d", (int)sbuflen); + + nv = evbuffer_peek(source, sbuflen, NULL, NULL, 0); + iv = xzalloc(sizeof(struct evbuffer_iovec) * nv); + + if (evbuffer_peek(source, sbuflen, NULL, iv, nv) != nv) { + free(iv); + return -1; + } + + cnt = 0; + for (i = 0; i < nv; i++) { + const unsigned char *p = iv[i].iov_base; + const unsigned char *limit = p + iv[i].iov_len; + while (p < limit && cnt < (int)sbuflen) { + data1[cnt++] = *p++; + } + } + + free(iv); + + log_debug("SERVER sbuflen = %d; cnt = %d", (int)sbuflen, cnt); + + mpdf = get_max_PDF_capacity(); + + if (mpdf <= 0) { + log_warn("SERVER ERROR: No pdfTemplate found\n"); + return -1; + } + + if (sbuflen > (size_t) mpdf) { + log_warn("SERVER ERROR: pdfTemplate cannot accommodate data %d %dn", + (int) sbuflen, (int) mpdf); + return -1; + } + + if (get_payload(HTTP_CONTENT_PDF, sbuflen, &pdfTemplate, &pdfTemplateSize) == 1) { + log_debug("SERVER found the next HTTP response template with size %d", pdfTemplateSize); + } else { + log_warn("SERVER couldn't find the next HTTP response template"); + return -1; + } + + hend = strstr(pdfTemplate, "\r\n\r\n"); + if (hend == NULL) { + log_warn("SERVER unable to find end of header in the HTTP template"); + return -1; + } + + hLen = hend+4-pdfTemplate; + + log_debug("SERVER calling pdfWrap for data1 with length %d", cnt); + outbuflen = pdfWrap(data1, cnt, hend+4, pdfTemplateSize-hLen, outbuf, HTTP_MSG_BUF_SIZE); + if (outbuflen < 0) { + log_warn("SERVER pdfWrap fails"); + return -1; + } + log_debug("SERVER pdfSteg sends resp with hdr len %d body len %d", hLen, outbuflen); + + + // debugging + // buf_dump((unsigned char *)data1, cnt, stderr); + + // data2len = pdfUnwrap(outbuf, outbuflen, data2, sbuflen); + // if ((int)sbuflen == data2len) { + // log_warn("sbuflen == data2len == %d", (int)sbuflen); + // if (memcmp(data1, data2, sbuflen) == 0) { + // log_warn("data1 and data2 match"); + // } else { + // log_warn("data1 and data2 DO NOT match!! Dumping data1 ..."); + // buf_dump((unsigned char *)data1, cnt, stderr); + // log_warn("data1 and data2 DO NOT match!! Dumping data2..."); + // buf_dump((unsigned char *)data2, data2len, stderr); + // } + // } else { + // log_warn("*** sbuflen = %d, data2len = %d *** Dumping data1 ...", (int)sbuflen, data2len); + // buf_dump((unsigned char *)data1, cnt, stderr); + // log_warn("*** sbuflen = %d, data2len = %d *** Dumping data2 ...", (int)sbuflen, data2len); + // buf_dump((unsigned char *)data2, data2len, stderr); + // } + + + if (evbuffer_add(dest, pdfTemplate, hLen)) { + log_warn("SERVER ERROR: evbuffer_add() fails for pdfTemplate"); + return -1; + } + if (evbuffer_add(dest, outbuf, outbuflen)) { + log_warn("SERVER ERROR: evbuffer_add() fails for outbuf"); + return -1; + } + + evbuffer_drain(source, sbuflen); + + conn_close_after_transmit(conn); + // downcast_steg(s)->have_transmitted = 1; + return 0; +} + + + +int +x_http2_handle_client_PDF_receive(steg_t *s, conn_t *conn, struct evbuffer *dest, struct evbuffer* source) { + struct evbuffer_ptr s2; + unsigned int response_len = 0, hdrLen; + char outbuf[HTTP_MSG_BUF_SIZE]; + int content_len = 0, outbuflen; + char *httpHdr, *httpBody; + + log_debug("Entering CLIENT PDF receive"); + + s2 = evbuffer_search(source, "\r\n\r\n", sizeof ("\r\n\r\n") -1 , NULL); + if (s2.pos == -1) { + log_warn("CLIENT Did not find end of HTTP header %d", (int) evbuffer_get_length(source)); + // evbuffer_dump(source, stderr); + return RECV_INCOMPLETE; + } + + log_debug("CLIENT received response header with len %d", (int)s2.pos); + + response_len = 0; + hdrLen = s2.pos + strlen("\r\n\r\n"); + response_len += hdrLen; + + httpHdr = (char *) evbuffer_pullup(source, s2.pos); + if (httpHdr == NULL) { + log_warn("CLIENT unable to pullup the complete HTTP header"); + return RECV_BAD; + } + + content_len = find_content_length(httpHdr, hdrLen); + if (content_len < 0) { + log_warn("CLIENT unable to find content length"); + return RECV_BAD; + } + log_debug("CLIENT received Content-Length = %d\n", content_len); + + response_len += content_len; + + if (response_len > evbuffer_get_length(source)) + return RECV_INCOMPLETE; + + httpHdr = (char *) evbuffer_pullup(source, response_len); + httpBody = httpHdr + hdrLen; + + + outbuflen = pdfUnwrap(httpBody, content_len, outbuf, HTTP_MSG_BUF_SIZE); + if (outbuflen < 0) { + log_warn("CLIENT ERROR: pdfUnwrap fails\n"); + return RECV_BAD; + } + + log_debug("CLIENT unwrapped data of length %d:", outbuflen); + + + // debugging + // buf_dump((unsigned char *)outbuf, outbuflen, stderr); + // ***** not sure why there is an extra char at the end of outbuf + outbuflen--; + + + if (evbuffer_add(dest, outbuf, outbuflen)) { + log_warn("CLIENT ERROR: evbuffer_add to dest fails\n"); + return RECV_BAD; + } + + // log_debug("Drained source for %d char\n", response_len); + if (evbuffer_drain(source, response_len) == -1) { + log_warn("CLIENT ERROR: failed to drain source\n"); + return RECV_BAD; + } + + // downcast_steg(s)->have_received = 1; + conn_expect_close(conn); + return RECV_GOOD; +} + + + + +/***** +int main() { + char data1[] = "this is a test?? yes!"; + char data2[100]; + char data3[100]; + int dlen1, dlen2, dlen3, end; + char last = ' '; + printf("hello world\n"); + + dlen2 = addDelimiter(data1, strlen(data1), data2, 100, '?', '.'); + printf("dlen2 = %d\n", dlen2); + dlen3 = removeDelimiter(data2, dlen2, data3, 100, '?', &end, &last); + printf("endflag = %d", end); + printf("dlen3 = %d\n", dlen3); + if (memcmp(data1, data3, dlen3) == 0) { + data1[dlen3] = 0; + printf("removeDelimiter(addDelimiter(x)) == x for |%s|\n", data1); + } else { + printf("removeDelimiter(addDelimiter(x)) != x for |%s|\n", data1); + } + return 1; +} + *****/ + +/***** +int main() { + char data1[] = "12345"; + char data2[] = "123456789012"; + char data3[] = "12345678901"; + char data4[] = "1234567890?"; + char pdf1[] = "[PDFHDR][STUFFS1]>>streamABCDEFGHIJYYendstream[STUFFS2]>>streamABCDEFGHIJYYendstream[STUFF3][PDFTRAILER]"; + char out[200]; + char orig[200]; + int r1, r2; + + printf("********************\n"); + printf("pdfwrap for %s\n", data1); + printf("strlen(pdf1) = %d\n", (int)strlen(pdf1)); + r1 = pdfWrap(data1, strlen(data1), pdf1, strlen(pdf1), out, (int)sizeof(out)); + if (r1 > 0) { + printf("pdfWrap returns %d\n", r1); + out[r1] = 0; + printf("out[] contains |%s|\n", out); + } else { + printf("pdfWrap returns %d\n", r1); + } + + r2 = pdfUnwrap(out, r1, orig, (int)sizeof(orig)); + if (r2 > 0) { + printf("pdfUnwrap returns %d\n", r2); + orig[r2] = 0; + printf("orig[] contains |%s|\n", orig); + } else { + printf("pdfUnwrap returns %d\n", r2); + } + + printf("********************\n"); + printf("pdfwrap for %s\n", data2); + r1 = pdfWrap(data2, strlen(data2), pdf1, strlen(pdf1), out, (int)sizeof(out)); + if (r1 > 0) { + printf("pdfWrap returns %d\n", r1); + out[r1] = 0; + printf("out[] contains |%s|\n", out); + } else { + printf("pdfWrap returns %d\n", r1); + } + + r2 = pdfUnwrap(out, r1, orig, (int)sizeof(orig)); + if (r2 > 0) { + printf("pdfUnwrap returns %d\n", r2); + orig[r2] = 0; + printf("orig[] contains |%s|\n", orig); + } else { + printf("pdfUnwrap returns %d\n", r2); + } + + printf("********************\n"); + printf("pdfwrap for %s\n", data3); + r1 = pdfWrap(data3, strlen(data3), pdf1, strlen(pdf1), out, (int)sizeof(out)); + if (r1 > 0) { + printf("pdfWrap returns %d\n", r1); + out[r1] = 0; + printf("out[] contains |%s|\n", out); + } else { + printf("pdfWrap returns %d\n", r1); + } + + r2 = pdfUnwrap(out, r1, orig, (int)sizeof(orig)); + if (r2 > 0) { + printf("pdfUnwrap returns %d\n", r2); + orig[r2] = 0; + printf("orig[] contains |%s|\n", orig); + } else { + printf("pdfUnwrap returns %d\n", r2); + } + + printf("********************\n"); + printf("pdfwrap for %s\n", data4); + r1 = pdfWrap(data4, strlen(data4), pdf1, strlen(pdf1), out, (int)sizeof(out)); + if (r1 > 0) { + printf("pdfWrap returns %d\n", r1); + out[r1] = 0; + printf("out[] contains |%s|\n", out); + } else { + printf("pdfWrap returns %d\n", r1); + } + + r2 = pdfUnwrap(out, r1, orig, (int)sizeof(orig)); + if (r2 > 0) { + printf("pdfUnwrap returns %d\n", r2); + orig[r2] = 0; + printf("orig[] contains |%s|\n", orig); + } else { + printf("pdfUnwrap returns %d\n", r2); + } + + return 0; +} + *****/ diff --git a/src/steg/pdfSteg.h b/src/steg/pdfSteg.h new file mode 100644 index 0000000..7e48449 --- /dev/null +++ b/src/steg/pdfSteg.h @@ -0,0 +1,29 @@ +#ifndef _PDFSTEG_H +#define _PDFSTEG_H + + +#include <stdio.h> +#include <string.h> +#include <ctype.h> +#include "util.h" +#include "connections.h" +#include "steg.h" +#include <event2/buffer.h> + + + +#define PDF_DELIMITER '?' +#define PDF_DELIMITER2 '.' + +int pdfWrap (char *data, unsigned int dlen, char *pdfTemplate, unsigned int plen, char *outbuf, unsigned int outbufsize); +int pdfUnwrap (char *data, unsigned int dlen, char *outbuf, unsigned int outbufsize); + +int addDelimiter(char *inbuf, int inbuflen, char *outbuf, int outbuflen, const char delimiter1, const char delimiter2); +int removeDelimiter(char *inbuf, int inbuflen, char *outbuf, int outbuflen, const char delimiter1, int* endFlag, int* escape); + +int x_http2_server_PDF_transmit (steg_t* s, struct evbuffer *source, conn_t *conn); +int +x_http2_handle_client_PDF_receive(steg_t *s, conn_t *conn, struct evbuffer *dest, struct evbuffer* source); + +#endif + diff --git a/src/steg/swfSteg.c b/src/steg/swfSteg.c new file mode 100644 index 0000000..ad3d5c8 --- /dev/null +++ b/src/steg/swfSteg.c @@ -0,0 +1,282 @@ +#include "swfSteg.h" + + +static const char http_response_1[] = + "HTTP/1.1 200 OK\r\n" + "Expires: Thu, 01 Jan 1970 00:00:00 GMT\r\n" + "Cache-Control: no-store\r\n" + "Connection: close\r\n" + "Content-Type: application/x-shockwave-flash\r\n" + "Content-Length: "; + + + + + + + + + + + +unsigned int +swf_wrap(char* inbuf, int in_len, char* outbuf, int out_sz) { + + char* swf; + int in_swf_len; + + char* tmp_buf; + int out_swf_len; + + char* resp; + int resp_len; + + char hdr[512]; + unsigned int hdr_len; + + char* tmp_buf2; + + + + if (!get_payload(HTTP_CONTENT_SWF, -1, &resp, &resp_len)) { + log_warn("swfsteg: no suitable payload found\n"); + return -1; + } + + swf = strstr(resp, "\r\n\r\n") + 4; + in_swf_len = resp_len - (swf - resp); + + + + + if (out_sz - in_len < (SWF_SAVE_FOOTER_LEN + SWF_SAVE_HEADER_LEN + 8 + 512)) { + fprintf(stderr, "swfsteg: outbuf too small %d \n", out_sz - in_len); + + log_warn("swfsteg: outbuf too small\n"); + return -1; + } + + + tmp_buf = malloc(in_len + SWF_SAVE_HEADER_LEN + SWF_SAVE_FOOTER_LEN); + + if (tmp_buf == NULL) { + log_warn("swfsteg: malloc failed\n"); + return -1; + } + + + tmp_buf2 = malloc(in_len + SWF_SAVE_HEADER_LEN + SWF_SAVE_FOOTER_LEN + 512); + + if (tmp_buf2 == NULL) { + free(tmp_buf); + log_warn("swfsteg: malloc failed\n"); + return -1; + } + + + memcpy(tmp_buf, swf+8, SWF_SAVE_HEADER_LEN); + memcpy(tmp_buf+SWF_SAVE_HEADER_LEN, inbuf, in_len); + memcpy(tmp_buf+SWF_SAVE_HEADER_LEN+in_len, swf + in_swf_len - SWF_SAVE_FOOTER_LEN, SWF_SAVE_FOOTER_LEN); + out_swf_len = def(tmp_buf, SWF_SAVE_HEADER_LEN + in_len + SWF_SAVE_FOOTER_LEN, tmp_buf2+8, + in_len + SWF_SAVE_HEADER_LEN + SWF_SAVE_FOOTER_LEN + 512-8, + Z_DEFAULT_COMPRESSION); + + // sprintf(hdr, "%s%d\r\n\r\n", http_response_1, out_swf_len + 8); + + + + // fprintf(stderr, "out_swf_len = %d\n", out_swf_len); + + + hdr_len = gen_response_header((char*) "application/x-shockwave-flash", 0, out_swf_len + 8, hdr, sizeof(hdr)); + + // fprintf(stderr, "hdr = %s\n", hdr); + + memcpy(tmp_buf2, swf, 4); + ((int*) (tmp_buf2))[1] = out_swf_len; + + memcpy(outbuf, hdr, hdr_len); + memcpy(outbuf+hdr_len, tmp_buf2, out_swf_len + 8); + + free(tmp_buf); + free(tmp_buf2); + return out_swf_len + 8 + hdr_len; +} + + + + +unsigned int +swf_unwrap(char* inbuf, int in_len, char* outbuf, int out_sz) { + char* tmp_buf; + int inf_len; + + tmp_buf = malloc(in_len * 8); + + inf_len = inf(inbuf + 8, in_len - 8, tmp_buf, in_len * 8); + + // fprintf(stderr, "in_swf_len = %d\n", in_len -8 ); + + + if (inf_len < 0 || out_sz < inf_len - SWF_SAVE_HEADER_LEN - SWF_SAVE_FOOTER_LEN) { + fprintf(stderr, "inf_len = %d\n", inf_len); + free(tmp_buf); + // buf_dump((unsigned char*) (inbuf+8), in_len -8, stderr); + + + + return -1; + } + + memcpy(outbuf, tmp_buf + SWF_SAVE_HEADER_LEN, inf_len - SWF_SAVE_HEADER_LEN - SWF_SAVE_FOOTER_LEN); + return inf_len - SWF_SAVE_HEADER_LEN - SWF_SAVE_FOOTER_LEN; +} + + + + + +int +x_http2_server_SWF_transmit (steg_t* s, struct evbuffer *source, conn_t *conn) { + + struct evbuffer *dest = conn_get_outbound(conn); + size_t sbuflen = evbuffer_get_length(source); + char* inbuf; + char* outbuf; + int outlen; + + + + inbuf = malloc(sbuflen); + + if (inbuf == NULL) { + log_warn("malloc inbuf failed\n"); + return -1; + } + + + if (evbuffer_remove(source, inbuf, sbuflen) == -1) { + log_debug("evbuffer_remove failed in x_http2_server_SWF_transmit"); + return -1; + } + + outbuf = malloc(4*sbuflen + SWF_SAVE_FOOTER_LEN + SWF_SAVE_HEADER_LEN + 512); + + if (outbuf == NULL) { + free(inbuf); + log_warn("malloc outbuf failed\n"); + return -1; + } + + // fprintf(stderr, "server wrapping swf len %d\n", (int) sbuflen); + outlen = swf_wrap(inbuf, sbuflen, outbuf, 4*sbuflen + SWF_SAVE_FOOTER_LEN + SWF_SAVE_HEADER_LEN + 512); + + if (outlen < 0) { + log_warn("swf_wrap failed\n"); + // fprintf(stderr, "swf_wrap failed\n"); + free(inbuf); + free(outbuf); + return -1; + } + + + if (evbuffer_add(dest, outbuf, outlen)) { + log_debug("SERVER ERROR: x_http2_server_transmit: evbuffer_add() fails for jsTemplate"); + free(inbuf); + free(outbuf); + return -1; + } + + + // conn_cease_transmission(conn); + conn_close_after_transmit(conn); + + + free(inbuf); + free(outbuf); + return 0; +} + + + + +int +x_http2_handle_client_SWF_receive(steg_t *s, conn_t *conn, struct evbuffer *dest, struct evbuffer* source) { + struct evbuffer_ptr s2; + unsigned int response_len = 0, hdrLen; + char outbuf[HTTP_MSG_BUF_SIZE]; + int content_len = 0, outbuflen; + char *httpHdr, *httpBody; + + + + s2 = evbuffer_search(source, "\r\n\r\n", sizeof ("\r\n\r\n") -1 , NULL); + if (s2.pos == -1) { + log_warn("CLIENT Did not find end of HTTP header %d", (int) evbuffer_get_length(source)); + fprintf(stderr, "client did not find end of HTTP header\n"); + + // evbuffer_dump(source, stderr); + return RECV_INCOMPLETE; + } + + log_debug("CLIENT received response header with len %d", (int)s2.pos); + + response_len = 0; + hdrLen = s2.pos + strlen("\r\n\r\n"); + response_len += hdrLen; + + httpHdr = (char *) evbuffer_pullup(source, s2.pos); + if (httpHdr == NULL) { + log_warn("CLIENT unable to pullup the complete HTTP header"); + return RECV_BAD; + } + + content_len = find_content_length(httpHdr, hdrLen); + + + + if (content_len < 0) { + log_warn("CLIENT unable to find content length"); + return RECV_BAD; + } + log_debug("CLIENT received Content-Length = %d\n", content_len); + + + + response_len += content_len; + + if (response_len > evbuffer_get_length(source)) + return RECV_INCOMPLETE; + + + + httpHdr = (char *) evbuffer_pullup(source, response_len); + httpBody = httpHdr + hdrLen; + + + outbuflen = swf_unwrap(httpBody, content_len, outbuf, HTTP_MSG_BUF_SIZE); + + if (outbuflen < 0) { + fprintf(stderr, "swf_unwrap failed\n"); + log_debug("CLIENT ERROR: swf_unwrap failed\n"); + return RECV_BAD; + } + + // fprintf(stderr, "CLIENT unwrapped data of length %d:", outbuflen); + // buf_dump(outbuf, outbuflen, stderr); + + if (evbuffer_add(dest, outbuf, outbuflen)) { + log_debug("CLIENT ERROR: evbuffer_add to dest fails\n"); + return RECV_BAD; + } + + // log_debug("Drained source for %d char\n", response_len); + if (evbuffer_drain(source, response_len) == -1) { + log_debug("CLIENT ERROR: failed to drain source\n"); + return RECV_BAD; + } + + // downcast_steg(s)->have_received = 1; + conn_expect_close(conn); + return RECV_GOOD; +} diff --git a/src/steg/swfSteg.c.old b/src/steg/swfSteg.c.old new file mode 100644 index 0000000..833ea9f --- /dev/null +++ b/src/steg/swfSteg.c.old @@ -0,0 +1,264 @@ +#include "swfSteg.h" + + +static const char http_response_1[] = + "HTTP/1.1 200 OK\r\n" + "Expires: Thu, 01 Jan 1970 00:00:00 GMT\r\n" + "Cache-Control: no-store\r\n" + "Connection: close\r\n" + "Content-Type: application/x-shockwave-flash\r\n" + "Content-Length: "; + + + + + + + + + + + +unsigned int +swf_wrap(char* inbuf, int in_len, char* outbuf, int out_sz) { + + char* swf; + int in_swf_len; + + char* tmp_buf; + int out_swf_len; + + char* resp; + int resp_len; + + char hdr[512]; + unsigned int hdr_len; + + char* tmp_buf2; + + + + if (!get_payload(HTTP_CONTENT_SWF, -1, &resp, &resp_len)) { + log_warn("swfsteg: no suitable payload found\n"); + return -1; + } + + swf = strstr(resp, "\r\n\r\n") + 4; + in_swf_len = resp_len - (swf - resp); + + + + + if (out_sz - in_len < (SWF_SAVE_FOOTER_LEN + SWF_SAVE_HEADER_LEN + 8 + 512)) { + fprintf(stderr, "swfsteg: outbuf too small %d \n", out_sz - in_len); + + log_warn("swfsteg: outbuf too small\n"); + return -1; + } + + + tmp_buf = malloc(in_len + SWF_SAVE_HEADER_LEN + SWF_SAVE_FOOTER_LEN); + + if (tmp_buf == NULL) { + log_warn("swfsteg: malloc failed\n"); + return -1; + } + + + tmp_buf2 = malloc(in_len + SWF_SAVE_HEADER_LEN + SWF_SAVE_FOOTER_LEN + 512); + + if (tmp_buf2 == NULL) { + free(tmp_buf); + log_warn("swfsteg: malloc failed\n"); + return -1; + } + + + memcpy(tmp_buf, swf+8, SWF_SAVE_HEADER_LEN); + memcpy(tmp_buf+SWF_SAVE_HEADER_LEN, inbuf, in_len); + memcpy(tmp_buf+SWF_SAVE_HEADER_LEN+in_len, swf + in_swf_len - SWF_SAVE_FOOTER_LEN, SWF_SAVE_FOOTER_LEN); + out_swf_len = def(tmp_buf, SWF_SAVE_HEADER_LEN + in_len + SWF_SAVE_FOOTER_LEN, tmp_buf2+8, + in_len + SWF_SAVE_HEADER_LEN + SWF_SAVE_FOOTER_LEN + 512-8, + Z_DEFAULT_COMPRESSION); + + // sprintf(hdr, "%s%d\r\n\r\n", http_response_1, out_swf_len + 8); + + + hdr_len = gen_response_header((char*) "application/x-shockwave-flash", 0, out_swf_len + 8, hdr, sizeof(hdr)); + + // fprintf(stderr, "hdr = %s\n", hdr); + + memcpy(tmp_buf2, swf, 4); + ((int*) (tmp_buf2))[1] = out_swf_len; + + memcpy(outbuf, hdr, hdr_len); + memcpy(outbuf+hdr_len, tmp_buf2, out_swf_len + 8); + + free(tmp_buf); + free(tmp_buf2); + return out_swf_len + 8 + hdr_len; +} + + + +unsigned int +swf_unwrap(char* inbuf, int in_len, char* outbuf, int out_sz) { + char* tmp_buf; + int inf_len; + + tmp_buf = malloc(in_len * 5); + inf_len = inf(inbuf + 8, in_len - 8, tmp_buf, in_len * 5); + + if (inf_len < 0 || out_sz < inf_len - SWF_SAVE_HEADER_LEN - SWF_SAVE_FOOTER_LEN) { + fprintf(stderr, "swf_unwrap failed \n"); + free(tmp_buf); + return -1; + } + + memcpy(outbuf, tmp_buf + SWF_SAVE_HEADER_LEN, inf_len - SWF_SAVE_HEADER_LEN - SWF_SAVE_FOOTER_LEN); + return inf_len - SWF_SAVE_HEADER_LEN - SWF_SAVE_FOOTER_LEN; +} + + + + + +int +x_http2_server_SWF_transmit (steg_t* s, struct evbuffer *source, conn_t *conn) { + + struct evbuffer *dest = conn_get_outbound(conn); + size_t sbuflen = evbuffer_get_length(source); + char* inbuf; + char* outbuf; + int outlen; + + + + inbuf = malloc(sbuflen); + + if (inbuf == NULL) { + log_warn("malloc inbuf failed\n"); + return -1; + } + + + if (evbuffer_remove(source, inbuf, sbuflen) == -1) { + log_debug("evbuffer_remove failed in x_http2_server_SWF_transmit"); + return -1; + } + + outbuf = malloc(4*sbuflen + SWF_SAVE_FOOTER_LEN + SWF_SAVE_HEADER_LEN + 512); + + if (outbuf == NULL) { + free(inbuf); + log_warn("malloc outbuf failed\n"); + return -1; + } + + // fprintf(stderr, "server wrapping swf len %d\n", (int) sbuflen); + outlen = swf_wrap(inbuf, sbuflen, outbuf, 4*sbuflen + SWF_SAVE_FOOTER_LEN + SWF_SAVE_HEADER_LEN + 512); + + if (outlen < 0) { + log_warn("swf_wrap failed\n"); + fprintf(stderr, "swf_wrap failed\n"); + free(inbuf); + free(outbuf); + return -1; + } + + + if (evbuffer_add(dest, outbuf, outlen)) { + log_debug("SERVER ERROR: x_http2_server_transmit: evbuffer_add() fails for jsTemplate"); + free(inbuf); + free(outbuf); + return -1; + } + + + // conn_cease_transmission(conn); + conn_close_after_transmit(conn); + + + free(inbuf); + free(outbuf); + return 0; +} + + + + +int +x_http2_handle_client_SWF_receive(steg_t *s, conn_t *conn, struct evbuffer *dest, struct evbuffer* source) { + struct evbuffer_ptr s2; + unsigned int response_len = 0, hdrLen; + char outbuf[HTTP_MSG_BUF_SIZE]; + int content_len = 0, outbuflen; + char *httpHdr, *httpBody; + + + + s2 = evbuffer_search(source, "\r\n\r\n", sizeof ("\r\n\r\n") -1 , NULL); + if (s2.pos == -1) { + log_warn("CLIENT Did not find end of HTTP header %d", (int) evbuffer_get_length(source)); + fprintf(stderr, "client did not find end of HTTP header\n"); + + // evbuffer_dump(source, stderr); + return RECV_INCOMPLETE; + } + + log_debug("CLIENT received response header with len %d", (int)s2.pos); + + response_len = 0; + hdrLen = s2.pos + strlen("\r\n\r\n"); + response_len += hdrLen; + + httpHdr = (char *) evbuffer_pullup(source, s2.pos); + if (httpHdr == NULL) { + log_warn("CLIENT unable to pullup the complete HTTP header"); + return RECV_BAD; + } + + content_len = find_content_length(httpHdr, hdrLen); + + + + if (content_len < 0) { + log_warn("CLIENT unable to find content length"); + return RECV_BAD; + } + log_debug("CLIENT received Content-Length = %d\n", content_len); + + + + response_len += content_len; + + if (response_len > evbuffer_get_length(source)) + return RECV_INCOMPLETE; + + httpBody = httpHdr + hdrLen; + outbuflen = swf_unwrap(httpBody, content_len, outbuf, HTTP_MSG_BUF_SIZE); + + if (outbuflen < 0) { + fprintf(stderr, "swf_unwrap failed\n"); + log_debug("CLIENT ERROR: swf_unwrap failed\n"); + return RECV_BAD; + } + + // fprintf(stderr, "CLIENT unwrapped data of length %d:", outbuflen); + // buf_dump(outbuf, outbuflen, stderr); + + if (evbuffer_add(dest, outbuf, outbuflen)) { + log_debug("CLIENT ERROR: evbuffer_add to dest fails\n"); + return RECV_BAD; + } + + // log_debug("Drained source for %d char\n", response_len); + if (evbuffer_drain(source, response_len) == -1) { + log_debug("CLIENT ERROR: failed to drain source\n"); + return RECV_BAD; + } + + // downcast_steg(s)->have_received = 1; + conn_expect_close(conn); + return RECV_GOOD; +} diff --git a/src/steg/swfSteg.h b/src/steg/swfSteg.h new file mode 100644 index 0000000..dc6bc04 --- /dev/null +++ b/src/steg/swfSteg.h @@ -0,0 +1,42 @@ +#ifndef _SWFSTEG_H +#define _SWFSTEG_H + + +#include "util.h" +#include "connections.h" +#include "steg.h" +#include "payloads.h" +#include "cookies.h" +#include "pdfSteg.h" +#include "zpack.h" + + +#include <event2/buffer.h> +#include <stdio.h> + + + + + + + +#define SWF_SAVE_HEADER_LEN 1500 +#define SWF_SAVE_FOOTER_LEN 1500 + + +unsigned int +swf_wrap(char* inbuf, int in_len, char* outbuf, int out_sz); + +unsigned int +swf_unwrap(char* inbuf, int in_len, char* outbuf, int out_sz); + +int +x_http2_server_SWF_transmit (steg_t* s, struct evbuffer *source, conn_t *conn); + + +int +x_http2_handle_client_SWF_receive(steg_t *s, conn_t *conn, struct evbuffer *dest, struct evbuffer* source); + +#endif + + diff --git a/src/steg/x_http.c.old b/src/steg/x_http.c.old new file mode 100644 index 0000000..2799078 --- /dev/null +++ b/src/steg/x_http.c.old @@ -0,0 +1,337 @@ +/* Copyright 2011 Zack Weinberg + See LICENSE for other credits and copying information +*/ + +#include "util.h" +#include "connections.h" +#include "steg.h" + +#include <event2/buffer.h> + + + +/* This is an example steganography module. Don't use it to disguise real + traffic! It packages client->server traffic as HTTP GET requests and + server->client traffic as HTTP responses, but makes no actual attempt + to obscure the data proper. */ + +struct x_http_steg_t +{ + steg_t super; + /* no extra stuff is presently necessary */ +}; + +STEG_DEFINE_MODULE(x_http, + 1024, /* client-server max data rate - made up */ + 10240, /* server-client max data rate - ditto */ + 1, /* max concurrent connections per IP */ + 1); /* max concurrent IPs */ + +/* Canned HTTP query and response headers. */ +static const char http_query_1[] = + "GET /"; +static const char http_query_2[] = + " HTTP/1.1\r\n" + "Host: "; +static const char http_query_3[] = + "\r\n" + "Connection: close\r\n\r\n"; + +static const char http_response_1[] = + "HTTP/1.1 200 OK\r\n" + "Expires: Thu, 01 Jan 1970 00:00:00 GMT\r\n" + "Cache-Control: no-store\r\n" + "Connection: close\r\n" + "Content-Type: application/octet-stream\r\n" + "Content-Length: "; +static const char http_response_2[] = + "%lu\r\n" + "\r\n"; + + +steg_t * +x_http_new(rng_t *rng, unsigned int is_clientside) +{ + STEG_NEW(x_http, state, rng, is_clientside); + /* if there were extra stuff to fill in, you would do it here */ + return upcast_steg(state); +} + +void +x_http_del(steg_t *s) +{ + x_http_steg_t *state = downcast_steg(s); + STEG_DEL(s); + /* if there were extra stuff to deallocate, you would do it here */ + free(state); +} + +unsigned int +x_http_detect(conn_t *conn) +{ + struct evbuffer *buf = conn_get_inbound(conn); + unsigned char *data; + + return 0; + + /* Look for the text of http_response_1. */ + if (evbuffer_get_length(buf) >= sizeof http_response_1 - 1) { + data = evbuffer_pullup(buf, sizeof http_response_1 - 1); + if (!memcmp(data, http_response_1, sizeof http_response_1 - 1)) + return 1; + } + + /* The client always transmits "GET /" followed by at least four + characters that are either lowercase hex digits or equals + signs, so we need nine bytes of incoming data. */ + if (evbuffer_get_length(buf) >= 9) { + data = evbuffer_pullup(buf, 9); + if (!memcmp(data, "GET /", 5) && + (ascii_isxdigit(data[5]) || data[5] == '=') && + (ascii_isxdigit(data[6]) || data[6] == '=') && + (ascii_isxdigit(data[7]) || data[7] == '=') && + (ascii_isxdigit(data[8]) || data[8] == '=')) + return 1; + } + + /* Didn't find either the client or the server pattern. */ + return 0; +} + +size_t +x_http_transmit_room(steg_t *s, conn_t *conn) +{ + if (s->is_clientside) + /* per http://www.boutell.com/newfaq/misc/urllength.html, + IE<9 can handle no more than 2048 characters in the path + component of a URL; we're not talking to IE, but this limit + means longer paths look fishy; we hex-encode the path, so + we have to cut the number in half. */ + return 1024; + else + /* no practical limit applies */ + return SIZE_MAX; +} + +int +x_http_transmit(steg_t *s, struct evbuffer *source, conn_t *conn) +{ + struct evbuffer *dest = conn_get_outbound(conn); + + if (s->is_clientside) { + /* On the client side, we have to embed the data in a GET query somehow; + the only plausible places to put it are the URL and cookies. This + presently uses the URL. And it can't be binary. */ + struct evbuffer *scratch; + struct evbuffer_iovec *iv; + int i, nv; + + /* Convert all the data in 'source' to hexadecimal and write it to + 'scratch'. Data is padded to a multiple of four characters with + equals signs. */ + size_t slen = evbuffer_get_length(source); + size_t dlen = slen * 2; + + dlen = dlen + 3 - (dlen-1)%4; + if (dlen == 0) dlen = 4; + + scratch = evbuffer_new(); + if (!scratch) return -1; + if (evbuffer_expand(scratch, dlen)) { + evbuffer_free(scratch); + return -1; + } + + nv = evbuffer_peek(source, slen, NULL, NULL, 0); + iv = xzalloc(sizeof(struct evbuffer_iovec) * nv); + if (evbuffer_peek(source, slen, NULL, iv, nv) != nv) { + evbuffer_free(scratch); + free(iv); + return -1; + } + + for (i = 0; i < nv; i++) { + const unsigned char *p = iv[i].iov_base; + const unsigned char *limit = p + iv[i].iov_len; + char hex[2], c; + while (p < limit) { + c = *p++; + hex[0] = "0123456789abcdef"[(c & 0xF0) >> 4]; + hex[1] = "0123456789abcdef"[(c & 0x0F) >> 0]; + evbuffer_add(scratch, hex, 2); + } + } + free(iv); + while (evbuffer_get_length(scratch) == 0 || + evbuffer_get_length(scratch) % 4 != 0) + evbuffer_add(scratch, "=", 1); + + if (evbuffer_add(dest, http_query_1, sizeof http_query_1-1) || + evbuffer_add_buffer(dest, scratch) || + evbuffer_add(dest, http_query_2, sizeof http_query_2-1) || + evbuffer_add(dest, conn->peername, strlen(conn->peername)) || + evbuffer_add(dest, http_query_3, sizeof http_query_3-1)) { + evbuffer_free(scratch); + return -1; + } + + evbuffer_free(scratch); + evbuffer_drain(source, slen); + conn_cease_transmission(conn); + return 0; + + } else { + /* On the server side, we just fake up some HTTP response headers and + then splat the data we were given. Binary is OK. */ + + if (evbuffer_add(dest, http_response_1, sizeof http_response_1-1)) + return -1; + if (evbuffer_add_printf(dest, http_response_2, + (unsigned long)evbuffer_get_length(source)) == -1) + return -1; + if (evbuffer_add_buffer(dest, source)) + return -1; + + conn_close_after_transmit(conn); + return 0; + } +} + +// enum recv_ret +static int +x_http_receive(steg_t *s, conn_t *conn, struct evbuffer *dest) +{ + struct evbuffer *source = conn_get_inbound(conn); + if (s->is_clientside) { + /* This loop should not be necessary, but we are currently not + enforcing the query-response pattern, so we can get more than + one response per request. */ + do { + /* Linearize the buffer out past the longest possible + Content-Length header and subsequent blank line. 2**64 fits in + 20 characters, and then we have two CRLFs; minus one for the + NUL in sizeof http_response_1. Note that this does _not_ + guarantee that that much data is available. */ + + unsigned char *data = evbuffer_pullup(source, sizeof http_response_1 + 23); + size_t hlen = evbuffer_get_length(source); + if (hlen > sizeof http_response_1 + 23) + hlen = sizeof http_response_1 + 23; + + /* Validate response headers. */ + if (hlen < sizeof http_response_1 - 1) + return 0; // RECV_INCOMPLETE; + if (memcmp(data, http_response_1, sizeof http_response_1 - 1)) + return -1; // RECV_BAD; + + /* There should be an unsigned number immediately after the text of + http_response_1, followed by the four characters \r\n\r\n. + We may not have the complete number yet. */ + unsigned char *p = data + sizeof http_response_1 - 1; + unsigned char *limit = data + hlen; + uint64_t clen = 0; + while (p < limit && '0' <= *p && *p <= '9') { + clen = clen*10 + *p - '0'; + p++; + } + if (p+4 > limit) + return 0; // RECV_INCOMPLETE; + if (p[0] != '\r' || p[1] != '\n' || p[2] != '\r' || p[3] != '\n') + return -1; // RECV_BAD; + + p += 4; + hlen = p - data; + /* Now we know how much data we're expecting after the blank line. */ + if (evbuffer_get_length(source) < hlen + clen) + return 0; // RECV_INCOMPLETE; + + /* we are go */ + if (evbuffer_drain(source, hlen)) + return -1; // RECV_BAD; + + if (evbuffer_remove_buffer(source, dest, clen) != clen) + return -1; // RECV_BAD; + + } while (evbuffer_get_length(source)); + + conn_expect_close(conn); + return 0; // RECV_GOOD; + } else { + /* We need a scratch buffer here because the contract is that if + we hit a decode error we *don't* write anything to 'dest'. */ + struct evbuffer *scratch; + + /* This loop should not be necessary either, but is, for the same + reason given above */ + do { + /* Search for the second and third invariant bits of the query headers + we expect. We completely ignore the contents of the Host header. */ + struct evbuffer_ptr s2 = evbuffer_search(source, http_query_2, + sizeof http_query_2 - 1, + NULL); + if (s2.pos == -1) { + log_debug("Did not find second piece of HTTP query"); + return 0; // RECV_INCOMPLETE; + } + struct evbuffer_ptr s3 = evbuffer_search(source, http_query_3, + sizeof http_query_3 - 1, + &s2); + if (s3.pos == -1) { + log_debug("Did not find third piece of HTTP query"); + return 0; // RECV_INCOMPLETE; + } + obfs_assert(s3.pos + sizeof http_query_3 - 1 + <= evbuffer_get_length(source)); + + unsigned char *data = evbuffer_pullup(source, s2.pos); + if (memcmp(data, "GET /", sizeof "GET /"-1)) { + log_debug("Unexpected HTTP verb: %.*s", 5, data); + return -1; // RECV_BAD; + } + + unsigned char *p = data + sizeof "GET /"-1; + unsigned char *limit = data + s2.pos; + + scratch = evbuffer_new(); + if (!scratch) return -1; // RECV_BAD; + if (evbuffer_expand(scratch, (limit - p)/2)) { + evbuffer_free(scratch); + return -1; // RECV_BAD; + } + + unsigned char c, h, secondhalf = 0; + while (p < limit) { + if (!secondhalf) c = 0; + if ('0' <= *p && *p <= '9') h = *p - '0'; + else if ('a' <= *p && *p <= 'f') h = *p - 'a' + 10; + else if ('A' <= *p && *p <= 'F') h = *p - 'A' + 10; + else if (*p == '=' && !secondhalf) { + p++; + continue; + } else { + evbuffer_free(scratch); + log_debug("Decode error: unexpected URI character %c", *p); + return -1; // RECV_BAD; + } + + c = (c << 4) + h; + if (secondhalf) + evbuffer_add(scratch, &c, 1); + secondhalf = !secondhalf; + p++; + } + + if (evbuffer_add_buffer(dest, scratch)) { + evbuffer_free(scratch); + log_debug("Failed to transfer buffer"); + return -1; // RECV_BAD; + } + evbuffer_drain(source, s3.pos + sizeof http_query_3 - 1); + evbuffer_free(scratch); + + } while (evbuffer_get_length(source)); + + conn_transmit_soon(conn, 100); + return 0; // RECV_GOOD; + } +} diff --git a/src/steg/x_http2.c b/src/steg/x_http2.c new file mode 100644 index 0000000..0710fa7 --- /dev/null +++ b/src/steg/x_http2.c @@ -0,0 +1,700 @@ +/* Copyright (c) 2011, SRI International + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + + * Neither the names of the copyright owners nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + Contributors: Zack Weinberg, Vinod Yegneswaran + See LICENSE for other credits and copying information +*/ + + + +#include "util.h" +#include "connections.h" +#include "steg.h" +#include "payloads.h" +#include "cookies.h" +#include "swfSteg.h" +#include "pdfSteg.h" +#include "jsSteg.h" + +#include <event2/buffer.h> +#include <stdio.h> + + + + +#define MIN_COOKIE_SIZE 128 +#define MAX_COOKIE_SIZE 2048 + + +int +x_http2_server_receive(steg_t *s, conn_t *conn, struct evbuffer *dest, struct evbuffer* source); + +int +lookup_peer_name_from_ip(char* p_ip, char* p_name); + + +static int has_peer_name = 0; +static char peername[512]; + + +struct x_http2_steg_t +{ + steg_t super; + + int have_transmitted; + int have_received; + int type; +}; + + +STEG_DEFINE_MODULE(x_http2, + 1024, /* client-server max data rate - made up */ + 10240, /* server-client max data rate - ditto */ + 1, /* max concurrent connections per IP */ + 1); /* max concurrent IPs */ + + + + + + +int x_http2_client_transmit (steg_t *s, struct evbuffer *source, conn_t *conn); + +void evbuffer_dump(struct evbuffer *buf, FILE *out); +void buf_dump(unsigned char* buf, int len, FILE *out); + + + +void +evbuffer_dump(struct evbuffer *buf, FILE *out) +{ + int nextent = evbuffer_peek(buf, SSIZE_MAX, 0, 0, 0); + struct evbuffer_iovec v[nextent]; + int i; + const unsigned char *p, *limit; + + if (evbuffer_peek(buf, -1, 0, v, nextent) != nextent) + abort(); + + for (i = 0; i < nextent; i++) { + p = v[i].iov_base; + limit = p + v[i].iov_len; + + putc('|', out); + while (p < limit) { + if (*p < 0x20 || *p >= 0x7F || *p == '\\' || *p == '|') + fprintf(out, "\\x%02x", *p); + else + putc(*p, out); + p++; + } + } + putc('|', out); +} + + + + + +void +buf_dump(unsigned char* buf, int len, FILE *out) +{ + int i=0; + putc('|', out); + while (i < len) { + if (buf[i] < 0x20 || buf[i] >= 0x7F || buf[i] == '\\' || buf[i]== '|') + fprintf(out, "\\x%02x", buf[i]); + else + putc(buf[i], out); + i++; + } + putc('|', out); + putc('\n', out); +} + + + + + +steg_t * +x_http2_new(rng_t *rng, unsigned int is_clientside) +{ + + STEG_NEW(x_http2, state, rng, is_clientside); + + if (is_clientside) + load_payloads("traces/client.out"); + else { + load_payloads("traces/server.out"); + init_JS_payload_pool(HTTP_MSG_BUF_SIZE, TYPE_HTTP_RESPONSE, JS_MIN_AVAIL_SIZE); + // init_PDF_payload_pool(HTTP_MSG_BUF_SIZE, TYPE_HTTP_RESPONSE, PDF_MIN_AVAIL_SIZE); + init_SWF_payload_pool(HTTP_MSG_BUF_SIZE, TYPE_HTTP_RESPONSE, 0); + } + + + /* if there were extra stuff to fill in, you would do it here */ + return upcast_steg(state); +} + +void +x_http2_del(steg_t *s) +{ + x_http2_steg_t *state = downcast_steg(s); + + STEG_DEL(s); + + /* if there were extra stuff to deallocate, you would do it here */ + free(state); +} + + +// x_http2_detect determines if a packet should be processed by the http2 steg module +unsigned int +x_http2_detect(conn_t *conn) +{ + struct evbuffer *buf = conn_get_inbound(conn); + unsigned char *data; + + // return 0; +/***** + Here is a list of HTTP response codes extracted from the server-portals.out trace + +7369 HTTP/1.1 200 OK + 470 HTTP/1.1 302 Found + 350 HTTP/1.1 304 Not Modified + 212 HTTP/1.1 302 Moved Temporarily + 184 HTTP/1.1 204 No Content + 451 HTTP/1.0 200 OK + 36 HTTP/1.0 204 No Content + 21 HTTP/1.1 301 Moved Permanently + 19 HTTP/1.1 302 Object moved + 15 HTTP/1.1 404 Not Found + + 7 HTTP/1.0 304 Not Modified + 6 HTTP/1.1 302 Redirect + 3 HTTP/1.0 200 Ok + 2 HTTP/1.1 303 Object Moved + 2 HTTP/1.0 301 Moved Permanently + 2 HTTP/1.0 302 Moved Temporarily + 2 HTTP/1.0 400 Bad request + 2 HTTP/1.0 403 Forbidden + 1 HTTP/1.0 404 Not Found + 1 HTTP/1.1 200 + 1 HTTP/1.1 302 FOUND + 1 HTTP/1.1 304 + 1 HTTP/1.1 400 Bad Request + 1 HTTP/1.1 403 Forbidden + 1 HTTP/1.1 503 Service Unavailable. + *****/ + + // The first part of a valid HTTP response should be of the form + // HTTP/1.x nnn + + if (evbuffer_get_length(buf) >= 12) { + data = evbuffer_pullup(buf, 12); + + if (data != NULL && + ((!memcmp(data, "HTTP/1.1 200", 12)) || + (!memcmp(data, "HTTP/1.1 302", 12)) || + (!memcmp(data, "HTTP/1.1 304", 12)) || + (!memcmp(data, "HTTP/1.1 204", 12)) || + (!memcmp(data, "HTTP/1.0 200", 12)) || + (!memcmp(data, "HTTP/1.0 204", 12)) || + (!memcmp(data, "HTTP/1.1 301", 12)) || + (!memcmp(data, "HTTP/1.1 302", 12)) || + (!memcmp(data, "HTTP/1.1 404", 12)))) { + log_debug("x_http2_detect: valid response"); + return 1; + } + } + + + + + + // SC: if we are only interested in jsSteg, we may want to + // consider HTTP/1.1 and HTTP/1.0 responses whose code is 200 only + + // check to see if this is a valid HTTP request + // + // the following is for HTTP requests used by the http2 steg module + // The client always transmits "GET /" followed by at least four + // characters that are either lowercase hex digits or equals + // signs, so we need nine bytes of incoming data. + + + + if (evbuffer_get_length(buf) >= 9) { + data = evbuffer_pullup(buf, 9); + if (data != NULL && (!memcmp(data, "GET /", 5) || !memcmp(data, "POST /", 5) || !memcmp(data, "Cookie", 6))) { + log_debug("x_http2_detect: valid request"); + return 1; + } + } + + log_debug("x_http2_detect: didn't find either HTTP request or response"); + /* Didn't find either the client or the server pattern. */ + return 0; +} + +size_t +x_http2_transmit_room(steg_t *s, conn_t *conn) +{ + unsigned int mjc; + + if (downcast_steg(s)->have_transmitted) + /* can't send any more on this connection */ + return 0; + + + if (s->is_clientside) { + /* per http://www.boutell.com/newfaq/misc/urllength.html, + IE<9 can handle no more than 2048 characters in the path + component of a URL; we're not talking to IE, but this limit + means longer paths look fishy; we hex-encode the path, so + we have to cut the number in half. */ + return (MIN_COOKIE_SIZE + rand() % (MAX_COOKIE_SIZE - MIN_COOKIE_SIZE)) / 4; + // return 1024; + } + else { + + if (!downcast_steg(s)->have_received) + return 0; + + switch(downcast_steg(s)->type) { + + case HTTP_CONTENT_SWF: + return 1024; + + case HTTP_CONTENT_JAVASCRIPT: + mjc = get_max_JS_capacity() / 2; + if (mjc > 1024) { + // it should be 1024 + ...., but seems like we need to be a little bit smaller (chopper bug?) + int rval = 512 + rand()%(mjc - 1024); + // fprintf(stderr, "returning rval %d, mjc %d\n", rval, mjc); + return rval; + } + log_warn("js capacity too small\n"); + exit(-1); + + case HTTP_CONTENT_PDF: + // return 1024 + rand()%(get_max_PDF_capacity() - 1024) + return PDF_MIN_AVAIL_SIZE; + } + + return SIZE_MAX; + } +} + + + + + + +int +lookup_peer_name_from_ip(char* p_ip, char* p_name) { + struct addrinfo* ailist; + struct addrinfo* aip; + struct addrinfo hint; + char buf[128]; + + hint.ai_flags = AI_CANONNAME; + hint.ai_family = 0; + hint.ai_socktype = 0; + hint.ai_protocol = 0; + hint.ai_addrlen = 0; + hint.ai_canonname = NULL; + hint.ai_addr = NULL; + hint.ai_next = NULL; + + strcpy(buf, p_ip); + buf[strchr(buf, ':') - buf] = 0; + + + if (getaddrinfo(buf, NULL, &hint, &ailist)) { + fprintf(stderr, "error: getaddrinfo() %s\n", p_ip); + exit(1); + } + + for (aip = ailist; aip != NULL; aip = aip->ai_next) { + char buf[512]; + if (getnameinfo(aip->ai_addr, sizeof(struct sockaddr), buf, 512, NULL, 0, 0) == 0) { + sprintf(p_name, "%s", buf); + return 1; + } + } + + return 0; +} + + + + + + + + +int +x_http2_client_transmit (steg_t *s, struct evbuffer *source, conn_t *conn) { + + /* On the client side, we have to embed the data in a GET query somehow; + the only plausible places to put it are the URL and cookies. This + presently uses the URL. And it can't be binary. */ + // struct evbuffer *scratch; + struct evbuffer_iovec *iv; + int i, nv; + struct evbuffer *dest = conn_get_outbound(conn); + size_t sbuflen = evbuffer_get_length(source); + char buf[10000]; + unsigned char data[(int) sbuflen*2]; + // unsigned char outbuf[MAX_COOKIE_SIZE]; + + unsigned char outbuf[(int) sbuflen*8]; + int datalen; + + + // size_t sofar = 0; + size_t cookie_len; + + + /* Convert all the data in 'source' to hexadecimal and write it to + 'scratch'. Data is padded to a multiple of four characters with + equals signs. */ + + + unsigned int len = 0; + unsigned int cnt = 0; + + + + datalen = 0; + cookie_len = 4 * sbuflen + rand() % 4; + + + nv = evbuffer_peek(source, sbuflen, NULL, NULL, 0); + iv = xzalloc(sizeof(struct evbuffer_iovec) * nv); + + if (evbuffer_peek(source, sbuflen, NULL, iv, nv) != nv) { + free(iv); + return -1; + } + + // retry up to 10 times + while (!len) { + len = find_client_payload(buf, sizeof(buf), TYPE_HTTP_REQUEST); + if (cnt++ == 10) return -1; + } + + + if (has_peer_name == 0 && lookup_peer_name_from_ip((char*) conn->peername, peername)) + has_peer_name = 1; + + // if (find_uri_type(buf) != HTTP_CONTENT_SWF) { + // fprintf(stderr, "%s\n", buf); + // exit(-1); + // } + + + + cnt = 0; + + for (i = 0; i < nv; i++) { + const unsigned char *p = iv[i].iov_base; + const unsigned char *limit = p + iv[i].iov_len; + char c; + while (p < limit && cnt < sbuflen) { + c = *p++; + data[datalen] = "0123456789abcdef"[(c & 0xF0) >> 4]; + data[datalen+1] = "0123456789abcdef"[(c & 0x0F) >> 0]; + datalen += 2; + cnt++; + } + } + + free(iv); + + if (cookie_len < 4) cookie_len = 4; + + datalen = gen_cookie_field(outbuf, cookie_len, data, datalen); + log_debug("CLIENT: sending cookie of length = %d %d\n", datalen, (int) cookie_len); + // fprintf(stderr, "CLIENT: sending cookie of length = %d %d\n", datalen, (int) cookie_len); + + if (datalen < 0) { + log_debug("cookie generation failed\n"); + return -1; + } + + + if (evbuffer_add(dest, buf, strstr(buf, "\r\n") - buf + 2) || // add uri field + evbuffer_add(dest, "Host: ", 6) || + evbuffer_add(dest, peername, strlen(peername)) || + evbuffer_add(dest, strstr(buf, "\r\n"), len - (unsigned int) (strstr(buf, "\r\n") - buf)) || // add everything but first line + evbuffer_add(dest, "Cookie: ", 8) || + evbuffer_add(dest, outbuf, cookie_len) || + evbuffer_add(dest, "\r\n\r\n", 4)) { + log_debug("error ***********************"); + return -1; + } + + // sofar += datalen/2; + evbuffer_drain(source, datalen/2); + + log_debug("CLIENT TRANSMITTED payload %d\n", (int) sbuflen); + + conn_cease_transmission(conn); + + downcast_steg(s)->type = find_uri_type(buf); + downcast_steg(s)->have_transmitted = 1; + return 0; +} + + + + + + + + + + + + + + + + + + + + +int +x_http2_transmit(steg_t *s, struct evbuffer *source, conn_t *conn) +{ + // struct evbuffer *dest = conn_get_outbound(conn); + + // fprintf(stderr, "in x_http2_ transmit %d\n", downcast_steg(s)->type); + + + + if (s->is_clientside) { + /* On the client side, we have to embed the data in a GET query somehow; + the only plausible places to put it are the URL and cookies. This + presently uses the URL. And it can't be binary. */ + + return x_http2_client_transmit(s, source, conn); //@@ + } + else { + int rval = -1; + switch(downcast_steg(s)->type) { + + case HTTP_CONTENT_SWF: + rval = x_http2_server_SWF_transmit(s, source, conn); + break; + case HTTP_CONTENT_JAVASCRIPT: + rval = x_http2_server_JS_transmit(s, source, conn); + break; + + case HTTP_CONTENT_PDF: + rval = x_http2_server_PDF_transmit(s, source, conn); + break; + } + + if (rval == 0) downcast_steg(s)->have_transmitted = 1; + return rval; + } +} + + + + + + +int +x_http2_server_receive(steg_t *s, conn_t *conn, struct evbuffer *dest, struct evbuffer* source) { + + int cnt = 0; + unsigned char* data; + int type; + + do { + struct evbuffer_ptr s2 = evbuffer_search(source, "\r\n\r\n", sizeof ("\r\n\r\n") -1 , NULL); + unsigned char* limit; + unsigned char *p; + int unwrapped_cookie_len; + struct evbuffer *scratch; + unsigned char c, h, secondhalf; + unsigned char buf[evbuffer_get_length(source)]; + + + if (s2.pos == -1) { + log_debug("Did not find end of request %d", (int) evbuffer_get_length(source)); + // evbuffer_dump(source, stderr); + return RECV_INCOMPLETE; + } + + log_debug("SERVER received request header of length %d", (int)s2.pos); + + data = evbuffer_pullup(source, s2.pos); + if (data == NULL) { + log_debug("SERVER evbuffer_pullup fails"); + return RECV_BAD; + } + + limit = data + s2.pos; + + type = find_uri_type((char *)data); + log_warn ("*** Got type %d", type); + + /* if (type != 3) { + fprintf(stderr, "type != 3, %d, data = %s \n", find_uri_type2((char *) data), data); + exit(-1); + }*/ + + data = (unsigned char*) strstr((char*) data, "Cookie:"); + + if (data == NULL || memcmp(data, "Cookie:", sizeof "Cookie:"-1)) { + log_debug("Unexpected HTTP verb: %.*s", 5, data); + return RECV_BAD; + } + + p = data + sizeof "Cookie: "-1; + unwrapped_cookie_len = unwrap_cookie(p, buf, (int) (limit - p)); + + log_debug("SERVER: received cookie of length = %d %d\n", unwrapped_cookie_len, (int) (limit-p)); + // buf_dump(buf, unwrapped_cookie_len, stderr); + // fprintf(stderr, "==========================\n"); + // buf_dump(p, (int) (limit-p), stderr); + + + // log_debug("hello SERVER received %d cnt = %d\n", (int) (limit - p), cnt); + // buf_dump(p, (int) (limit-p), stderr); + + /* We need a scratch buffer here because the contract is that if + we hit a decode error we *don't* write anything to 'dest'. */ + scratch = evbuffer_new(); + + if (!scratch) return RECV_BAD; + + + if (evbuffer_expand(scratch, unwrapped_cookie_len/2)) { + log_debug("Evbuffer expand failed \n"); + evbuffer_free(scratch); + return RECV_BAD; + } + p = buf; + + + secondhalf = 0; + while ((int) (p - buf) < unwrapped_cookie_len) { + if (!secondhalf) c = 0; + if ('0' <= *p && *p <= '9') h = *p - '0'; + else if ('a' <= *p && *p <= 'f') h = *p - 'a' + 10; + else if ('A' <= *p && *p <= 'F') h = *p - 'A' + 10; + else if (*p == '=' && !secondhalf) { + p++; + continue; + } else { + evbuffer_free(scratch); + log_debug("Decode error: unexpected URI characterasdfaf %d", *p); + return RECV_BAD; + } + + c = (c << 4) + h; + if (secondhalf) { + evbuffer_add(scratch, &c, 1); + // log_debug("adding to scratch"); + cnt++; + } + secondhalf = !secondhalf; + p++; + } + + + + if (evbuffer_add_buffer(dest, scratch)) { + evbuffer_free(scratch); + log_debug("Failed to transfer buffer"); + return RECV_BAD; + } + evbuffer_drain(source, s2.pos + sizeof("\r\n\r\n") - 1); + evbuffer_free(scratch); + } while (evbuffer_get_length(source)); + + + downcast_steg(s)->have_received = 1; + downcast_steg(s)->type = type; + // fprintf(stderr, "SERVER RECEIVED payload %d %d\n", cnt, type); + + conn_transmit_soon(conn, 100); + return RECV_GOOD; +} + + + + + + + + +static int +x_http2_receive(steg_t *s, conn_t *conn, struct evbuffer *dest) +{ + struct evbuffer *source = conn_get_inbound(conn); + // unsigned int type; + int rval = RECV_BAD; + + + if (s->is_clientside) { + + // fprintf(stderr, "client type = %d\n", downcast_steg(s)->type); + + switch(downcast_steg(s)->type) { + + case HTTP_CONTENT_SWF: + rval = x_http2_handle_client_SWF_receive(s, conn, dest, source); + break; + case HTTP_CONTENT_JAVASCRIPT: + rval = x_http2_handle_client_JS_receive(s, conn, dest, source); + break; + + case HTTP_CONTENT_PDF: + rval = x_http2_handle_client_PDF_receive(s, conn, dest, source); + break; + } + + if (rval == RECV_GOOD) downcast_steg(s)->have_received = 1; + return rval; + + } else { + return x_http2_server_receive(s, conn, dest, source); + } + + +} diff --git a/src/steg/zpack.c b/src/steg/zpack.c new file mode 100644 index 0000000..57cd2b2 --- /dev/null +++ b/src/steg/zpack.c @@ -0,0 +1,408 @@ +#include <stdio.h> +#include <string.h> +#include <assert.h> +#include <time.h> +#include <stdlib.h> +#include "zlib.h" +#include "zpack.h" + + +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(__CYGWIN__) +# include <fcntl.h> +# include <io.h> +# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) +#else +# define SET_BINARY_MODE(file) +#endif + +#define CHUNK 16384 + +/* Compress from file source to file dest until EOF on source. + def() returns Z_OK on success, Z_MEM_ERROR if memory could not be + allocated for processing, Z_STREAM_ERROR if an invalid compression + level is supplied, Z_VERSION_ERROR if the version of zlib.h and the + version of the library linked do not match, or Z_ERRNO if there is + an error reading or writing the files. */ + + +int def(char *source, int slen, char *dest, int dlen, int level) +{ + int ret, flush; + unsigned have; + z_stream strm; + unsigned char in[CHUNK]; + unsigned char out[CHUNK]; + int dlen_orig = dlen; + + /* allocate deflate state */ + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + ret = deflateInit(&strm, level); + if (ret != Z_OK) + return ret; + + /* compress until end of file */ + do { + + if (slen > CHUNK) + strm.avail_in = CHUNK; + else + strm.avail_in = slen; + + memcpy (in, source, strm.avail_in); + slen = slen - strm.avail_in; + source - source + strm.avail_in; + + flush = (slen == 0) ? Z_FINISH : Z_NO_FLUSH; + strm.next_in = in; + + /* run deflate() on input until output buffer not full, finish + compression if all of source has been read in */ + do { + strm.avail_out = CHUNK; + strm.next_out = out; + ret = deflate(&strm, flush); /* no bad return value */ + assert(ret != Z_STREAM_ERROR); /* state not clobbered */ + have = CHUNK - strm.avail_out; + + if ((unsigned int) dlen < have) { + fprintf(stderr, "dest buf too small!\n"); + return Z_ERRNO; + } + + memcpy(dest, out, have); + dest += have; + dlen = dlen - have; + } while (strm.avail_out == 0); + assert(strm.avail_in == 0); /* all input will be used */ + + /* done when last data in file processed */ + } while (flush != Z_FINISH); + assert(ret == Z_STREAM_END); /* stream will be complete */ + + /* clean up and return */ + (void)deflateEnd(&strm); + + printf("hello here...\n"); + return (dlen_orig - dlen); + // return Z_OK; +} + +/* Decompress from file source to file dest until stream ends or EOF. + inf() returns Z_OK on success, Z_MEM_ERROR if memory could not be + allocated for processing, Z_DATA_ERROR if the deflate data is + invalid or incomplete, Z_VERSION_ERROR if the version of zlib.h and + the version of the library linked do not match, or Z_ERRNO if there + is an error reading or writing the files. */ + + + + +int inf(char *source, int slen, char *dest, int dlen) +{ + int ret; + unsigned have; + z_stream strm; + unsigned char in[CHUNK]; + unsigned char out[CHUNK]; + int dlen_orig = dlen; + + + /* allocate inflate state */ + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = 0; + strm.next_in = Z_NULL; + ret = inflateInit(&strm); + if (ret != Z_OK) + return ret; + + /* decompress until deflate stream ends or end of file */ + do { + + if (slen == 0) + break; + + if (slen > CHUNK) + strm.avail_in = CHUNK; + else + strm.avail_in = slen; + + memcpy(in, source, strm.avail_in); + slen = slen - strm.avail_in; + source = source + strm.avail_in; + + + + strm.next_in = in; + + /* run inflate() on input until output buffer not full */ + do { + strm.avail_out = CHUNK; + strm.next_out = out; + ret = inflate(&strm, Z_NO_FLUSH); + assert(ret != Z_STREAM_ERROR); /* state not clobbered */ + switch (ret) { + case Z_NEED_DICT: + ret = Z_DATA_ERROR; /* and fall through */ + case Z_DATA_ERROR: + case Z_MEM_ERROR: + (void)inflateEnd(&strm); + return ret; + } + have = CHUNK - strm.avail_out; + + + if ((unsigned int) dlen < have) { + fprintf(stderr, "dest buf too small!\n"); + return Z_ERRNO; + } + + memcpy(dest, out, have); + dest += have; + dlen = dlen - have; + + } while (strm.avail_out == 0); + + /* done when inflate() says it's done */ + } while (ret != Z_STREAM_END); + + /* clean up and return */ + (void)inflateEnd(&strm); + + if (ret == Z_STREAM_END) + return dlen_orig - dlen; + return Z_DATA_ERROR; +} + +/* report a zlib or i/o error */ +void zerr(int ret) + +{ + fputs("zpipe: ", stderr); + switch (ret) { + case Z_ERRNO: + if (ferror(stdin)) + fputs("error reading stdin\n", stderr); + if (ferror(stdout)) + fputs("error writing stdout\n", stderr); + break; + case Z_STREAM_ERROR: + fputs("invalid compression level\n", stderr); + break; + case Z_DATA_ERROR: + fputs("invalid or incomplete deflate data\n", stderr); + break; + case Z_MEM_ERROR: + fputs("out of memory\n", stderr); + break; + case Z_VERSION_ERROR: + fputs("zlib version mismatch!\n", stderr); + } +} + + + + + + + + +/* assumes that we know there is exactly 10 bytes of gzip header */ + +int gzInflate(char *source, int slen, char *dest, int dlen) +{ + int ret; + unsigned have; + z_stream strm; + unsigned char in[CHUNK]; + unsigned char out[CHUNK]; + int dlen_orig = dlen; + + + /* allocate inflate state */ + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = 0; + strm.next_in = Z_NULL; + + + ret = inflateInit2(&strm, -MAX_WBITS); + if (ret != Z_OK) + return ret; + + source = source + 10; + slen -= 10; + + /* decompress until deflate stream ends or end of file */ + do { + + if (slen == 0) + break; + + if (slen > CHUNK) + strm.avail_in = CHUNK; + else + strm.avail_in = slen; + + memcpy(in, source, strm.avail_in); + slen = slen - strm.avail_in; + source = source + strm.avail_in; + + + + strm.next_in = in; + + /* run inflate() on input until output buffer not full */ + do { + strm.avail_out = CHUNK; + strm.next_out = out; + ret = inflate(&strm, Z_NO_FLUSH); + assert(ret != Z_STREAM_ERROR); /* state not clobbered */ + switch (ret) { + case Z_NEED_DICT: + ret = Z_DATA_ERROR; /* and fall through */ + case Z_DATA_ERROR: + case Z_MEM_ERROR: + (void)inflateEnd(&strm); + return ret; + } + have = CHUNK - strm.avail_out; + + if ((unsigned int) dlen < have) { + fprintf(stderr, "dest buf too small!\n"); + return Z_ERRNO; + } + + memcpy(dest, out, have); + dest += have; + dlen = dlen - have; + + } while (strm.avail_out == 0); + + /* done when inflate() says it's done */ + } while (ret != Z_STREAM_END); + + /* clean up and return */ + (void)inflateEnd(&strm); + + if (ret == Z_STREAM_END) + return dlen_orig - dlen; + return Z_DATA_ERROR; +} + + + + + + + +int gzDeflate(char* start, off_t insz, char *buf, off_t outsz, time_t mtime) { + unsigned char *c; + unsigned long crc; + z_stream z; + + z.zalloc = Z_NULL; + z.zfree = Z_NULL; + z.opaque = Z_NULL; + + if (Z_OK != deflateInit2(&z, + Z_DEFAULT_COMPRESSION, + Z_DEFLATED, + -MAX_WBITS, /* supress zlib-header */ + 8, + Z_DEFAULT_STRATEGY)) { + return -1; + } + + z.next_in = (unsigned char *)start; + z.avail_in = insz; + z.total_in = 0; + + + /* write gzip header */ + + c = (unsigned char *) buf; + c[0] = 0x1f; + c[1] = 0x8b; + c[2] = Z_DEFLATED; + c[3] = 0; /* options */ + c[4] = (mtime >> 0) & 0xff; + c[5] = (mtime >> 8) & 0xff; + c[6] = (mtime >> 16) & 0xff; + c[7] = (mtime >> 24) & 0xff; + c[8] = 0x00; /* extra flags */ + c[9] = 0x03; /* UNIX */ + + z.next_out = c + 10; + z.avail_out = outsz - 10 - 8; + z.total_out = 0; + + if (Z_STREAM_END != deflate(&z, Z_FINISH)) { + deflateEnd(&z); + return -1; + } + + + crc = generate_crc32c(start, insz); + + c = (unsigned char *)buf + 10 + z.total_out; + + c[0] = (crc >> 0) & 0xff; + c[1] = (crc >> 8) & 0xff; + c[2] = (crc >> 16) & 0xff; + c[3] = (crc >> 24) & 0xff; + c[4] = (z.total_in >> 0) & 0xff; + c[5] = (z.total_in >> 8) & 0xff; + c[6] = (z.total_in >> 16) & 0xff; + c[7] = (z.total_in >> 24) & 0xff; + + + + if (Z_OK != deflateEnd(&z)) { + return -1; + } + + return 10 + z.total_out + 8; + +} + + + + + +/* compress or decompress from stdin to stdout */ +/* int main(int argc, char **argv) */ +/* { */ +/* int ret; */ +/* char buf1[32] = "abcasdfadfadfadf23fasdfa23sdfsdf"; */ +/* char buf2[100]; */ +/* char buf3[100]; */ +/* int i; */ + +/* bzero(buf2, sizeof(buf2)); */ +/* bzero(buf3, sizeof(buf3)); */ + + +/* // ret = def(buf1, 3, buf2, 100, Z_DEFAULT_COMPRESSION); */ +/* ret = gzDeflate(buf1, sizeof(buf1), buf2, sizeof(buf2), time(NULL)); */ +/* if (ret <= 0) */ +/* zerr(ret); */ + +/* /\* for (i=0; i < ret; i++) */ +/* putc(buf2[i], stdout); */ +/* *\/ */ + + +/* // printf("len = %d\n", ret); */ + +/* ret = gzInflate(buf2, ret, buf3, 100); */ +/* if (ret <= 0) */ +/* zerr(ret); */ +/* printf("hello %s\n", buf3); */ + + +/* } */ diff --git a/src/steg/zpack.h b/src/steg/zpack.h new file mode 100644 index 0000000..65cd28d --- /dev/null +++ b/src/steg/zpack.h @@ -0,0 +1,14 @@ +#include <stdio.h> +#include <string.h> +#include <assert.h> +#include <time.h> +#include <stdlib.h> +#include "zlib.h" + + +int def(char *source, int slen, char *dest, int dlen, int level); +int inf(char *source, int slen, char *dest, int dlen); +void zerr(int ret); +int gzInflate(char *source, int slen, char *dest, int dlen); +int gzDeflate(char* start, off_t insz, char *buf, off_t outsz, time_t mtime); +unsigned int generate_crc32c(char *buffer, size_t length); diff --git a/start-client.csh b/start-client.csh new file mode 100644 index 0000000..65f4465 --- /dev/null +++ b/start-client.csh @@ -0,0 +1,8 @@ +#!/bin/csh +# ./obfsproxy --log-min-severity=debug x_dsteg socks 127.0.0.1:1080 x_http + +setenv EVENT_NOKQUEUE yes +#./obfsproxy --log-min-severity=debug chop socks 127.0.0.1:1080 127.0.0.1:8080 x_http2 127.0.0.1:8081 x_http2 +# ./obfsproxy --log-min-severity=warn chop socks 127.0.0.1:1080 127.0.0.1:8080 x_http2 127.0.0.1:8081 x_http2 +./obfsproxy --log-min-severity=error chop socks 127.0.0.1:1080 127.0.0.1:8080 x_http2 127.0.0.1:8081 x_http2 + diff --git a/start-server.csh b/start-server.csh new file mode 100644 index 0000000..50b781a --- /dev/null +++ b/start-server.csh @@ -0,0 +1,6 @@ +#!/bin/csh +setenv EVENT_NOKQUEUE yes +#./obfsproxy --log-min-severity=debug chop server 87.73.82.145:8080 127.0.0.1:8080 127.0.0.1:8081 +# ./obfsproxy --log-min-severity=warn chop server 87.73.82.145:8080 127.0.0.1:8080 127.0.0.1:8081 +./obfsproxy --log-min-severity=error chop server 87.73.82.145:8080 127.0.0.1:8080 127.0.0.1:8081 + diff --git a/torrc b/torrc new file mode 100644 index 0000000..ff27e61 --- /dev/null +++ b/torrc @@ -0,0 +1,12 @@ +SocksPort 9060 # what port to open for local application connections +SocksListenAddress 127.0.0.1 # accept connections only from localhost + +SafeLogging 0 +Log info file ./info.log +Log debug file ./debug.log + +Socks4Proxy 127.0.0.1:1080 + +# Bridge 87.73.82.145:8080 +Bridge 127.0.0.1:8080 +UseBridges 1

1 0

[torspec/master] Add proposal 205: Remove global client-side DNS caching
by nickm＠torproject.org 20 Jul '12

20 Jul '12

commit 610f21f2f6c4efda748d95c37234c63344b3037f Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jul 20 16:26:56 2012 -0400 Add proposal 205: Remove global client-side DNS caching --- proposals/205-local-dnscache.txt | 146 ++++++++++++++++++++++++++++++++++++++ 1 files changed, 146 insertions(+), 0 deletions(-) diff --git a/proposals/205-local-dnscache.txt b/proposals/205-local-dnscache.txt new file mode 100644 index 0000000..c0e19e5 --- /dev/null +++ b/proposals/205-local-dnscache.txt @@ -0,0 +1,146 @@ +Filename: 205-local-dnscache.txt +Title: Remove global client-side DNS caching +Author: Nick Mathewson +Created: 20 July 2012 +Status: Open + + +0. Overview + + This proposal suggests that, for reasons of security, we move + client-side DNS caching from a global cache to a set of per-circuit + caches. + + This will break some things that used to work. I'll explain how to + fix them. + +1. Background and Motivation + + Since the earliest Tor releases, we've kept a client-side DNS + cache. This lets us implement exit policies and exit enclaves -- + if we remember that www.mit.edu is 18.9.22.169 the first time we + see it, then we can avoid making future requests for www.mit.edu + via any node that blocks net 18. Also, if there happened to be a + Tor node at 18.9.22.169, we could use that node as an exit enclave. + + But there are security issues with DNS caches. A malicious exit + node or DNS server can lie. And unlike other traffic, where the + effect of a lie is confined to the request in question, a malicious + exit node can affect the behavior of future circuits when it gives + a false DNS reply. This false reply could be used to keep a client + connecting to an MITM'd target, or to make a client use a chosen + node as an exit enclave for that node, or so on. + + With IPv6, tracking attacks will become even more possible: A + hostile exit node can give every client a different IPv6 address + for every hostname they want to resolve, such that every one of + those addresses is under the attacker's control. + + And even if the exit node is honest, having a cached DNS result can + cause Tor clients to build their future circuits distinguishably: + the exit on any subsequent circuit can tell whether the client knew + the IP for the address yet or not. + + So client-side DNS caching needs to go away. + +2. Design + +2.1. The basic idea + + I propose that clients should cache DNS results in per-circuit DNS + caches, not in the global address map. + +2.2. What about exit policies? + + Microdescriptor-based clients have already dropped the ability to + track which nodes declare which exit policies, without much ill + effect. As we go forward, I think that remembering the IP address + of each request so that we can match it to exit policies will even + less effective, especially if proposals to allow AS-based exit + policies can succeed. + +2.3. What about exit enclaves? + + Exit enclaves are already borken. They need to move towards a + cross-certification solution where a node advertises that it can + exit to a hostname or domain X.Y.Z, and a signed record at X.Y.Z + advertises that . That's out-of-scope for this proposal, except to + note that nothing proposed here keeps that design from working. + +2.4. What about address mapping? + + Our current address map algorithm is, more or less: + + N = 0 + while N < MAX_MAPPING && exists map[address]: + address = map[address] + N = N + 1 + if N == MAX_MAPPING: + Give up, it's a loop. + + Where 'map' is the union of all mapping entries derived from the + controller, the configuration file, trackhostexits maps, + virtual-address mps, DNS replies, and so on. + + With this design, the DNS cache will not be part of the address + map. That means that entries in the address map which relied on + happening after the DNS cache entries can no longer work so well. + These would include: + + A) Mappings from an IP address to a particular exit, either + manually declared or inserted by TrackHostExits. + B) Mappings from IP addresses to other IP addresses. + C) Mappings from IP addresses to hostnames. + + We can try to solve these by introducing an extra step of address + mapping after the DNS cache is applied. In other words, we should + apply the address map, then see if we can attach to a circuit. If + we can, we try to apply that circuit's dns cache, then apply the + address map again. + + +2.5. What about the performance impact? + + That all depends on application behavior. + + If the application continues to make all of its requests with the + hostname, there shouldn't be much trouble. Exit-side DNS caches and + exit-side DNS will avoid any additional round trips across the Tor + network; compared to that, the time to do a DNS resolution at the + exit node *should* be small. + + That said, this will hurt performance a little in the case where + the exit node and its resolver don't have the answer cached, and it + takes a long time to resolve the hostname. + + + If the application is doing "resolve, then connect to an IP", see + 2.6 below. + +2.6. What about DNSPort? + + If the application is doing its own DNS caching, they won't get + much security benefit from here. + + If the application is doing a resolve before each connect, there + will be a performance hit when the resolver is using a circuit that + hadn't previously resolved the address. + + Also, DNSPort users: AutomapHostsOnResolve is your friend. + +3. Alternate designs and future directions + +3.1. Why keep client-side DNS caching at all? + + A fine question! I am not sure it actually buys us anything any + longer, since exits also have DNS caching. Shall we discuss that? + It would sure simplify matters. + +3.2. The impact of DNSSec + + Once we get DNSSec support, clients will be able to verify whether + an exit's answers are correctly signed or not. When that happens, + we could get most of the benefits of global DNS caching back, + without most of the security issues, if we restrict it to + DNSSec-signed answers. +

1 0

[torspec/master] Cleanups of typos found by arma
by nickm＠torproject.org 20 Jul '12

20 Jul '12

commit 60cec021a3ec4fa79c14dbe087578dc95c225f09 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jul 20 18:26:22 2012 -0400 Cleanups of typos found by arma --- proposals/205-local-dnscache.txt | 16 ++++++++++------ 1 files changed, 10 insertions(+), 6 deletions(-) diff --git a/proposals/205-local-dnscache.txt b/proposals/205-local-dnscache.txt index c0e19e5..e25e456 100644 --- a/proposals/205-local-dnscache.txt +++ b/proposals/205-local-dnscache.txt @@ -39,7 +39,10 @@ Status: Open And even if the exit node is honest, having a cached DNS result can cause Tor clients to build their future circuits distinguishably: the exit on any subsequent circuit can tell whether the client knew - the IP for the address yet or not. + the IP for the address yet or not. Further, if the site's DNS + provides different answers to clients from different parts of the + world, then the client's cached choice of IP will reveal where it + first learned about the website. So client-side DNS caching needs to go away. @@ -55,8 +58,8 @@ Status: Open Microdescriptor-based clients have already dropped the ability to track which nodes declare which exit policies, without much ill effect. As we go forward, I think that remembering the IP address - of each request so that we can match it to exit policies will even - less effective, especially if proposals to allow AS-based exit + of each request so that we can match it to exit policies will be + even less effective, especially if proposals to allow AS-based exit policies can succeed. 2.3. What about exit enclaves? @@ -64,8 +67,9 @@ Status: Open Exit enclaves are already borken. They need to move towards a cross-certification solution where a node advertises that it can exit to a hostname or domain X.Y.Z, and a signed record at X.Y.Z - advertises that . That's out-of-scope for this proposal, except to - note that nothing proposed here keeps that design from working. + advertises that the node is an enclave exit for X.Y.Z. That's + out-of-scope for this proposal, except to note that nothing + proposed here keeps that design from working. 2.4. What about address mapping? @@ -80,7 +84,7 @@ Status: Open Where 'map' is the union of all mapping entries derived from the controller, the configuration file, trackhostexits maps, - virtual-address mps, DNS replies, and so on. + virtual-address maps, DNS replies, and so on. With this design, the DNS cache will not be part of the address map. That means that entries in the address map which relied on

1 0

[torspec/master] Add proposal 205 to the index
by nickm＠torproject.org 20 Jul '12

20 Jul '12

commit ec9ca811f71b36d2f3cf498da68fafc250d863b5 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jul 20 18:26:34 2012 -0400 Add proposal 205 to the index --- proposals/000-index.txt | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/proposals/000-index.txt b/proposals/000-index.txt index 78de7dc..7671ce5 100644 --- a/proposals/000-index.txt +++ b/proposals/000-index.txt @@ -125,6 +125,7 @@ Proposals by number: 202 Two improved relay encryption protocols for Tor cells [OPEN] 203 Avoiding censorship by impersonating an HTTPS server [DRAFT] 204 Subdomain support for Hidden Service addresses [OPEN] +205 Remove global client-side DNS caching [OPEN] Proposals by status: @@ -166,6 +167,7 @@ Proposals by status: 201 Make bridges report statistics on daily v3 network status requests [for 0.2.4.x] 202 Two improved relay encryption protocols for Tor cells 204 Subdomain support for Hidden Service addresses + 205 Remove global client-side DNS caching ACCEPTED: 117 IPv6 exits [for 0.2.4.x] 140 Provide diffs between consensuses

1 0