- tor-commits - lists.torproject.org

[stegotorus/master] Update some lists so 'make distcheck' succeeds.
by zwol＠torproject.org 20 Jul '12

20 Jul '12

commit 3624a2825d806507534a99367c3f9e39ae42263b Author: Zack Weinberg <zackw(a)cmu.edu> Date: Sun Jul 8 20:18:28 2012 +0200 Update some lists so 'make distcheck' succeeds. --- Makefile.am | 13 ++++++++++++- 1 files changed, 12 insertions(+), 1 deletions(-) diff --git a/Makefile.am b/Makefile.am index b449619..c275aac 100644 --- a/Makefile.am +++ b/Makefile.am @@ -71,15 +71,25 @@ nodist_unittests_SOURCES = unitgrplist.cc tltester_SOURCES = src/test/tltester.cc noinst_HEADERS = \ + src/base64.h \ + src/compression.h \ src/connections.h \ src/crypt.h \ src/listener.h \ + src/mkem.h \ src/protocol.h \ src/rng.h \ src/socks.h \ + src/subprocess.h \ src/steg.h \ src/util.h \ src/protocol/chop_blk.h \ + src/steg/b64cookies.h \ + src/steg/cookies.h \ + src/steg/jsSteg.h \ + src/steg/payloads.h \ + src/steg/pdfSteg.h \ + src/steg/swfSteg.h \ src/test/tinytest.h \ src/test/tinytest_macros.h \ src/test/unittest.h @@ -96,7 +106,8 @@ EXTRA_DIST = doc \ # Generated source files CLEANFILES = protolist.cc steglist.cc unitgrplist.cc \ - stamp-protolist stamp-steglist stamp-unitgrplist + stamp-protolist stamp-steglist stamp-unitgrplist \ + stamp-audit-globals GMOD = $(SHELL) $(srcdir)/src/genmodtable.sh GUNIT = $(SHELL) $(srcdir)/src/test/genunitgrps.sh

1 0

[stegotorus/master] Add Vinod's program for generating HTTP trace files from pcap files.
by zwol＠torproject.org 20 Jul '12

20 Jul '12

commit 949aa9547d89d4b2d463a0460941a0cc35cec56d Author: Zack Weinberg <zackw(a)cmu.edu> Date: Thu Jul 12 16:24:20 2012 +0200 Add Vinod's program for generating HTTP trace files from pcap files. * pgen.h, pgen_pcap.cc: New files. * util.cc: Split libevent-using routines to util-net.cc. * configure.ac: Detect availability of libpcap. * Makefile.am: Build pgen_pcap if we have libpcap. Shuffle linkage variables around a little so each program is only linked against the libraries it needs. --- Makefile.am | 27 ++- configure.ac | 27 ++- src/audit-globals.sh | 2 +- src/pgen.h | 21 ++ src/pgen_pcap.cc | 617 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/util-net.cc | 134 +++++++++++ src/util.cc | 129 ----------- 7 files changed, 822 insertions(+), 135 deletions(-) diff --git a/Makefile.am b/Makefile.am index 42eaede..d19aab6 100644 --- a/Makefile.am +++ b/Makefile.am @@ -6,7 +6,6 @@ ACLOCAL_AMFLAGS = -I config-aux --install AM_CXXFLAGS = -Werror -Wall -Wextra -Wformat=2 AM_CPPFLAGS = -I. -I$(srcdir)/src -D_FORTIFY_SOURCE=2 $(lib_CPPFLAGS) -LDADD = libstegotorus.a noinst_LIBRARIES = libstegotorus.a noinst_PROGRAMS = unittests tltester @@ -41,6 +40,7 @@ libstegotorus_a_SOURCES = \ src/socks.cc \ src/steg.cc \ src/util.cc \ + src/util-net.cc \ $(PROTOCOLS) $(STEGANOGRAPHERS) if WINDOWS @@ -54,8 +54,25 @@ nodist_libstegotorus_a_SOURCES = protolist.cc steglist.cc stegotorus_SOURCES = \ src/main.cc +stegotorus_LDADD = libstegotorus.a $(lib_LIBS) + # prevent stegotorus from being linked if s-a-g fails -stegotorus_DEPENDENCIES = $(LDADD) stamp-audit-globals +# it is known that $(lib_LIBS) contains nothing that needs to be depended upon +stegotorus_DEPENDENCIES = libstegotorus.a stamp-audit-globals + +## payload trace generators + +# pgen_pcap is only built if we have libpcap +if HAVE_PCAP +bin_PROGRAMS += pgen_pcap + +pgen_pcap_SOURCES = \ + src/pgen_pcap.cc \ + src/compression.cc \ + src/util.cc + +pgen_pcap_LDADD = $(pcap_LIBS) $(libz_LIBS) +endif UTGROUPS = \ src/test/unittest_base64.cc \ @@ -71,7 +88,10 @@ unittests_SOURCES = \ nodist_unittests_SOURCES = unitgrplist.cc -tltester_SOURCES = src/test/tltester.cc +unittests_LDADD = libstegotorus.a $(lib_LIBS) + +tltester_SOURCES = src/test/tltester.cc src/util.cc src/util-net.cc +tltester_LDADD = $(libevent_LIBS) noinst_HEADERS = \ src/base64.h \ @@ -80,6 +100,7 @@ noinst_HEADERS = \ src/crypt.h \ src/listener.h \ src/mkem.h \ + src/pgen.h \ src/protocol.h \ src/rng.h \ src/socks.h \ diff --git a/configure.ac b/configure.ac index 889cfae..dcad0db 100644 --- a/configure.ac +++ b/configure.ac @@ -125,8 +125,6 @@ PKG_CHECK_MODULES([libevent], [libevent >= 2.0]) PKG_CHECK_MODULES([libz], [zlib >= 1.2.3.4]) LIBS="$libevent_LIBS $libcrypto_LIBS $libz_LIBS" -lib_CPPFLAGS="$libevent_CFLAGS $libcrypto_CFLAGS $libz_CFLAGS" -AC_SUBST(lib_CPPFLAGS) # ntohl and a bunch of related functions require a special library on Windows. # It is possible that libevent or libcrypto has hooked us up already. @@ -139,6 +137,31 @@ AC_SEARCH_LIBS([floor], [m], [], [ AC_MSG_ERROR([unable to find 'floor']) ]) +lib_LIBS="$LIBS" +lib_CPPFLAGS="$libevent_CFLAGS $libcrypto_CFLAGS $libz_CFLAGS" +LIBS= +AC_SUBST(lib_LIBS) +AC_SUBST(lib_CPPFLAGS) + +# pgen_pcap needs libpcap. +pcap_LIBS= +HAVE_PCAP=no +AC_SEARCH_LIBS([pcap_open_offline], [pcap], + [AC_CACHE_CHECK([whether libpcap is usable], ac_cv_libpcap_usable, + [AC_LINK_IFELSE([AC_LANG_PROGRAM( + [[#include <pcap/pcap.h>]], + [[char f; + pcap_t *p = pcap_open_offline("",&f);]])], + [ac_cv_libpcap_usable=yes], [ac_cv_libpcap_usable=no])]) + if test $ac_cv_libpcap_usable = yes; then + HAVE_PCAP=yes + pcap_LIBS="$LIBS" + fi], + []) +LIBS= +AC_SUBST(pcap_LIBS) +AM_CONDITIONAL(HAVE_PCAP, test $HAVE_PCAP = yes) + ### System features ### AC_CHECK_HEADERS([execinfo.h paths.h],,,[/**/]) diff --git a/src/audit-globals.sh b/src/audit-globals.sh index 2512bb6..21ff6d2 100644 --- a/src/audit-globals.sh +++ b/src/audit-globals.sh @@ -46,7 +46,7 @@ sed ' /^util log_min_sev$/d /^util log_timestamps$/d /^util log_ts_base$/d - /^util the_evdns_base$/d + /^util-net the_evdns_base$/d ') if [ -n "$symbols" ]; then diff --git a/src/pgen.h b/src/pgen.h new file mode 100644 index 0000000..c8e7a55 --- /dev/null +++ b/src/pgen.h @@ -0,0 +1,21 @@ +/* Copyright 2011, 2012 SRI International + * See LICENSE for other credits and copying information + */ + +#ifndef PGEN_H +#define PGEN_H + +// NOTE: this must be kept in sync with steg/payloads.h + +#define TYPE_SERVICE_DATA 0x1 +#define TYPE_HTTP_REQUEST 0x2 +#define TYPE_HTTP_RESPONSE 0x4 + +/* struct for reading in the payload_gen dump file */ +struct pentry_header { + uint16_t ptype; + uint32_t length; + uint16_t port; /* network format */ +}; + +#endif diff --git a/src/pgen_pcap.cc b/src/pgen_pcap.cc new file mode 100644 index 0000000..cb6e475 --- /dev/null +++ b/src/pgen_pcap.cc @@ -0,0 +1,617 @@ +/* Copyright 2011, 2012 SRI International + * See LICENSE for other credits and copying information + */ + +#include "util.h" +#include "pgen.h" +#include "compression.h" + +#include <pcap/pcap.h> +#include <netinet/in.h> +#include <netinet/if_ether.h> +#include <netinet/ip.h> +#include <netinet/tcp.h> +#include <dirent.h> + +#define NUM_FLOWS 1000 +#define NUM_LISTS 1000 + +#define CONN_DATA_REQUEST 1 /* payload packet sent by client */ +#define CONN_DATA_REPLY 2 /* payload packet sent by server */ + +#define RECV_MTU 64000 +// #define PKT_MTU 1500 +#define MAX_CHAIN_LEN 4000 + +#define MSG_INSERTED 1 +#define MSG_INVALID 0 +#define MSG_SEQ_WRAP -2 +#define MSG_DUPLICATE -3 +#define CHAIN_TOO_LONG -4 +#define MSG_OVERLAP -5 +#define CHAIN_HAS_GAPS_OVERLAPS -6 +#define CHAIN_EMPTY -7 + +struct msg { + uint8_t *buf; + uint16_t len; + uint32_t seqno; + msg *next_msg; +}; + +struct flow { + uint32_t src_ip; + uint32_t dst_ip; + uint16_t sport; + uint16_t dport; + uint8_t flags; + uint8_t proto; + struct timeval change_time; + int sockfd; + flow *next_flow; + msg *msg_buf_chain; + int chain_len; + int msg_len_so_far; + int dir; /* data request or data reply */ + uint32_t ack_so_far; /* what's acknowledged by other end so far */ +}; + +static flow *flows[NUM_LISTS]; +static pcap_t *descr; +static int dir_flag = 0; +static char *bp_filter; +static char errbuf[PCAP_ERRBUF_SIZE]; +static struct bpf_program fp; +static uint32_t netp; +static const char *argv0; + +#define RECV_MTU 64000 +#define PORT_HTTP 80 + +static FILE *client_file; +static FILE *server_file; + +static void ATTR_NORETURN +usage() +{ + fprintf(stderr, "Usage: %s [-d dumpdir] [-r dumpfile] \"bpf filter\"\n", + argv0); + exit(1); +} + +static void ATTR_NORETURN +terminate(int) +{ + struct pcap_stat ps; + if (pcap_stats(descr, &ps) < 0) { + fputs("err: pcap stats not supported?\n", stderr); + exit(1); + } + + printf("packets rcvd: %u, packets dropped: %u, interface drops: %u\n", + ps.ps_recv, ps.ps_drop, ps.ps_ifdrop); + exit(1); +} + +static void +free_msg_chain(flow *f) +{ + msg *m = f->msg_buf_chain; + + while (m && f->chain_len > 0) { + msg *n = m->next_msg; + free(m); + m = n; + } + + f->chain_len = 0; + f->msg_len_so_far = 0; + f->msg_buf_chain = 0; +} + +static bool +has_chain_gaps(flow *f) +{ + msg *m = f->msg_buf_chain; + + while (m) { + if (!m->next_msg) + return false; + + if (m->seqno + m->len < m->next_msg->seqno) { + fprintf(stderr, "gap seqnos: %u %u %u %d\n", + m->seqno, m->len, m->next_msg->seqno, f->dport); + return true; + } + + if (m->seqno + m->len > m->next_msg-> seqno) { + fprintf(stderr, "overlap seqnos: %u %u %u\n", + m->seqno, m->len, m->next_msg->seqno); + return true; + } + m = m->next_msg; + } + + return false; +} + +static int +write_inflate_msg(flow *f, FILE *file, pentry_header *ph) +{ + msg *m = f->msg_buf_chain; + uint8_t *buf; + int pos = 0; + uint8_t *outbuf; + int outlen; + + uint8_t *hdr_end; + uint8_t *hdr; + int hdrlen; + + if (!f->msg_buf_chain) + return CHAIN_EMPTY; + + if (strstr((char*) m->buf, "Transfer-Encoding: chunked")) + // we don't handle this yet....need a loop to unzip chunks individually... + return MSG_INVALID; + + hdr_end = (uint8_t*) strstr((char*) m->buf, "\r\n\r\n"); + if (!hdr_end) { + fprintf(stderr, "hdr too long?? \n"); + return MSG_INVALID; + } + + hdr_end += 4; + hdrlen = hdr_end - m->buf; + hdr = (uint8_t *) xmemdup(m->buf, hdrlen); + + buf = (uint8_t *) xmalloc(f->msg_len_so_far); + outbuf = (uint8_t *) xmalloc(f->msg_len_so_far * 20); + + pos = 0; + + if (!m) + return CHAIN_EMPTY; + + memcpy(buf, hdr_end, m->len - hdrlen); + pos += m->len - hdrlen; + m = m->next_msg; + + while (m) { + memcpy(buf+pos, m->buf, m->len); + pos += m->len; + m = m->next_msg; + } + + outlen = decompress(buf, f->msg_len_so_far - hdrlen, + outbuf, f->msg_len_so_far*20); + + if (outlen < 0) { + fprintf(stderr, "unzip failed outlen = %d %d %d\n", + outlen, pos, f->msg_len_so_far-hdrlen); + return MSG_INVALID; + } + + ph->length = htonl(outlen+hdrlen); + fwrite(ph, sizeof(pentry_header), 1, file); + fwrite(hdr, hdrlen, 1, file); + fwrite(outbuf, outlen, 1, file); + free(buf); + free(outbuf); + free(hdr); + return 1; +} + +static int +write_msg_chains(flow *f, FILE *file, pentry_header *ph) +{ + msg *m = f->msg_buf_chain; + int cnt = 0; + + if (has_chain_gaps(f)) + return CHAIN_HAS_GAPS_OVERLAPS; + + if (!m) + return CHAIN_EMPTY; + + if (strstr((char*) m->buf, "200 OK") && + strstr((char*) m->buf, "Content-Encoding: gzip")) + return write_inflate_msg(f, file, ph); + + fwrite(ph, sizeof(pentry_header), 1, file); + + while (m) { + fwrite(m->buf, m->len, 1, file); + cnt += m->len; + m = m->next_msg; + } + + if (cnt != f->msg_len_so_far) + fprintf(stderr, "something funky in writing message\n"); + return 1; +} + +static bool +is_valid_http_request(flow *f) +{ + if (!f->msg_buf_chain) { + fprintf(stderr, "is_valid_http_request: invalid chain %d\n", f->chain_len); + return false; + } + + if (!strncmp((char*) f->msg_buf_chain->buf, "GET", 3) || + !strncmp((char*) f->msg_buf_chain->buf, "POST", 4)) { + msg *m = f->msg_buf_chain; + while (m->next_msg) + m = m->next_msg; + + if (m->buf[m->len-2] == '\r' && m->buf[m->len-1] == '\n') { + return true; + } + } + + return false; +} + +static int +add_msg_to_flow(flow *f, uint8_t *buf, uint seq, int len) +{ + if (len > RECV_MTU) + return MSG_INVALID; + + if (f->chain_len >= MAX_CHAIN_LEN) + return CHAIN_TOO_LONG; + + if (seq > seq + len) + return MSG_SEQ_WRAP; + + msg *p = 0; + msg *m = f->msg_buf_chain; + + if (!m) { + m = (msg *)xzalloc(sizeof(msg)); + m->buf = (uint8_t *)xmalloc(len); + memcpy(m->buf, buf, len); + m->seqno = seq; + f->chain_len = 1; + f->msg_len_so_far += len; + f->msg_buf_chain = m; + m->len = len; + return MSG_INSERTED; + } + + while (m) { + if (m->seqno == seq) + return MSG_DUPLICATE; + + if (m->seqno < seq) { + if (m->seqno > seq + len) + return MSG_OVERLAP; + p = m; + m = m->next_msg; + continue; + } + + if (m->seqno < seq + len) + return MSG_OVERLAP; + + msg *n; + if (!p) { + p = (msg *)xzalloc(sizeof(msg)); + p->buf = (uint8_t *)xmalloc(len); + memcpy(p->buf, buf, len); + p->seqno = seq; + p->next_msg = m; + f->chain_len++; + f->msg_len_so_far += len; + f->msg_buf_chain = p; + p->len = len; + return MSG_INSERTED; + } + + n = (msg *)xzalloc(sizeof(msg)); + n->buf = (uint8_t *)xmemdup(buf, len); + memcpy(n->buf, buf, len); + n->seqno = seq; + n->next_msg = m; + p->next_msg = n; + f->chain_len++; + f->msg_len_so_far += len; + n->len = len; + return MSG_INSERTED; + } + + m = (msg *)xzalloc(sizeof(msg)); + m->buf = (uint8_t *)xmemdup(buf, len); + m->seqno = seq; + p->next_msg = m; + f->chain_len++; + f->msg_len_so_far += len; + m->len = len; + return MSG_INSERTED; +} + +static int +hash_flow(flow *f) +{ + return (f->src_ip + f->dst_ip + f->sport + f->dport) % NUM_LISTS; +} + +static bool +flow_compare(flow *f1, flow *f2) +{ + return (f1->src_ip == f2->src_ip && f1->dst_ip == f2->dst_ip + && f1->sport == f2->sport && f1->dport == f2->dport); +} + +static flow * +add_to_flows(flow *f) +{ + int hval = hash_flow(f); + + flow *cflow = flows[hval]; + + if (!cflow) { + cflow = (flow *) xmalloc(sizeof(flow)); + memcpy(cflow, f, sizeof(flow)); + cflow->next_flow = 0; + flows[hval] = cflow; + return cflow; + } + else { + // add flow to the beginning of the chain + flow *old_flow = (flow *) xmalloc(sizeof(flow)); + memcpy(old_flow, cflow, sizeof(flow)); + memcpy(cflow, f, sizeof(flow)); + cflow->next_flow = old_flow; + } + return cflow; +} + +static flow * +has_seen_flow(flow *f) +{ + int hval = hash_flow(f); + flow *cflow = flows[hval]; + + while (cflow) { + if (flow_compare(cflow, f)) + return cflow; + cflow = cflow->next_flow; + } + + return 0; +} + +static flow * +reverse_flow(flow *f) +{ + uint32_t tmp_ip; + uint16_t tmp_port; + + tmp_ip = f->src_ip; + f->src_ip = f->dst_ip; + f->dst_ip = tmp_ip; + + tmp_port = f->sport; + f->sport = f->dport; + f->dport = tmp_port; + return f; +} + +static void +write_http_packet(flow *f) +{ + pentry_header ph; + ph.length = htonl(f->msg_len_so_far); + ph.port = htons(80); + + if (f->dir == CONN_DATA_REQUEST) { + ph.ptype = htons(TYPE_HTTP_REQUEST); + if (is_valid_http_request(f)) + write_msg_chains(f, client_file, &ph); + } + else { + ph.ptype = htons(TYPE_HTTP_RESPONSE); + write_msg_chains(f, server_file, &ph); + } +} + +static void +write_packet(flow *f) +{ + uint16_t tport; + if (f->dir == CONN_DATA_REQUEST) + tport = f->dport; + else + tport = f->sport; + + switch(tport) { + case PORT_HTTP: + write_http_packet(f); + } +} + +static void +my_callback(uint8_t * /*unused*/, + const struct pcap_pkthdr *pkthdr, + const uint8_t *packet) +{ + + struct ether_header *eth = (struct ether_header*) (packet) ; + int rval; + + if (ntohs(eth->ether_type) == ETHERTYPE_IP) { + + struct ip *iph = (struct ip*) (packet + sizeof(struct ether_header)); + struct tcphdr *tcph = (struct tcphdr*) + ((uint8_t*)iph + sizeof(struct ip)); + + int len = htons(iph->ip_len) - 4*tcph->th_off - sizeof(struct ip); + uint8_t *payload = (uint8_t*) tcph + 4*tcph->th_off; + flow f; + flow *cflow; + flow *rflow; + + memset(&f, 0, sizeof(flow)); + f.src_ip = iph->ip_src.s_addr; + f.dst_ip = iph->ip_dst.s_addr; + f.sport = ntohs(tcph->th_sport); + f.dport = ntohs(tcph->th_dport); + f.flags = tcph->th_flags; + f.proto = iph->ip_p; + f.change_time = pkthdr->ts; + + if (tcph->th_flags & TH_SYN && !(tcph->th_flags & TH_ACK)) { + f.dir = CONN_DATA_REQUEST; + add_to_flows(&f); + return; + } + + else if ((tcph->th_flags & TH_SYN) && (tcph->th_flags & TH_ACK)) { + f.dir = CONN_DATA_REPLY; + add_to_flows(&f); + return; + } + + cflow = has_seen_flow(&f); + if (!cflow) + return; + + rflow = has_seen_flow(reverse_flow(&f)); + if (!rflow) + return; + + rflow->ack_so_far = ntohl(tcph->th_ack); + cflow->flags = cflow->flags | tcph->th_flags; + + if (len > 0 && ntohl(tcph->th_seq) >= cflow->ack_so_far) { + if (rflow->msg_len_so_far > 0) { + write_packet(rflow); + free_msg_chain(rflow); + } + + rval = add_msg_to_flow(cflow, payload, ntohl(tcph->th_seq), len); + + if (rval <= 0 && rval !=MSG_DUPLICATE && rval != CHAIN_TOO_LONG) { + fprintf(stderr, "adding msg to flow failed %d %d\n", rval, len); + } + } + + + if (cflow->flags & TH_RST || cflow->flags & TH_FIN) { + if (rflow->msg_len_so_far > 0) { + write_packet(rflow); + free_msg_chain(rflow); + } + return; + } + } +} + +static void +handle_pcap_file(const char *filename) +{ + descr = pcap_open_offline(filename, errbuf); + if (!descr) { + fprintf(stderr, "%s: %s\n", filename, errbuf); + exit(1); + } + + if (pcap_compile(descr, &fp, bp_filter, 1, netp) == -1) { + fprintf(stderr, "Error calling pcap_compile on \"%s\"\n", bp_filter); + exit(1); + } + + /* set the compiled program as the filter */ + if (pcap_setfilter(descr, &fp) == -1) { + fprintf(stderr,"Error setting filter\n"); + exit(1); + } + + /* main pcap loop */ + pcap_loop(descr, -1, my_callback, 0); + pcap_close(descr); +} + +static void +list_files(const char *dirname) +{ + DIR *dip; + struct dirent *dit; + char *fname; + size_t plen = strlen(dirname); + + if ((dip = opendir(dirname)) == 0) { + perror("opendir"); + return; + } + + while ((dit = readdir(dip)) != 0) { + if (!strcmp(dit->d_name, ".") || !strcmp(dit->d_name, "..")) + continue; + + size_t dlen = strlen(dit->d_name); + fname = (char *)xmalloc(plen + dlen + 2); + memcpy(fname, dirname, plen); + fname[plen] = '/'; + memcpy(fname + plen + 1, dit->d_name, dlen); + fname[plen + dlen + 1] = '\0'; + fprintf(stderr, "%s\n", fname); + handle_pcap_file(fname); + free(fname); + } + + closedir(dip); +} + +int +main(int argc, char **argv) +{ + int c; + const char *dumpfile = 0; + + argv0 = argv[0]; + + while ((c = getopt (argc, argv, "r:d:")) != -1) { + switch (c) { + case 'r': + dumpfile = optarg; + break; + case 'd': + dir_flag = 1; + dumpfile = optarg; + break; + default: + usage(); + } + } + + if (!argv[optind] || !dumpfile) + usage(); + + bp_filter = xstrdup(argv[optind]); + + client_file = fopen("traces/client.out", "w"); + if (!client_file) { + perror("traces/client.out"); + return 1; + } + server_file = fopen("traces/server.out", "w"); + if (!server_file) { + perror("traces/server.out"); + return 1; + } + + /* catch ^C print stats and exit */ + signal(SIGTERM, terminate); + signal(SIGINT, terminate); + signal(SIGHUP, terminate); + + if (dir_flag) + list_files(dumpfile); + else + handle_pcap_file(dumpfile); + + return 0; +} diff --git a/src/util-net.cc b/src/util-net.cc new file mode 100644 index 0000000..0826eaa --- /dev/null +++ b/src/util-net.cc @@ -0,0 +1,134 @@ +/* Copyright 2011 Nick Mathewson, George Kadianakis + * Copyright 2011, 2012 SRI International + * See LICENSE for other credits and copying information + */ + +#include "util.h" + +#include <event2/dns.h> + +#include <errno.h> +#include <netinet/in.h> +#include <arpa/inet.h> +#ifdef AF_LOCAL +#include <sys/un.h> +#endif + +/** + Accepts a string 'address' of the form ADDRESS:PORT and attempts to + parse it into an 'evutil_addrinfo' structure. + + If 'nodns' is set it means that 'address' was an IP address. + If 'passive' is set it means that the address is destined for + listening and not for connecting. + + If no port was given in 'address', we set 'default_port' as the + port. +*/ +struct evutil_addrinfo * +resolve_address_port(const char *address, int nodns, int passive, + const char *default_port) +{ + struct evutil_addrinfo *ai = NULL; + struct evutil_addrinfo ai_hints; + int ai_res, ai_errno; + char *a = xstrdup(address), *cp; + const char *portstr; + + if ((cp = strchr(a, ':'))) { + portstr = cp+1; + *cp = '\0'; + } else if (default_port) { + portstr = default_port; + } else { + log_debug("error in address %s: port required", address); + free(a); + return NULL; + } + + memset(&ai_hints, 0, sizeof(ai_hints)); + ai_hints.ai_family = AF_UNSPEC; + ai_hints.ai_socktype = SOCK_STREAM; + ai_hints.ai_flags = EVUTIL_AI_ADDRCONFIG | EVUTIL_AI_NUMERICSERV; + if (passive) + ai_hints.ai_flags |= EVUTIL_AI_PASSIVE; + if (nodns) + ai_hints.ai_flags |= EVUTIL_AI_NUMERICHOST; + + ai_res = evutil_getaddrinfo(a, portstr, &ai_hints, &ai); + ai_errno = errno; + + free(a); + + if (ai_res) { + if (ai_res == EVUTIL_EAI_SYSTEM) + log_warn("error resolving %s: %s [%s]", + address, evutil_gai_strerror(ai_res), strerror(ai_errno)); + else + log_warn("error resolving %s: %s", address, evutil_gai_strerror(ai_res)); + + if (ai) { + evutil_freeaddrinfo(ai); + ai = NULL; + } + } else if (ai == NULL) { + log_warn("address resolution failed for %s", address); + } + + return ai; +} + +char * +printable_address(struct sockaddr *addr, socklen_t addrlen) +{ + char apbuf[INET6_ADDRSTRLEN + 8]; /* []:65535 is 8 characters */ + + switch (addr->sa_family) { +#ifndef _WIN32 /* Windows XP doesn't have inet_ntop. Fix later. */ + case AF_INET: { + char abuf[INET6_ADDRSTRLEN]; + struct sockaddr_in *sin = (struct sockaddr_in*)addr; + log_assert(addrlen >= sizeof(struct sockaddr_in)); + if (!inet_ntop(AF_INET, &sin->sin_addr, abuf, INET6_ADDRSTRLEN)) + break; + xsnprintf(apbuf, sizeof apbuf, "%s:%d", abuf, ntohs(sin->sin_port)); + return xstrdup(apbuf); + } + + case AF_INET6: { + char abuf[INET6_ADDRSTRLEN]; + struct sockaddr_in6 *sin6 = (struct sockaddr_in6*)addr; + log_assert(addrlen >= sizeof(struct sockaddr_in6)); + if (!inet_ntop(AF_INET, &sin6->sin6_addr, abuf, INET6_ADDRSTRLEN)) + break; + xsnprintf(apbuf, sizeof apbuf, "[%s]:%d", abuf, ntohs(sin6->sin6_port)); + return xstrdup(apbuf); + } +#endif + +#ifdef AF_LOCAL + case AF_LOCAL: + return xstrdup(((struct sockaddr_un*)addr)->sun_path); +#endif + default: + break; + } + + xsnprintf(apbuf, sizeof apbuf, "<addr family %d>", addr->sa_family); + return xstrdup(apbuf); +} + +static struct evdns_base *the_evdns_base = NULL; + +struct evdns_base * +get_evdns_base(void) +{ + return the_evdns_base; +} + +int +init_evdns_base(struct event_base *base) +{ + the_evdns_base = evdns_base_new(base, 1); + return the_evdns_base == NULL ? -1 : 0; +} diff --git a/src/util.cc b/src/util.cc index 3900f21..d06a1c8 100644 --- a/src/util.cc +++ b/src/util.cc @@ -9,14 +9,6 @@ #include <fcntl.h> #include <unistd.h> -#include <event2/dns.h> - -#include <netinet/in.h> -#include <arpa/inet.h> -#ifdef AF_LOCAL -#include <sys/un.h> -#endif - /**************************** Memory Allocation ******************************/ static void ATTR_NORETURN @@ -132,127 +124,6 @@ ui64_log2(uint64_t u64) return r; } -/************************ Network Routines *************************/ - -/** - Accepts a string 'address' of the form ADDRESS:PORT and attempts to - parse it into an 'evutil_addrinfo' structure. - - If 'nodns' is set it means that 'address' was an IP address. - If 'passive' is set it means that the address is destined for - listening and not for connecting. - - If no port was given in 'address', we set 'default_port' as the - port. -*/ -struct evutil_addrinfo * -resolve_address_port(const char *address, int nodns, int passive, - const char *default_port) -{ - struct evutil_addrinfo *ai = NULL; - struct evutil_addrinfo ai_hints; - int ai_res, ai_errno; - char *a = xstrdup(address), *cp; - const char *portstr; - - if ((cp = strchr(a, ':'))) { - portstr = cp+1; - *cp = '\0'; - } else if (default_port) { - portstr = default_port; - } else { - log_debug("error in address %s: port required", address); - free(a); - return NULL; - } - - memset(&ai_hints, 0, sizeof(ai_hints)); - ai_hints.ai_family = AF_UNSPEC; - ai_hints.ai_socktype = SOCK_STREAM; - ai_hints.ai_flags = EVUTIL_AI_ADDRCONFIG | EVUTIL_AI_NUMERICSERV; - if (passive) - ai_hints.ai_flags |= EVUTIL_AI_PASSIVE; - if (nodns) - ai_hints.ai_flags |= EVUTIL_AI_NUMERICHOST; - - ai_res = evutil_getaddrinfo(a, portstr, &ai_hints, &ai); - ai_errno = errno; - - free(a); - - if (ai_res) { - if (ai_res == EVUTIL_EAI_SYSTEM) - log_warn("error resolving %s: %s [%s]", - address, evutil_gai_strerror(ai_res), strerror(ai_errno)); - else - log_warn("error resolving %s: %s", address, evutil_gai_strerror(ai_res)); - - if (ai) { - evutil_freeaddrinfo(ai); - ai = NULL; - } - } else if (ai == NULL) { - log_warn("address resolution failed for %s", address); - } - - return ai; -} - -char * -printable_address(struct sockaddr *addr, socklen_t addrlen) -{ - char apbuf[INET6_ADDRSTRLEN + 8]; /* []:65535 is 8 characters */ - - switch (addr->sa_family) { -#ifndef _WIN32 /* Windows XP doesn't have inet_ntop. Fix later. */ - case AF_INET: { - char abuf[INET6_ADDRSTRLEN]; - struct sockaddr_in *sin = (struct sockaddr_in*)addr; - log_assert(addrlen >= sizeof(struct sockaddr_in)); - if (!inet_ntop(AF_INET, &sin->sin_addr, abuf, INET6_ADDRSTRLEN)) - break; - xsnprintf(apbuf, sizeof apbuf, "%s:%d", abuf, ntohs(sin->sin_port)); - return xstrdup(apbuf); - } - - case AF_INET6: { - char abuf[INET6_ADDRSTRLEN]; - struct sockaddr_in6 *sin6 = (struct sockaddr_in6*)addr; - log_assert(addrlen >= sizeof(struct sockaddr_in6)); - if (!inet_ntop(AF_INET, &sin6->sin6_addr, abuf, INET6_ADDRSTRLEN)) - break; - xsnprintf(apbuf, sizeof apbuf, "[%s]:%d", abuf, ntohs(sin6->sin6_port)); - return xstrdup(apbuf); - } -#endif - -#ifdef AF_LOCAL - case AF_LOCAL: - return xstrdup(((struct sockaddr_un*)addr)->sun_path); -#endif - default: - break; - } - - xsnprintf(apbuf, sizeof apbuf, "<addr family %d>", addr->sa_family); - return xstrdup(apbuf); -} - -static struct evdns_base *the_evdns_base = NULL; - -struct evdns_base * -get_evdns_base(void) -{ - return the_evdns_base; -} - -int -init_evdns_base(struct event_base *base) -{ - the_evdns_base = evdns_base_new(base, 1); - return the_evdns_base == NULL ? -1 : 0; -} - /************************ String Functions *************************/ /** Many of the functions in this section were carbon copied off tor. Thank you tor! */

1 0

[stegotorus/master] Three tiny bugfixes in the build system.
by zwol＠torproject.org 20 Jul '12

20 Jul '12

commit 542ed83ed4d63d9cee91155adf33bc93dc3d799e Author: Zack Weinberg <zackw(a)cmu.edu> Date: Sun Jul 8 20:36:27 2012 +0200 Three tiny bugfixes in the build system. * Use quadrigraphs for "rm -f conft[ABC].o" in ranlib.m4 so they actually get deleted. * Stop using EXTRA_*_DEPENDENCIES which autoconf 1.11 doesn't support. * Make the spacing of the silent-mode custom printouts match autoconf 1.12 (purely cosmetic). --- Makefile.am | 10 +++++----- config-aux/ranlib.m4 | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile.am b/Makefile.am index c275aac..42eaede 100644 --- a/Makefile.am +++ b/Makefile.am @@ -54,6 +54,9 @@ nodist_libstegotorus_a_SOURCES = protolist.cc steglist.cc stegotorus_SOURCES = \ src/main.cc +# prevent stegotorus from being linked if s-a-g fails +stegotorus_DEPENDENCIES = $(LDADD) stamp-audit-globals + UTGROUPS = \ src/test/unittest_base64.cc \ src/test/unittest_compression.cc \ @@ -115,11 +118,11 @@ AGLOB = $(SHELL) $(srcdir)/src/audit-globals.sh AM_V_gs = $(AM_V_gs_$(V)) AM_V_gs_ = $(AM_V_gs_$(AM_DEFAULT_VERBOSITY)) -AM_V_gs_0 = @echo " GEN " $(patsubst stamp-%,%.cc,$@); +AM_V_gs_0 = @echo " GEN " $(patsubst stamp-%,%.cc,$@); AM_V_ag = $(AM_V_ag_$(V)) AM_V_ag_ = $(AM_V_ag_$(AM_DEFAULT_VERBOSITY)) -AM_V_ag_0 = @echo " AGLOB "; +AM_V_ag_0 = @echo " AGLOB"; protolist.cc: stamp-protolist ; stamp-protolist: $(PROTOCOLS) Makefile src/genmodtable.sh @@ -141,9 +144,6 @@ stamp-audit-globals: src/audit-globals.sh Makefile \ $(AM_V_ag) $(AGLOB) $(libstegotorus_a_OBJECTS) $(stegotorus_OBJECTS) $(AM_V_at) touch stamp-audit-globals -# prevent stegotorus from being linked if s-a-g fails -EXTRA_stegotorus_DEPENDENCIES = stamp-audit-globals - # Testing check-local: @echo --- Unit tests --- diff --git a/config-aux/ranlib.m4 b/config-aux/ranlib.m4 index ce4fe33..731d320 100644 --- a/config-aux/ranlib.m4 +++ b/config-aux/ranlib.m4 @@ -59,7 +59,7 @@ if test x$RANLIB != x:; then [ac_cv_prog_RANLIB_necessary=yes], [AC_MSG_ERROR([test link failed with and without ranlib])])]) - rm -f conftest$ac_exeext conft[ABC].$ac_objext conftest.a + rm -f conftest$ac_exeext conft@<:@ABC@:>@.$ac_objext conftest.a AC_LANG_POP([C]) ]) if test $ac_cv_prog_RANLIB_necessary = no; then

1 0

[stegotorus/master] Add 'pgen_fake' payload generator, use it if traces aren't available.
by zwol＠torproject.org 20 Jul '12

20 Jul '12

commit ddc1ab0525d718072ff0805490b09eb33d631ff1 Author: Zack Weinberg <zackw(a)cmu.edu> Date: Fri Jul 13 17:04:28 2012 +0200 Add 'pgen_fake' payload generator, use it if traces aren't available. * src/pgen_fake.cc: New file. * Makefile.am: Build it. Integrate it with 'make check'. * src/compression.cc: Distinguish an inflate failure due to inadequate output buffer space from other causes. * src/steg/swfSteg.cc: Enlarge the inflate output buffer as necessary. * src/pgen.h: Make internal padding in pentry_header explicit. * src/steg/payloads.cc: Remove overly chatty log_debug messages which were causing test_tl to deadlock. --- Makefile.am | 19 ++- src/compression.cc | 4 + src/pgen.h | 2 + src/pgen_fake.cc | 662 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/steg/payloads.cc | 2 - src/steg/swfSteg.cc | 35 ++-- 6 files changed, 704 insertions(+), 20 deletions(-) diff --git a/Makefile.am b/Makefile.am index d19aab6..5416842 100644 --- a/Makefile.am +++ b/Makefile.am @@ -62,6 +62,15 @@ stegotorus_DEPENDENCIES = libstegotorus.a stamp-audit-globals ## payload trace generators +bin_PROGRAMS += pgen_fake +pgen_fake_SOURCES = \ + src/pgen_fake.cc \ + src/util.cc \ + src/rng.cc \ + src/base64.cc + +pgen_fake_LDADD = $(libcrypto_LIBS) + # pgen_pcap is only built if we have libpcap if HAVE_PCAP bin_PROGRAMS += pgen_pcap @@ -171,7 +180,15 @@ check-local: $(AM_V_at) ./unittests if INTEGRATION_TESTS @echo --- Integration tests --- - [ -e traces ] || ln -s $(srcdir)/traces . + @set -ex; if [ ! -e traces ]; then \ + if [ -e $(srcdir)/../steg-traces ]; then \ + ln -s $(srcdir)/../steg-traces traces; \ + elif [ -e $(srcdir)/traces ]; then \ + ln -s $(srcdir)/traces traces; \ + else \ + mkdir traces && ./pgen_fake; \ + fi; \ + fi $(AM_V_at) $(PYTHON) -m unittest discover -s $(srcdir)/src/test -p 'test_*.py' -v else @echo !!! Integration tests skipped !!! diff --git a/src/compression.cc b/src/compression.cc index 2a0b2ef..eefd727 100644 --- a/src/compression.cc +++ b/src/compression.cc @@ -91,6 +91,10 @@ decompress(const uint8_t *source, size_t slen, uint8_t *dest, size_t dlen) strm.avail_out = dlen; ret = inflate(&strm, Z_FINISH); + if (ret == Z_BUF_ERROR) { + inflateEnd(&strm); + return -2; // need more space + } if (ret != Z_STREAM_END) { log_warn("decompression failure: %s", strm.msg); inflateEnd(&strm); diff --git a/src/pgen.h b/src/pgen.h index c8e7a55..7dba3d4 100644 --- a/src/pgen.h +++ b/src/pgen.h @@ -14,8 +14,10 @@ /* struct for reading in the payload_gen dump file */ struct pentry_header { uint16_t ptype; + uint8_t pad1[2]; uint32_t length; uint16_t port; /* network format */ + uint8_t pad2[2]; }; #endif diff --git a/src/pgen_fake.cc b/src/pgen_fake.cc new file mode 100644 index 0000000..0d0e260 --- /dev/null +++ b/src/pgen_fake.cc @@ -0,0 +1,662 @@ +/* Copyright 2012 SRI International + * See LICENSE for other credits and copying information + */ + +#include "util.h" +#include "pgen.h" +#include "rng.h" +#include "base64.h" + +#include <string> +#include <sstream> + +using std::string; +using std::ostringstream; + +// John Bauman's 1995 revision of the "General Service List" of common +// English words -- see http://jbauman.com/aboutgsl.html -- "a" and "I" +// removed. +const char *const words[] = { + "the", "be", "of", "and", "to", "in", "he", "have", "it", "that", + "for", "they", "with", "as", "not", "on", "she", "at", "by", + "this", "we", "you", "do", "but", "from", "or", "which", "one", + "would", "all", "will", "there", "say", "who", "make", "when", "can", + "more", "if", "no", "man", "out", "other", "so", "what", "time", "up", + "go", "about", "than", "into", "could", "state", "only", "new", + "year", "some", "take", "come", "these", "know", "see", "use", "get", + "like", "then", "first", "any", "work", "now", "may", "such", "give", + "over", "think", "most", "even", "find", "day", "also", "after", + "way", "many", "must", "look", "before", "great", "back", "through", + "long", "where", "much", "should", "well", "people", "down", "own", + "just", "because", "good", "each", "those", "feel", "seem", "how", + "high", "too", "place", "little", "world", "very", "still", "nation", + "hand", "old", "life", "tell", "write", "become", "here", "show", + "house", "both", "between", "need", "mean", "call", "develop", + "under", "last", "right", "move", "thing", "general", "school", + "never", "same", "another", "begin", "while", "number", "part", + "turn", "real", "leave", "might", "want", "point", "form", "off", + "child", "few", "small", "since", "against", "ask", "late", "home", + "interest", "large", "person", "end", "open", "public", "follow", + "during", "present", "without", "again", "hold", "govern", "around", + "possible", "head", "consider", "word", "program", "problem", + "however", "lead", "system", "set", "order", "eye", "plan", "run", + "keep", "face", "fact", "group", "play", "stand", "increase", "early", + "course", "change", "help", "line", "city", "put", "close", "case", + "force", "meet", "once", "water", "upon", "war", "build", "hear", + "light", "unite", "live", "every", "country", "bring", "center", + "let", "side", "try", "provide", "continue", "name", "certain", + "power", "pay", "result", "question", "study", "woman", "member", + "until", "far", "night", "always", "service", "away", "report", + "something", "company", "week", "church", "toward", "start", "social", + "room", "figure", "nature", "though", "young", "less", "enough", + "almost", "read", "include", "president", "nothing", "yet", "better", + "big", "boy", "cost", "business", "value", "second", "why", "clear", + "expect", "family", "complete", "act", "sense", "mind", "experience", + "art", "next", "near", "direct", "car", "law", "industry", + "important", "girl", "god", "several", "matter", "usual", "rather", + "per", "often", "kind", "among", "white", "reason", "action", + "return", "foot", "care", "simple", "within", "love", "human", + "along", "appear", "doctor", "believe", "speak", "active", "student", + "month", "drive", "concern", "best", "door", "hope", "example", + "inform", "body", "ever", "least", "probable", "understand", "reach", + "effect", "different", "idea", "whole", "control", "condition", + "field", "pass", "fall", "note", "special", "talk", "particular", + "today", "measure", "walk", "teach", "low", "hour", "type", "carry", + "rate", "remain", "full", "street", "easy", "although", "record", + "sit", "determine", "level", "local", "sure", "receive", "thus", + "moment", "spirit", "train", "college", "religion", "perhaps", + "music", "grow", "free", "cause", "serve", "age", "book", "board", + "recent", "sound", "office", "cut", "step", "class", "true", + "history", "position", "above", "strong", "friend", "necessary", + "add", "court", "deal", "tax", "support", "party", "whether", + "either", "land", "material", "happen", "education", "death", "agree", + "arm", "mother", "across", "quite", "anything", "town", "past", + "view", "society", "manage", "answer", "break", "organize", "half", + "fire", "lose", "money", "stop", "actual", "already", "effort", + "wait", "department", "able", "political", "learn", "voice", "air", + "together", "shall", "cover", "common", "subject", "draw", "short", + "wife", "treat", "limit", "road", "letter", "color", "behind", + "produce", "send", "term", "total", "university", "rise", "century", + "success", "minute", "remember", "purpose", "test", "fight", "watch", + "situation", "south", "ago", "difference", "stage", "father", "table", + "rest", "bear", "entire", "market", "prepare", "explain", "offer", + "plant", "charge", "ground", "west", "picture", "hard", "front", + "lie", "modern", "dark", "surface", "rule", "regard", "dance", + "peace", "observe", "future", "wall", "farm", "claim", "firm", + "operation", "further", "pressure", "property", "morning", "amount", + "top", "outside", "piece", "sometimes", "beauty", "trade", "fear", + "demand", "wonder", "list", "accept", "judge", "paint", "mile", + "soon", "responsible", "allow", "secretary", "heart", "union", "slow", + "island", "enter", "drink", "story", "experiment", "stay", "paper", + "space", "apply", "decide", "share", "desire", "spend", "sign", + "therefore", "various", "visit", "supply", "officer", "doubt", + "private", "immediate", "wish", "contain", "feed", "raise", + "describe", "ready", "horse", "son", "exist", "north", "suggest", + "station", "effective", "food", "deep", "wide", "alone", "character", + "english", "happy", "critic", "unit", "product", "respect", "drop", + "nor", "fill", "cold", "represent", "sudden", "basic", "kill", "fine", + "trouble", "mark", "single", "press", "heavy", "attempt", "origin", + "standard", "everything", "committee", "moral", "black", "red", "bad", + "earth", "accord", "else", "mere", "die", "remark", "basis", "except", + "equal", "east", "event", "employ", "defense", "smile", "river", + "improve", "game", "detail", "account", "cent", "sort", "reduce", + "club", "buy", "attention", "ship", "decision", "wear", "inside", + "win", "suppose", "ride", "operate", "realize", "sale", "choose", + "park", "square", "vote", "price", "district", "dead", "foreign", + "window", "beyond", "direction", "strike", "instead", "trial", + "practice", "catch", "opportunity", "likely", "recognize", "permit", + "serious", "attack", "floor", "association", "spring", "lot", "stock", + "lack", "hair", "science", "relation", "profession", "pattern", + "quick", "medical", "influence", "occasion", "machine", "compare", + "husband", "blue", "international", "fair", "especially", "indeed", + "imagine", "surprise", "average", "official", "temperature", + "difficult", "sing", "hit", "tree", "race", "police", "touch", + "relative", "throw", "quality", "former", "pull", "chance", "prove", + "argue", "settle", "growth", "date", "heat", "save", "performance", + "count", "production", "listen", "main", "pick", "size", "cool", + "army", "patient", "combine", "summer", "hall", "slight", "command", + "enjoy", "length", "proper", "express", "health", "chief", "evening", + "store", "language", "degree", "lay", "current", "gun", "dog", + "hotel", "strange", "separate", "boat", "fail", "clean", "dress", + "anyone", "gain", "pain", "object", "knowledge", "depend", "relate", + "below", "dollar", "advance", "shape", "arrange", "population", "yes", + "sell", "mention", "dry", "check", "poet", "sleep", "join", "hot", + "bed", "electric", "dream", "due", "season", "manner", "fit", "left", + "progress", "neither", "strength", "notice", "finish", "opinion", + "bill", "western", "truth", "wrong", "travel", "suit", "bank", + "exact", "honor", "brother", "quiet", "marry", "corner", "handle", + "danger", "hospital", "pool", "promise", "blood", "shoot", "scene", + "literature", "arrive", "film", "base", "freedom", "bar", "maybe", + "hang", "suffer", "manufacture", "frequent", "rock", "loss", "burn", + "sun", "audience", "essential", "glass", "prevent", "poem", "poor", + "inch", "song", "skill", "post", "popular", "radio", "animal", + "conscious", "worth", "eat", "election", "faith", "wave", "murder", + "model", "forget", "extend", "edge", "distance", "memory", + "recommend", "division", "staff", "leg", "discussion", "address", + "fly", "dependent", "ball", "shake", "frame", "extreme", "engineer", + "thick", "comfort", "latter", "camp", "oil", "discover", "examine", + "difficulty", "tooth", "middle", "choice", "refer", "enemy", + "practical", "marriage", "bridge", "declare", "lady", "cross", + "daily", "afternoon", "attend", "director", "balance", "wash", + "capital", "speed", "block", "citizen", "mouth", "hill", "green", + "please", "motor", "agency", "encourage", "governor", "worry", + "affair", "shoulder", "bright", "mass", "sample", "pretty", "repeat", + "roll", "push", "trip", "council", "clothe", "parent", "forward", + "sharp", "straight", "gas", "weight", "discuss", "fix", "load", + "master", "whatever", "round", "rapid", "laugh", "finger", "spot", + "propose", "shop", "broad", "replace", "reply", "extent", "lock", + "employee", "ahead", "sight", "spread", "wind", "approve", "destroy", + "none", "pound", "fame", "importance", "reflect", "advantage", + "match", "regular", "wage", "refuse", "existence", "hardly", + "perform", "title", "tend", "exercise", "thin", "coat", "bit", + "mountain", "youth", "behavior", "newspaper", "secret", "ability", + "sea", "soft", "justice", "reasonable", "circle", "solid", "page", + "weapon", "fast", "representative", "search", "pure", "escape", + "crowd", "stick", "telephone", "avoid", "garden", "favor", "news", + "unless", "dinner", "someone", "signal", "yard", "ideal", "warm", + "miss", "shelter", "soldier", "article", "cry", "captain", "familiar", + "seat", "guest", "weak", "excite", "king", "everyone", "wine", "hole", + "duty", "beat", "perfect", "bottom", "compose", "battle", "expense", + "cattle", "flow", "kitchen", "dust", "bottle", "admit", "tear", + "tire", "expression", "exception", "application", "belong", "rich", + "failure", "struggle", "instrument", "variety", "narrow", "theater", + "collection", "rain", "review", "preserve", "leadership", "clay", + "daughter", "fellow", "swing", "thank", "library", "fat", "reserve", + "tour", "nice", "warn", "ring", "bitter", "chair", "yesterday", + "scientific", "flower", "wheel", "solution", "aim", "gather", + "invite", "moreover", "fresh", "forest", "winter", "box", "belief", + "ordinary", "impossible", "print", "gray", "taste", "lip", "speech", + "reference", "stain", "connection", "otherwise", "stretch", "knife", + "village", "blow", "mistake", "sweet", "shout", "divide", "guard", + "worse", "exchange", "rare", "commercial", "request", "appoint", + "agent", "dependence", "bird", "wild", "motion", "guess", "neighbor", + "seed", "fashion", "loan", "correct", "plain", "mail", "retire", + "opposite", "prefer", "safe", "evil", "double", "wood", "empty", + "baby", "advise", "content", "sport", "lift", "literary", "curious", + "tie", "flat", "message", "neck", "hate", "dirt", "delight", "trust", + "nobody", "valley", "tool", "presence", "cook", "railroad", + "minister", "coffee", "brush", "beside", "collect", "guide", "luck", + "profit", "lord", "everybody", "prison", "cloud", "slave", "chairman", + "soil", "distinguish", "introduce", "urge", "blind", "arise", "upper", + "curve", "membership", "key", "entertain", "soul", "neighborhood", + "friendly", "pair", "stone", "lean", "protect", "advertise", + "mystery", "welcome", "knee", "jump", "snake", "stream", "avenue", + "brown", "disease", "hat", "excellent", "formal", "snow", "sheet", + "somehow", "unity", "sky", "rough", "smooth", "weather", "steady", + "threaten", "depth", "oppose", "deliver", "ancient", "pray", "adopt", + "birth", "appearance", "universe", "busy", "hurry", "coast", "forth", + "smell", "furnish", "female", "hide", "wire", "proposal", "ought", + "victory", "quarter", "engine", "customer", "waste", "fool", "intend", + "intention", "desk", "politics", "passage", "lawyer", "root", "climb", + "metal", "gradual", "hunt", "protection", "satisfy", "roof", "branch", + "pleasure", "witness", "loose", "nose", "mine", "band", "aside", + "risk", "tomorrow", "remind", "ear", "fish", "shore", "operator", + "civilize", "being", "silent", "screen", "bind", "earn", "pack", + "colony", "besides", "slip", "cousin", "scale", "relief", "explore", + "stem", "brain", "musician", "defend", "bend", "somebody", "shadow", + "mix", "smoke", "description", "fruit", "guilt", "yield", "sensitive", + "salt", "pale", "sweep", "completion", "throat", "agriculture", + "admire", "gentle", "dozen", "particle", "pleasant", "bay", "cup", + "competition", "moon", "terrible", "strip", "mechanic", "shock", + "conversation", "angle", "tall", "plenty", "star", "yellow", "sick", + "thorough", "absolute", "succeed", "surround", "proud", "dear", + "card", "lake", "breath", "afraid", "silence", "onto", "shoe", + "somewhere", "chain", "slide", "copy", "machinery", "wake", "severe", + "pocket", "bone", "honest", "freeze", "dictionary", "calm", "swim", + "ice", "male", "skin", "crack", "rush", "wet", "meat", "commerce", + "joint", "gift", "host", "suspect", "path", "uncle", "afford", + "instant", "satisfactory", "height", "track", "confidence", "grass", + "suggestion", "favorite", "breakfast", "apart", "chest", "entrance", + "march", "sink", "northern", "iron", "alive", "ill", "bag", "disturb", + "native", "bedroom", "violent", "beneath", "pause", "tough", + "substance", "threat", "charm", "absence", "factory", "spite", "meal", + "universal", "accident", "highway", "sentence", "liberty", "wise", + "noise", "discovery", "tube", "flash", "twist", "fence", "childhood", + "joy", "sister", "sad", "efficiency", "disappear", "defeat", + "extensive", "rent", "comparison", "possess", "grace", "flesh", + "liquid", "scientist", "ease", "heaven", "milk", "sympathy", "rank", + "restaurant", "frequency", "angry", "shade", "accuse", "necessity", + "knock", "loud", "permanent", "row", "lovely", "confuse", "gold", + "frighten", "solve", "grave", "salary", "photograph", "advice", + "abroad", "wound", "virtue", "dare", "queen", "extra", "attract", + "numerous", "pink", "gate", "expensive", "shut", "chicken", "forgive", + "holy", "wooden", "prompt", "crime", "sorry", "republic", "anger", + "visitor", "pile", "violence", "steel", "wing", "stair", "partner", + "delay", "gentleman", "pour", "confusion", "damage", "kick", "safety", + "burst", "network", "resistance", "screw", "pride", "till", "hire", + "verb", "preach", "clerk", "everywhere", "anyway", "fan", "connect", + "egg", "efficient", "grain", "calculate", "drag", "opposition", + "worship", "arrest", "discipline", "string", "harbor", "camera", + "mechanism", "cow", "grand", "funny", "insurance", "reduction", + "strict", "lesson", "tight", "sand", "plate", "qualify", "elsewhere", + "mad", "interference", "pupil", "fold", "royal", "valuable", + "whisper", "anybody", "hurt", "excess", "quantity", "fun", "mud", + "extension", "recognition", "kiss", "crop", "sail", "attractive", + "habit", "relieve", "wisdom", "persuade", "certainty", "cloth", + "eager", "deserve", "sympathetic", "cure", "trap", "puzzle", "powder", + "raw", "mankind", "glad", "blame", "whenever", "anxiety", "bus", + "tremble", "sacred", "fortunate", "glory", "golden", "neat", + "weekend", "treasury", "overcome", "cat", "sacrifice", "complain", + "elect", "roar", "sake", "temple", "self", "compete", "nurse", + "stuff", "stomach", "peculiar", "repair", "storm", "ton", "desert", + "allowance", "servant", "hunger", "conscience", "bread", "crash", + "tip", "strengthen", "proof", "generous", "sir", "tonight", "whip", + "tongue", "mill", "merchant", "coal", "ruin", "introduction", + "courage", "actor", "belt", "stir", "package", "punish", "reflection", + "breathe", "anywhere", "amuse", "dull", "fate", "net", "fellowship", + "fault", "furniture", "beam", "pencil", "border", "disappoint", + "flame", "joke", "bless", "corn", "shell", "tempt", "supper", + "destruction", "dive", "anxious", "shine", "cheap", "dish", "distant", + "greet", "flood", "excuse", "insect", "ocean", "ceremony", "decrease", + "prize", "harm", "insure", "verse", "pot", "sincere", "cotton", + "leaf", "rub", "medicine", "stroke", "bite", "lung", "lonely", + "admission", "stupid", "scratch", "composition", "broadcast", "drum", + "resist", "neglect", "absent", "passenger", "adventure", "beg", + "pipe", "beard", "bold", "meanwhile", "devil", "cheer", "nut", + "split", "melt", "swear", "sugar", "bury", "wipe", "faint", + "creature", "tail", "wealth", "earnest", "translate", "suspicion", + "noble", "inquiry", "journey", "hesitate", "extraordinary", "borrow", + "owe", "funeral", "ambition", "mixture", "slope", "criminal", + "seldom", "map", "spin", "praise", "spare", "plow", "telegraph", + "barrel", "straighten", "scarce", "lunch", "slavery", "creep", + "sweat", "gay", "stiff", "brave", "seize", "convenient", "horizon", + "moderate", "complicate", "dig", "curse", "weigh", "priest", + "excessive", "quarrel", "widow", "modest", "dine", "politician", + "custom", "educate", "salesman", "nail", "tap", "eastern", + "possession", "satisfaction", "behave", "mercy", "scatter", + "objection", "silver", "tent", "saddle", "wrap", "nest", "grind", + "spell", "plaster", "arch", "swell", "friendship", "bath", "bundle", + "grateful", "crown", "boundary", "nowhere", "asleep", "clock", "boil", + "altogether", "lend", "holiday", "precious", "wander", "ugly", + "reputation", "ticket", "pretend", "dismiss", "delicate", "despair", + "awake", "tea", "false", "fortune", "cap", "thread", "haste", "bare", + "shirt", "bargain", "leather", "rail", "butter", "dot", "inquire", + "warmth", "decisive", "vessel", "pity", "steam", "pin", "bound", + "companion", "toe", "reward", "forbid", "wherever", "tower", "bathe", + "lodge", "swallow", "multiply", "bow", "kingdom", "garage", + "permission", "pump", "prevention", "urgent", "aunt", "zero", "idle", + "fever", "christmas", "regret", "jaw", "soap", "pronounce", "empire", + "bowl", "outline", "organ", "imitation", "caution", "mineral", + "disagree", "blade", "trick", "treasure", "immense", "convenience", + "disapprove", "destructive", "fork", "noon", "ownership", "tune", + "polish", "poison", "shame", "loyalty", "cottage", "astonish", + "shave", "feather", "sauce", "lid", "debt", "fade", "confess", + "classification", "descend", "cape", "mild", "clever", "envelope", + "invention", "sheep", "splendid", "stamp", "float", "brick", "rice", + "businessman", "backward", "qualification", "artificial", + "attraction", "lamp", "curl", "shower", "elder", "bunch", "bell", + "steer", "flavor", "spit", "rob", "cream", "interrupt", "pen", + "weave", "orange", "rescue", "crush", "humble", "fancy", "decay", + "polite", "tribe", "bleed", "coin", "fond", "autumn", "classify", + "omit", "loyal", "needle", "lessen", "complaint", "pad", "steep", + "skirt", "curtain", "calculation", "laughter", "solemn", "grease", + "interfere", "explode", "fasten", "flag", "resign", "postpone", + "patience", "boast", "rope", "envy", "airplane", "rid", "shield", + "veil", "kneel", "tray", "explosive", "brass", "taxi", "wax", "duck", + "button", "invent", "remedy", "bush", "thunder", "weaken", "poverty", + "scrape", "arrow", "tender", "cruel", "soften", "mouse", "hay", + "anyhow", "alike", "circular", "juice", "shelf", "bake", "hatred", + "cautious", "basket", "wreck", "width", "confident", "log", "heap", + "suck", "ladder", "gap", "obey", "hut", "axe", "translation", + "collar", "delivery", "reproduce", "confession", "pan", "prejudice", + "voyage", "tobacco", "simplicity", "paste", "cake", "elephant", + "ribbon", "harvest", "ashamed", "cave", "customary", "thief", "damp", + "sew", "rust", "separation", "waiter", "pet", "straw", "upset", + "towel", "refresh", "essence", "fur", "ambitious", "defendant", + "daylight", "dip", "suspicious", "imaginary", "ash", "carriage", + "educator", "saw", "stove", "rubber", "rug", "misery", "awkward", + "rival", "roast", "deed", "preference", "explosion", "theatrical", + "cultivate", "collector", "miserable", "wrist", "rabbit", "accustom", + "tide", "insult", "thumb", "lump", "annoy", "toy", "heal", "shallow", + "repetition", "soup", "whistle", "scenery", "apple", "offense", + "cork", "ripe", "temper", "sore", "pinch", "diamond", "razor", + "imaginative", "hook", "copper", "landlord", "influential", "rot", + "hollow", "enclose", "harden", "wicked", "stiffen", "silk", "upright", + "selfish", "stripe", "pig", "inward", "excellence", "rake", "purple", + "hasten", "shorten", "applause", "ache", "apology", "knot", "nephew", + "cushion", "drown", "nursery", "pint", "fierce", "imitate", "aloud", + "gaiety", "robbery", "tighten", "perfection", "scorn", "whoever", + "trunk", "wool", "sailor", "competitor", "moonlight", "deer", "bean", + "everyday", "drawer", "disregard", "nowadays", "patriotic", "tin", + "penny", "cage", "pardon", "lately", "offend", "coarse", "spoil", + "horizontal", "sting", "ditch", "librarian", "meantime", "cough", + "deaf", "sword", "messenger", "vain", "castle", "elastic", "comb", + "rod", "widen", "sorrow", "inventor", "cliff", "umbrella", + "interruption", "merry", "gallon", "conquest", "headache", "tailor", + "bucket", "scent", "signature", "cart", "darken", "sometime", + "applaud", "underneath", "hello", "pretense", "descent", "conquer", + "framework", "confidential", "adoption", "disgust", "waist", + "momentary", "receipt", "pearl", "ray", "lazy", "limb", "grammatical", + "beast", "monkey", "jewel", "persuasion", "obedience", "sock", + "vowel", "hammer", "inn", "chimney", "dissatisfaction", "annoyance", + "ornament", "honesty", "outward", "sharpen", "handkerchief", "greed", + "heavenly", "thirst", "niece", "spill", "loaf", "wheat", "worm", + "secrecy", "rude", "heighten", "flatten", "loosen", "cheese", + "rivalry", "royalty", "discontent", "complication", "fright", + "indoor", "flour", "actress", "congratulation", "ounce", "fry", + "everlasting", "goat", "ink", "disappearance", "reproduction", + "thicken", "avoidance", "spoon", "strap", "deceive", "lengthen", + "revenge", "correction", "descendant", "hesitation", "spade", "basin", + "weed", "omission", "old-fashioned", "bicycle", "breadth", + "photography", "coward", "mat", "rejoice", "cheat", "congratulate", + "discomfort", "enclosure", "attentive", "paw", "overflow", + "dissatisfy", "multiplication", "whichever", "tidy", "bribe", "mend", + "stocking", "feast", "nuisance", "thorn", "tame", "inclusive", + "homemade", "handwriting", "chalk", "sour", "slippery", "procession", + "ripen", "jealous", "jealousy", "liar", "homecoming", "barber", + "whiten", "berry", "lighten", "pigeon", "hinder", "bravery", + "baggage", "noun", "amongst", "grammar", "cultivation", + "companionship", "rubbish", "modesty", "woolen", "deepen", "pastry", + "cupboard", "quart", "canal", "notebook", "deceit", "parcel", + "brighten", "moderation", "punctual", "hurrah", "lipstick", + "uppermost", "fatten", "conqueror", "hindrance", "cowardice", + "obedient", "saucer", "madden", "scold", "weekday", "rotten", + "disrespect", "widower", "deafen", "donkey", "businesslike", + "motherhood", "sadden", "handshake", "calculator", "headdress", + "scissors", "translator", "possessor", "shilling", "redden", + "motherly", "whose", "cultivator", "whom", "homework", "electrician", + "oar", "bribery", "sweeten", "sow", "pronunciation", "beak", "plural", +}; + +const size_t nwords = (sizeof words) / (sizeof words[0]); + +enum payload_type { + PT_HTML, + PT_JS, + PT_SWF, + PT_PDF +}; + +const char *const type_extensions[] = { ".html", ".js", ".swf", ".pdf" }; +const char *const type_mimes[] = { + "text/html; charset=utf-8", + "text/javascript", + "application/x-shockwave-flash", + "application/pdf" +}; + +// payloads.cc uses the *file extension* on the URL to decide what to +// send back. Use HTML half of the time, JS three-quarters of the +// remaining time, and PDF or SWF each half of what's left over. +static payload_type +pick_payload_type() +{ + uint8_t b; + rng_bytes(&b, 1); + if (b >= 128) + return PT_HTML; + else if (b >= 32) + return PT_JS; + else if (b >= 16) + return PT_SWF; + else + return PT_PDF; +} + +static void +gen_one_uripath(ostringstream& os) +{ + int n = rng_range_geom(10, 3); + for (int i = 0; i < n; i++) + os << words[rng_range_geom(nwords, nwords/3)] << '/'; + + os << words[rng_range_geom(nwords, nwords/3)]; + os << type_extensions[pick_payload_type()]; +} + +static void +gen_one_hostname(ostringstream& os) +{ + unsigned int choices = rng_int(0x10); + bool use_www = choices & 0x01; + bool use_subd = choices & 0x02; + unsigned int tld = (choices & 0x0C) >> 2; + + const char *const tlds[4] = { ".com", ".org", ".sv", ".ac.uk" }; + + if (use_www) + os << "www."; + if (use_subd) + os << words[rng_range_geom(nwords, nwords/3)] << '.'; + + os << words[rng_range_geom(nwords, nwords/3)] << tlds[tld]; +} + +static void +gen_one_cookie_header(ostringstream& os) +{ + int n = rng_range(1,5); + uint8_t buf[80]; + char obuf[160]; + int m; + ptrdiff_t mo; + base64::encoder enc(false, '_', '.', '-'); + + for (int i = 0; i < n; i++) { + os << words[rng_range_geom(nwords, nwords/3)] << '='; + + m = rng_range_geom(80, 20); + rng_bytes(buf, m); + mo = enc.encode((const char *)buf, m, obuf); + mo += enc.encode_end(obuf + mo); + obuf[mo] = '\0'; + os << obuf; + + if (i+1 < n) + os << ','; + } +} + +static void +gen_one_html(ostringstream& cs, size_t approx_size) +{ + // HTML needs to be substantially bigger than anything else, + // since we can only use the scripts which are only a small part + // of the file. + approx_size *= 5; + + cs << "<!doctype html>\n<html><head>\n<title>"; + int n = rng_range_geom(6, 2); + for (int i = 0; i < n; i++) + cs << words[rng_int(nwords)] << ' '; + cs << words[rng_int(nwords)] + << "</title>\n</head><body>\n<p>"; + + n = rng_range_geom(50, 20); + bool in_script = false; + do { + cs << words[rng_int(nwords)] << ' '; + n--; + if (n <= 0) { + n = rng_range_geom(50, 20); + if (in_script) { + cs << "</script>\n<p>"; + in_script = false; + } else { + // jsSteg insists on <script type="text/javascript"> for no + // apparent reason (and this is as a fixed string, not as + // properly parsed HTML). + cs << "</p>\n<script type=\"text/javascript\">"; + in_script = true; + } + } + } while (size_t(cs.tellp()) < approx_size); + + cs << (in_script ? "</script>" : "</p>") << "\n</body></html>\n"; +} + +static void +gen_one_js(ostringstream& cs, size_t approx_size) +{ + const char *const js_keywords[] = { + "break", "case", "catch", "class", "continue", "debugger", "default", + "delete", "do", "else", "enum", "export", "extends", "false", + "finally", "for", "function", "if", "implements", "import", "in", + "instanceof", "interface", "let", "new", "null", "package", "private", + "protected", "public", "return", "static", "super", "switch", "this", + "throw", "true", "try", "typeof", "var", "void", "while", "with", "yield", + }; + const size_t n_js_keywords = (sizeof js_keywords) / (sizeof js_keywords[0]); + + const char *const js_punct[] = { + "(", ")", "[", "]", "{", "}", ":", ";", ".", ",", + "+", "-", "/", "*", "%", "++", "--", "&", "|", "<<", ">>", ">>>", + "=", "*=", "/=", "%=", "+=", "-=", "<<=", ">>=", ">>>=", "&=", "^=", "|=", + "==", "!=", "===", "!==", ">", ">=", "<", "<=", "&&", "||", "!" + }; + const size_t n_js_punct = (sizeof js_punct) / (sizeof js_punct[0]); + + do { + uint8_t which; + rng_bytes(&which, 1); + if (which < 32) + cs << js_keywords[rng_int(n_js_keywords)]; + else + cs << words[rng_range_geom(nwords, nwords/3)]; + + rng_bytes(&which, 1); + if (which < 128) + cs << " "; + else + cs << js_punct[rng_int(n_js_punct)]; + + } while (size_t(cs.tellp()) < approx_size); +} + +static void +gen_one_swf(ostringstream& cs, size_t approx_size) +{ + // This does not attempt to produce the SWF file format *at all*, + // only its magic number and length field. swfSteg.cc is presently + // very nearly as callous: it preserves the first 1508 and last 1500 + // bytes of the file, and makes the length field be accurate. + uint32_t size = approx_size + 3008; + + cs << "CWS\t"; // compressed, version 9; it's not compressed now, + // but it will be after swfSteg.cc gets done with it + cs << uint8_t((size & 0x000000ff)) // length is little endian + << uint8_t((size & 0x0000ff00) >> 8) + << uint8_t((size & 0x00ff0000) >> 16) + << uint8_t((size & 0xff000000) >> 24); + + for (int i = 0; i < 1500; i++) + cs << '\xEE'; + for (size_t i = 0; i < approx_size; i++) + cs << '\xDD'; + for (int i = 0; i < 1500; i++) + cs << '\xCC'; +} + +static void +gen_one_pdf(ostringstream& cs, size_t approx_size) +{ + // This only duplicates the part of the PDF format that pdfSteg.cc + // actually looks for: in particular, we do not attempt to generate + // a valid trailer. (It wouldn't be terribly hard to add.) + + int ctr = 1; + cs << "%PDF-1.5\n%\xA0\xA1\xA2\xA3\n"; + do { + int size = rng_range_geom(2048, 512); + + cs << ctr << " 0 obj <</Length " << size << ">>\nstream\n"; + ctr++; + + for (int i = 0; i < size; i++) + cs << '\xBB'; + + cs << "\nendstream\nendobj\n"; + } while (size_t(cs.tellp()) < approx_size); + + cs << "%%EOF\n"; +} + +static void +gen_one_client_trace(ostringstream& os, pentry_header& pe) +{ + pe.ptype = htons(TYPE_HTTP_REQUEST); + pe.port = htons(80); + + os << "GET /"; + + gen_one_uripath(os); + + os << " HTTP/1.1\r\nHost: "; + + gen_one_hostname(os); + + os << + "\r\nUser-Agent: Mozilla/5.0 (Macintosh; " + "Intel Mac OS X 10.6; rv:10.0) Gecko/20100101 Firefox/10.0" + "\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + "\r\nAccept-Language: en-us,en;q=0.5" + "\r\nAccept-Encoding: gzip, deflate" + "\r\nCookie: "; + + gen_one_cookie_header(os); + + os << "\r\nConnection: keep-alive\r\n\r\n"; +} + +static void +gen_one_server_trace(ostringstream& os, pentry_header& pe) +{ + typedef void (*gen_payload_f)(ostringstream&, size_t); + const gen_payload_f type_payloadgens[] = { + gen_one_html, gen_one_js, gen_one_swf, gen_one_pdf + }; + + pe.ptype = htons(TYPE_HTTP_RESPONSE); + pe.port = htons(80); + + payload_type pt = pick_payload_type(); + size_t approx_size = rng_range_geom(16384, 4096); + + ostringstream cs; + type_payloadgens[pt](cs, approx_size); + string const& content = cs.str(); + + os << + "HTTP/1.1 200 OK\r\n" + "Server: Apache\r\n" + "Accept-Ranges: bytes\r\n" + "Content-Type: " << type_mimes[pt] << "\r\n" + "Content-Length: " << content.size() << "\r\n" + "Connection: keep-alive\r\n\r\n" << content; +} + +static void +gen_traces(unsigned long n, const char *fname, + void (*gen_one)(ostringstream&, pentry_header&)) +{ + FILE *fp = fopen(fname, "wb"); + if (!fp) { + perror(fname); + exit(1); + } + + for (unsigned long i = 0; i < n; i++) { + pentry_header pe; + memset(&pe, 0, sizeof(pe)); + + ostringstream os; + gen_one(os, pe); + + string const& o = os.str(); + pe.length = htonl(o.size()); + fwrite(&pe, sizeof(pentry_header), 1, fp); + fwrite(o.data(), o.size(), 1, fp); + } + + if (ferror(fp) || fclose(fp)) { + perror(fname); + exit(1); + } +} + +int +main() +{ + gen_traces(10000, "traces/client.out", gen_one_client_trace); + gen_traces(10000, "traces/server.out", gen_one_server_trace); +} diff --git a/src/steg/payloads.cc b/src/steg/payloads.cc index 3af80d4..20dd624 100644 --- a/src/steg/payloads.cc +++ b/src/steg/payloads.cc @@ -1297,9 +1297,7 @@ init_PDF_payload_pool(payloads& pl, int len, int type, int minCapacity) // can encode in the pdf doc // cap = minCapacity+1; cap = capacityPDF(msgbuf, p->length); - log_debug("got pdf (index %d) with capacity %d", r, cap); if (cap > minCapacity) { - log_debug("pdf (index %d) greater than mincapacity %d", cnt, minCapacity); pl.typePayloadCap[contentType][cnt] = (cap-PDF_DELIMITER_SIZE)/2; pl.typePayload[contentType][cnt] = r; cnt++; diff --git a/src/steg/swfSteg.cc b/src/steg/swfSteg.cc index 1510043..8fb29ee 100644 --- a/src/steg/swfSteg.cc +++ b/src/steg/swfSteg.cc @@ -86,30 +86,31 @@ swf_wrap(payloads& pl, char* inbuf, int in_len, char* outbuf, int out_sz) { -unsigned int -swf_unwrap(char* inbuf, int in_len, char* outbuf, int out_sz) { - char* tmp_buf; +unsigned int +swf_unwrap(char* inbuf, int in_len, char* outbuf, int out_sz) +{ int inf_len; + size_t tmp_len = in_len * 32; + char* tmp_buf = (char *)xmalloc(tmp_len); + + for (;;) { + inf_len = decompress((const uint8_t *)inbuf + 8, in_len - 8, + (uint8_t *)tmp_buf, tmp_len); + if (inf_len != -2) + break; + tmp_len *= 2; + tmp_buf = (char *)xrealloc(tmp_buf, tmp_len); + } - tmp_buf = (char *)xmalloc(in_len * 8); - - inf_len = decompress((const uint8_t *)inbuf + 8, in_len - 8, - (uint8_t *)tmp_buf, in_len * 8); - - // fprintf(stderr, "in_swf_len = %d\n", in_len -8 ); - - - if (inf_len < 0 || out_sz < inf_len - SWF_SAVE_HEADER_LEN - SWF_SAVE_FOOTER_LEN) { + if (inf_len < 0 || + out_sz < inf_len - SWF_SAVE_HEADER_LEN - SWF_SAVE_FOOTER_LEN) { fprintf(stderr, "inf_len = %d\n", inf_len); free(tmp_buf); - // buf_dump((unsigned char*) (inbuf+8), in_len -8, stderr); - - - return -1; } - memcpy(outbuf, tmp_buf + SWF_SAVE_HEADER_LEN, inf_len - SWF_SAVE_HEADER_LEN - SWF_SAVE_FOOTER_LEN); + memcpy(outbuf, tmp_buf + SWF_SAVE_HEADER_LEN, + inf_len - SWF_SAVE_HEADER_LEN - SWF_SAVE_FOOTER_LEN); return inf_len - SWF_SAVE_HEADER_LEN - SWF_SAVE_FOOTER_LEN; }

1 0

[stegotorus/master] Delete traces in 'make clean' if we generated them.
by zwol＠torproject.org 20 Jul '12

20 Jul '12

commit e5429ffb1badee2547c685c62bef3685b7798c0b Author: Zack Weinberg <zackw(a)cmu.edu> Date: Sat Jul 14 18:42:40 2012 +0200 Delete traces in 'make clean' if we generated them. --- Makefile.am | 7 ++++++- 1 files changed, 6 insertions(+), 1 deletions(-) diff --git a/Makefile.am b/Makefile.am index 5416842..ee1d46d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -186,10 +186,15 @@ if INTEGRATION_TESTS elif [ -e $(srcdir)/traces ]; then \ ln -s $(srcdir)/traces traces; \ else \ - mkdir traces && ./pgen_fake; \ + mkdir traces && touch traces/.faked && ./pgen_fake; \ fi; \ fi $(AM_V_at) $(PYTHON) -m unittest discover -s $(srcdir)/src/test -p 'test_*.py' -v else @echo !!! Integration tests skipped !!! endif + +if INTEGRATION_TESTS +clean-local: + [ ! -f traces/.faked ] || rm -r traces +endif

1 0

[stegotorus/master] Don't allow connection count to grow without limit in HTTP steg.
by zwol＠torproject.org 20 Jul '12

20 Jul '12

commit 83648b9054337c6276077a342c7d7653a9daafca Author: Zack Weinberg <zackw(a)cmu.edu> Date: Mon Jul 16 15:04:12 2012 -0700 Don't allow connection count to grow without limit in HTTP steg. The fix involves two complementary changes: (1) restore the upper limit of 64 outstanding downstream connections per circuit; (2) HTTP steg needs to call ->cease_transmission() and ->expect_close() on its connections at the right times (for now, unconditionally; in the future, paying attention to the HTTP "Connection:" header). --- src/protocol/chop.cc | 3 ++- src/steg/http.cc | 13 ++++++++++++- src/steg/jsSteg.cc | 2 -- src/steg/pdfSteg.cc | 2 -- src/steg/swfSteg.cc | 3 --- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/protocol/chop.cc b/src/protocol/chop.cc index bb4a95c..3059c1b 100644 --- a/src/protocol/chop.cc +++ b/src/protocol/chop.cc @@ -479,7 +479,8 @@ chop_circuit_t::send() // reopening new connections. If we're the server, we have to // just twiddle our thumbs and hope the client does that. if (no_target_connection) { - if (config->mode != LSN_SIMPLE_SERVER) + if (config->mode != LSN_SIMPLE_SERVER && + downstreams.size() < 64) circuit_reopen_downstreams(this); else circuit_arm_axe_timer(this, axe_interval()); diff --git a/src/steg/http.cc b/src/steg/http.cc index 7caf2e7..381258c 100644 --- a/src/steg/http.cc +++ b/src/steg/http.cc @@ -588,7 +588,13 @@ http_steg_t::transmit(struct evbuffer *source) break; } - if (rval == 0) have_transmitted = 1; + if (rval == 0) { + have_transmitted = 1; + // FIXME: should decide whether or not to do this based on the + // Connection: header. (Needs additional changes elsewhere, esp. + // in transmit_room.) + conn->cease_transmission(); + } return rval; } } @@ -671,6 +677,11 @@ http_server_receive(http_steg_t *s, conn_t *conn, struct evbuffer *dest, struct s->have_received = 1; s->type = type; + // FIXME: should decide whether or not to do this based on the + // Connection: header. (Needs additional changes elsewhere, esp. + // in transmit_room.) + conn->expect_close(); + conn->transmit_soon(100); return RECV_GOOD; } diff --git a/src/steg/jsSteg.cc b/src/steg/jsSteg.cc index dbe4431..3954829 100644 --- a/src/steg/jsSteg.cc +++ b/src/steg/jsSteg.cc @@ -893,8 +893,6 @@ http_server_JS_transmit (payloads& pl, struct evbuffer *source, conn_t *conn, evbuffer_drain(source, sbuflen); free(outbuf2); - conn->cease_transmission(); - // downcast_steg(s)->have_transmitted = 1; return 0; } diff --git a/src/steg/pdfSteg.cc b/src/steg/pdfSteg.cc index f76da99..8bff423 100644 --- a/src/steg/pdfSteg.cc +++ b/src/steg/pdfSteg.cc @@ -464,8 +464,6 @@ http_server_PDF_transmit(payloads &pl, struct evbuffer *source, } evbuffer_drain(source, sbuflen); - - conn->cease_transmission(); return 0; } diff --git a/src/steg/swfSteg.cc b/src/steg/swfSteg.cc index 8fb29ee..8e12d21 100644 --- a/src/steg/swfSteg.cc +++ b/src/steg/swfSteg.cc @@ -152,9 +152,6 @@ http_server_SWF_transmit(payloads& pl, struct evbuffer *source, conn_t *conn) return -1; } - - conn->cease_transmission(); - free(inbuf); free(outbuf); return 0;

1 0

[stegotorus/master] Fix pgen_pcap.cc compile failure on Linux.
by zwol＠torproject.org 20 Jul '12

20 Jul '12

commit a96d571658c89b40b1bb9829b0dcd24ce37187af Author: Zack Weinberg <zackw(a)cmu.edu> Date: Mon Jul 16 15:36:18 2012 -0700 Fix pgen_pcap.cc compile failure on Linux. Glibc's headers are not as promiscuous about including each other as OSX's are. Also, apparently there exist at least two incompatible definitions of 'struct tcphdr'. Down, not across. --- src/pgen_pcap.cc | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-) diff --git a/src/pgen_pcap.cc b/src/pgen_pcap.cc index cb6e475..ff17c43 100644 --- a/src/pgen_pcap.cc +++ b/src/pgen_pcap.cc @@ -6,12 +6,16 @@ #include "pgen.h" #include "compression.h" +#include <dirent.h> +#include <signal.h> +#include <unistd.h> #include <pcap/pcap.h> + +#define __FAVOR_BSD 1 #include <netinet/in.h> #include <netinet/if_ether.h> #include <netinet/ip.h> #include <netinet/tcp.h> -#include <dirent.h> #define NUM_FLOWS 1000 #define NUM_LISTS 1000

1 0

[stegotorus/master] b64 encoding support for http cookies
by zwol＠torproject.org 20 Jul '12

20 Jul '12

commit 696648906b49690340ea16517060a932d61e1e2e Author: Vinod Yegneswaran <vinod(a)safdef.isc.org> Date: Tue Feb 14 12:21:51 2012 -0800 b64 encoding support for http cookies --- src/steg/b64cookies.cc | 233 ++++++++++++++++++++++++++++++++++++++++++++++++ src/steg/b64cookies.h | 20 ++++ src/steg/b64decode.cc | 88 ++++++++++++++++++ src/steg/b64decode.h | 66 ++++++++++++++ src/steg/b64encode.cc | 139 ++++++++++++++++++++++++++++ src/steg/b64encode.h | 75 +++++++++++++++ 6 files changed, 621 insertions(+), 0 deletions(-) diff --git a/src/steg/b64cookies.cc b/src/steg/b64cookies.cc new file mode 100644 index 0000000..975c201 --- /dev/null +++ b/src/steg/b64cookies.cc @@ -0,0 +1,233 @@ + +#include "b64cookies.h" +#include <string.h> + +int unwrap_b64_cookie(char* inbuf, char* outbuf, int buflen) { + int i,j; + j = 0; + + for (i=0; i < buflen; i++) { + char c = inbuf[i]; + + if (c != ' ' && c != ';' && c != '=') + outbuf[j++] = c; + } + + return j; + +} + + + + + +int gen_one_b64cookie(char* outbuf, int& cookielen, char* data, int datalen) { + int sofar = 0; + int namelen; + int data_consumed = 0; + + cookielen = 4 + rand() % (datalen - 3); + + if (cookielen > 13) + namelen = rand() % 10 + 1; + else + namelen = rand() % (cookielen - 3) + 1; + + + while (sofar < namelen) { + outbuf[sofar++] = data[data_consumed++]; + } + + + outbuf[sofar++] = '='; + + while (sofar < cookielen) { + outbuf[sofar++] = data[data_consumed++]; + } + + return data_consumed; + +} + + + + +/* returns length of cookie */ +int gen_b64_cookie_field(char* outbuf, char* data, int datalen) { + int onecookielen; + int consumed = 0; + int cookielen = 0; + + + + + while (datalen - consumed > 0) { + int cnt = gen_one_b64cookie(outbuf, onecookielen, data + consumed, datalen - consumed); + + if (cnt < 0) { + fprintf(stderr, "error: couldn't create cookie %d\n", cnt); + return cnt; + } + + consumed += cnt; + outbuf += onecookielen; + cookielen += onecookielen; + + if (datalen - consumed < 5) { + memcpy(outbuf, data+consumed, datalen-consumed); + return cookielen + datalen - consumed; + } + + + if (datalen - consumed > 0) { + outbuf[0] = ';'; + outbuf++; + cookielen++; + } + } + + + return cookielen; +} + + + +void sanitize_b64(char* input, int len) { + char* i = strchr(input, '/'); + int eqcnt = 0; + + printf("len = %d\n", len); + + while (i != NULL) { + *i = '_'; + i = strchr(i+1, '/'); + } + + i = strchr(input, '+'); + + while (i != NULL) { + *i = '.'; + i = strchr(i+1, '+'); + } + + if (input[len-2] == '=') + eqcnt = 2; + else if (input[len-1] == '=') + eqcnt = 1; + + + while (eqcnt > 0) { + int pos = rand() % (len - 1) + 1; + if (pos >= len - eqcnt) { + input[pos] = '-'; + eqcnt--; + continue; + } + + //shift characters up and insert '-' in the middle + for (int j=len-eqcnt; j > pos; j--) + input[j] = input[j-1]; + + input[pos] = '-'; + eqcnt--; + + } + +} + + +void desanitize_b64(char* input, int len) { + char* i = strchr(input, '_'); + + + printf("len = %d\n", len); + while (i != NULL) { + *i = '/'; + i = strchr(i+1, '_'); + } + + i = strchr(input, '.'); + + while (i != NULL) { + *i = '+'; + i = strchr(i+1, '.'); + } + + + i = strchr(input, '-'); + if (i != NULL) { + int j; + for (j=i-input; j < len-1; j++) + input[j] = input[j+1]; + input[len-1] = '='; + + + i = strchr(input, '-'); + + if (i != NULL) { + for (j=i-input; j < len-2; j++) + input[j] = input[j+1]; + input[len-2] = '='; + } + + } + +} + + + + + +// int main () { +// char outbuf[200]; + +// char data[56] = "ba239023820389023802380389abc2322132321932847203aedfasd"; +// char data2[200]; +// int cookielen; + +// srand(time(NULL)); + +// bzero(data2, sizeof(data2)); +// bzero(outbuf, sizeof(outbuf)); + +// base64::encoder E; +// E.encode(data, strlen(data), data2); +// E.encode_end(data2+strlen(data2)); +// printf("%s", data2); +// // remove trailing newline +// data2[strlen(data2) - 1] = 0; + +// // /* substitute / with _, + with ., = with - that maybe inserted anywhere in the middle */ + +// sanitize_b64(data2, strlen(data2)); +// printf("%s\n", data2); + +// cookielen = gen_b64_cookie_field((char*) outbuf, (char*) data2, strlen(data2)); +// printf("cookie=%s\n", outbuf); + +// bzero(data2, sizeof(data2)); +// cookielen = unwrap_b64_cookie((char*) outbuf, (char*) data2, strlen(outbuf)); + + +// desanitize_b64(data2, cookielen); +// printf("%s\n", data2); +// printf("%d\n", cookielen); + +// data2[strlen(data2)] = '\n'; + + +// bzero(outbuf, 200); + + + +// base64::decoder D; +// D.decode(data2, strlen(data2), outbuf); +// printf("%s\n", outbuf); + + + +// } + + + + diff --git a/src/steg/b64cookies.h b/src/steg/b64cookies.h new file mode 100644 index 0000000..d0c6e6d --- /dev/null +++ b/src/steg/b64cookies.h @@ -0,0 +1,20 @@ +#ifndef _B64_COOKIES_H +#define _B64_COOKIES_H + + + +#include <stdio.h> +#include <strings.h> +#include <stdlib.h> +#include <string.h> +#include "b64encode.h" +#include "b64decode.h" + +int unwrap_b64_cookie( char* inbuf, char* outbuf, int buflen); +int gen_b64_cookie_field( char* outbuf, char* data, int datalen); +int gen_one_b64cookie( char* outbuf, int& cookielen, char* data, int datalen); +void sanitize_b64(char* input, int len); +void desanitize_b64(char* input, int len); + + +#endif diff --git a/src/steg/b64decode.cc b/src/steg/b64decode.cc new file mode 100755 index 0000000..3beffba --- /dev/null +++ b/src/steg/b64decode.cc @@ -0,0 +1,88 @@ +/* +cdecoder.c - c source to a base64 decoding algorithm implementation + +This is part of the libb64 project, and has been placed in the public domain. +For details, see http://sourceforge.net/projects/libb64 +*/ + +#include "b64decode.h" + +int base64_decode_value(char value_in) +{ + static const char decoding[] = {62,-1,-1,-1,63,52,53,54,55,56,57,58,59,60,61,-1,-1,-1,-2,-1,-1,-1,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,-1,-1,-1,-1,-1,-1,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51}; + static const char decoding_size = sizeof(decoding); + value_in -= 43; + if (value_in < 0 || value_in > decoding_size) return -1; + return decoding[(int)value_in]; +} + +void base64_init_decodestate(base64_decodestate* state_in) +{ + state_in->step = step_a; + state_in->plainchar = 0; +} + +int base64_decode_block(const char* code_in, const int length_in, char* plaintext_out, base64_decodestate* state_in) +{ + const char* codechar = code_in; + char* plainchar = plaintext_out; + char fragment; + + *plainchar = state_in->plainchar; + + switch (state_in->step) + { + while (1) + { + case step_a: + do { + if (codechar == code_in+length_in) + { + state_in->step = step_a; + state_in->plainchar = *plainchar; + return plainchar - plaintext_out; + } + fragment = (char)base64_decode_value(*codechar++); + } while (fragment < 0); + *plainchar = (fragment & 0x03f) << 2; + case step_b: + do { + if (codechar == code_in+length_in) + { + state_in->step = step_b; + state_in->plainchar = *plainchar; + return plainchar - plaintext_out; + } + fragment = (char)base64_decode_value(*codechar++); + } while (fragment < 0); + *plainchar++ |= (fragment & 0x030) >> 4; + *plainchar = (fragment & 0x00f) << 4; + case step_c: + do { + if (codechar == code_in+length_in) + { + state_in->step = step_c; + state_in->plainchar = *plainchar; + return plainchar - plaintext_out; + } + fragment = (char)base64_decode_value(*codechar++); + } while (fragment < 0); + *plainchar++ |= (fragment & 0x03c) >> 2; + *plainchar = (fragment & 0x003) << 6; + case step_d: + do { + if (codechar == code_in+length_in) + { + state_in->step = step_d; + state_in->plainchar = *plainchar; + return plainchar - plaintext_out; + } + fragment = (char)base64_decode_value(*codechar++); + } while (fragment < 0); + *plainchar++ |= (fragment & 0x03f); + } + } + /* control should not reach here */ + return plainchar - plaintext_out; +} + diff --git a/src/steg/b64decode.h b/src/steg/b64decode.h new file mode 100755 index 0000000..9514c87 --- /dev/null +++ b/src/steg/b64decode.h @@ -0,0 +1,66 @@ +/* +cdecode.h - c header for a base64 decoding algorithm + +This is part of the libb64 project, and has been placed in the public domain. +For details, see http://sourceforge.net/projects/libb64 +*/ + +#ifndef BASE64_CDECODE_H +#define BASE64_CDECODE_H + +#include <iostream> +#include "b64encode.h" + +typedef enum +{ + step_a, step_b, step_c, step_d +} base64_decodestep; + +typedef struct +{ + base64_decodestep step; + char plainchar; +} base64_decodestate; + +void base64_init_decodestate(base64_decodestate* state_in); + +int base64_decode_value(char value_in); + +int base64_decode_block(const char* code_in, const int length_in, char* plaintext_out, base64_decodestate* state_in); + + + + +namespace base64 +{ + + struct decoder + { + base64_decodestate _state; + int _buffersize; + + decoder(int buffersize_in = BUFFERSIZE) + : _buffersize(buffersize_in) + { + } + + int decode(char value_in) + { + + return base64_decode_value(value_in); + } + + int decode(const char* code_in, const int length_in, char* plaintext_out) + { + base64_init_decodestate(&_state); + return base64_decode_block(code_in, length_in, plaintext_out, &_state); + } + + }; + +} // namespace base64 + + + + +#endif /* BASE64_CDECODE_H */ diff --git a/src/steg/b64encode.cc b/src/steg/b64encode.cc new file mode 100755 index 0000000..1f1352d --- /dev/null +++ b/src/steg/b64encode.cc @@ -0,0 +1,139 @@ +/* +cencoder.c - c source to a base64 encoding algorithm implementation + +This is part of the libb64 project, and has been placed in the public domain. +For details, see http://sourceforge.net/projects/libb64 +*/ + + +#include "b64encode.h" +#include "b64decode.h" + + +const int CHARS_PER_LINE = 72; + +void base64_init_encodestate(base64_encodestate* state_in) +{ + state_in->step = step_A; + state_in->result = 0; + state_in->stepcount = 0; +} + +char base64_encode_value(char value_in) +{ + static const char* encoding = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + if (value_in > 63) return '='; + return encoding[(int)value_in]; +} + +int base64_encode_block(const char* plaintext_in, int length_in, char* code_out, base64_encodestate* state_in) +{ + const char* plainchar = plaintext_in; + const char* const plaintextend = plaintext_in + length_in; + char* codechar = code_out; + char result; + char fragment; + + result = state_in->result; + + switch (state_in->step) + { + while (1) + { + case step_A: + if (plainchar == plaintextend) + { + state_in->result = result; + state_in->step = step_A; + return codechar - code_out; + } + fragment = *plainchar++; + result = (fragment & 0x0fc) >> 2; + *codechar++ = base64_encode_value(result); + result = (fragment & 0x003) << 4; + case step_B: + if (plainchar == plaintextend) + { + state_in->result = result; + state_in->step = step_B; + return codechar - code_out; + } + fragment = *plainchar++; + result |= (fragment & 0x0f0) >> 4; + *codechar++ = base64_encode_value(result); + result = (fragment & 0x00f) << 2; + case step_C: + if (plainchar == plaintextend) + { + state_in->result = result; + state_in->step = step_C; + return codechar - code_out; + } + fragment = *plainchar++; + result |= (fragment & 0x0c0) >> 6; + *codechar++ = base64_encode_value(result); + result = (fragment & 0x03f) >> 0; + *codechar++ = base64_encode_value(result); + + ++(state_in->stepcount); + if (state_in->stepcount == CHARS_PER_LINE/4) + { + *codechar++ = '\n'; + state_in->stepcount = 0; + } + } + } + + + + + /* control should not reach here */ + return codechar - code_out; +} + +int base64_encode_blockend(char* code_out, base64_encodestate* state_in) +{ + char* codechar = code_out; + + switch (state_in->step) + { + case step_B: + *codechar++ = base64_encode_value(state_in->result); + *codechar++ = '='; + *codechar++ = '='; + break; + case step_C: + *codechar++ = base64_encode_value(state_in->result); + *codechar++ = '='; + break; + case step_A: + break; + } + *codechar++ = '\n'; + + return codechar - code_out; +} + + + + +/* +int main() { + + char foo[12] = "AbcdwefA#"; + char foo2[22]; + + base64::encoder E; + + printf("%d\n", (int) strlen(foo)); + E.encode(foo, strlen(foo), foo2); + printf("%s\n", foo2); + printf("%d\n", E.encode_end(foo2+strlen(foo2))); + printf("%s\n", foo2); + + base64::decoder D; + D.decode(foo2, strlen(foo2), foo); + printf("%s\n", foo); +} + +*/ diff --git a/src/steg/b64encode.h b/src/steg/b64encode.h new file mode 100755 index 0000000..be8c666 --- /dev/null +++ b/src/steg/b64encode.h @@ -0,0 +1,75 @@ +/* +cencode.h - c header for a base64 encoding algorithm + +This is part of the libb64 project, and has been placed in the public domain. +For details, see http://sourceforge.net/projects/libb64 +*/ + +#ifndef BASE64_CENCODE_H +#define BASE64_CENCODE_H +#include <iostream> + + +typedef enum +{ + step_A, step_B, step_C +} base64_encodestep; + +typedef struct +{ + base64_encodestep step; + char result; + int stepcount; +} base64_encodestate; + +void base64_init_encodestate(base64_encodestate* state_in); + +char base64_encode_value(char value_in); + +int base64_encode_block(const char* plaintext_in, int length_in, char* code_out, base64_encodestate* state_in); + +int base64_encode_blockend(char* code_out, base64_encodestate* state_in); + + + + + +static int BUFFERSIZE = 16777216; + +namespace base64 +{ + + struct encoder + { + base64_encodestate _state; + int _buffersize; + + encoder(int buffersize_in = BUFFERSIZE) + : _buffersize(buffersize_in) + {} + + int encode(char value_in) + { + + return base64_encode_value(value_in); + } + + int encode(const char* code_in, const int length_in, char* plaintext_out) + { + base64_init_encodestate(&_state); + + return base64_encode_block(code_in, length_in, plaintext_out, &_state); + } + + int encode_end(char* plaintext_out) + { + return base64_encode_blockend(plaintext_out, &_state); + } + + }; + +} // namespace base64 + + + +#endif /* BASE64_CENCODE_H */

1 0

[stegotorus/master] More benchmark tweakage
by zwol＠torproject.org 20 Jul '12

20 Jul '12

commit d65537f80ad1f2171a9bcca8531d17a9d3281097 Author: Zack Weinberg <zackw(a)panix.com> Date: Tue Feb 14 15:01:57 2012 -0800 More benchmark tweakage --- scripts/benchmark.py | 50 +++++++++++++++++--------------------------------- scripts/bm-mcurl.c | 4 ++-- 2 files changed, 19 insertions(+), 35 deletions(-) diff --git a/scripts/benchmark.py b/scripts/benchmark.py index c6487b6..f4f8830 100755 --- a/scripts/benchmark.py +++ b/scripts/benchmark.py @@ -20,8 +20,6 @@ # # bwm-ng: http://www.gropp.org/?id=projects&sub=bwm-ng # curl: http://curl.haxx.se/ -# httperf: http://www.hpl.hp.com/research/linux/httperf/ -# tsocks: http://tsocks.sourceforge.net/about.php # tor: https://torproject.org/ # stegotorus: you already have it :) # @@ -61,10 +59,9 @@ FUDGE_FIXEDRATE = 2.5 C_bwm = "bwm-ng" C_curl = "curl" -C_httperf = "httperf" +C_mcurl = "bm-mcurl" C_storus = "stegotorus-wrapper" C_tor = "/usr/sbin/tor" -C_tsocks = "/usr/lib/libtsocks.so" P_nylon = "nylon" P_storus = "stegotorus-wrapper" @@ -97,9 +94,10 @@ def monitor(report, label, period): (stamp, iface, upbytes, dnbytes, rest) = line.split(';', 4) if iface == 'total': continue - # convert to most compact possible form - upbytes = str(float(upbytes)) - dnbytes = str(float(dnbytes)) + # convert to most compact possible form, + # scale to decimal kilobytes + upbytes = str(float(upbytes)/1000) + dnbytes = str(float(dnbytes)/1000) report.write("%s,%d,%s,%s\n" % (label,n,upbytes,dnbytes)) n += 1 @@ -279,24 +277,10 @@ def c_curl(url, proxyhost): proxyhost + ":" + PROXY_PORT, url, "-o", "/dev/null")) -def c_httperf(prefix, rate, proxyhost): - fp = open("tsocks.conf", "w") - fp.write("""\ -server = %s -local = %s/255.255.255.255 -server_port = %s -server_type = 5 -""" % (proxyhost, proxyhost, PROXY_PORT)) - fp.close() - return ClientProcess((C_httperf, "--hog", - "--server=" + TARGET, - "--uri=" + prefix, - "--period=" + str(rate), - "--num-calls=5", "--num-conns=2000", - "--wset=10000,1"), - { 'LD_PRELOAD' : C_tsocks, - 'TSOCKS_CONF_FILE' : - os.path.join(os.getcwd(), "tsocks.conf") }) +def c_mcurl(prefix, cps, proxyhost): + return ClientProcess((C_mcurl, str(cps), '200', proxyhost, + 'http://' + TARGET + '/' + prefix + + '/[0-9]/[0-9]/[0-9]/[0-9].html')) # Benchmarks. @@ -306,13 +290,13 @@ def bench_fixedrate_direct(report): try: proxy = p_nylon() - for cap in range(10, 810, 10): + for cap in range(10,810,10): sys.stderr.write("fixedrate,direct,%d\n" % (cap * 1000)) try: client = c_curl('http://' + TARGET + '/bm-fixedrate.cgi/' + str(int(cap * 1000 * FUDGE_FIXEDRATE)), PROXY) - monitor(report, "fixedrate,direct,%d" % (cap * 1000), 60) + monitor(report, "fixedrate,direct,%d" % cap, 60) finally: if client is not None: client.terminate() @@ -333,12 +317,12 @@ def bench_fixedrate_tor(report): time.sleep(5) # tor startup is slow for cap in range(10,810,10): - sys.stderr.write("fixedrate,tor,%d\n" % (cap * 1000)) + sys.stderr.write("fixedrate,tor,%d\n" % cap) try: client = c_curl('http://' + TARGET + '/bm-fixedrate.cgi/' + str(int(cap * 1000 * FUDGE_FIXEDRATE)), '127.0.0.1') - monitor(report, "fixedrate,tor,%d" % (cap * 1000), 60) + monitor(report, "fixedrate,tor,%d" % cap, 60) finally: if client is not None: client.terminate() @@ -358,10 +342,10 @@ def bench_files_direct(report, prefix): try: proxy = p_nylon() - for cps in range(1,81): + for cps in range(1,151): sys.stderr.write("files.%s,direct,%d\n" % (prefix, cps)) try: - client = c_httperf(prefix, 1./cps, PROXY_IP) + client = c_mcurl(prefix, cps, PROXY_IP) monitor(report, "files.%s,direct,%d" % (prefix, cps), 60) finally: if client is not None: @@ -382,10 +366,10 @@ def bench_files_tor(report, prefix): proxyl = c_tor_direct() time.sleep(5) # tor startup is slow - for cps in range(1,81): + for cps in range(1,151): sys.stderr.write("files.%s,tor,%d\n" % (prefix, cps)) try: - client = c_httperf(prefix, 1./cps, '127.0.0.1') + client = c_mcurl(prefix, cps, '127.0.0.1') monitor(report, "files.%s,tor,%d" % (prefix, cps), 60) finally: if client is not None: diff --git a/scripts/bm-mcurl.c b/scripts/bm-mcurl.c index 890b9e4..7fb03a0 100644 --- a/scripts/bm-mcurl.c +++ b/scripts/bm-mcurl.c @@ -162,9 +162,9 @@ process_urls(struct url_iter *it, CURLM *multi, CURL **handles, last = now; url = url_next(it); if (url) { - if (curl_easy_setopt(handles[maxh], CURLOPT_URL, url) != CURLM_OK) + if (curl_easy_setopt(handles[maxh], CURLOPT_URL, url)) abort(); - if (curl_multi_add_handle(multi, handles[maxh]) != CURLM_OK) + if (curl_multi_add_handle(multi, handles[maxh])) abort(); maxh++; free(url); /* curl takes a copy */

1 0

[stegotorus/master] adding changes to Makefiles.am and http.cc
by zwol＠torproject.org 20 Jul '12

20 Jul '12

commit 7539220546986e3a5e35498ba6980f429da1427f Author: Vinod Yegneswaran <vinod(a)safdef.isc.org> Date: Tue Feb 14 15:30:47 2012 -0800 adding changes to Makefiles.am and http.cc --- Makefile.am | 3 + src/steg/http.cc | 245 ++++++++++++++++++++++++++++------------------------- 2 files changed, 132 insertions(+), 116 deletions(-) diff --git a/Makefile.am b/Makefile.am index 9affaed..d81d072 100644 --- a/Makefile.am +++ b/Makefile.am @@ -19,6 +19,9 @@ PROTOCOLS = \ src/protocol/null.cc STEGANOGRAPHERS = \ + src/steg/b64cookies.cc \ + src/steg/b64encode.cc \ + src/steg/b64decode.cc \ src/steg/cookies.cc \ src/steg/crc32.cc \ src/steg/embed.cc \ diff --git a/src/steg/http.cc b/src/steg/http.cc index 3402ba7..3251995 100644 --- a/src/steg/http.cc +++ b/src/steg/http.cc @@ -40,6 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "swfSteg.h" #include "pdfSteg.h" #include "jsSteg.h" +#include "b64cookies.h" #include <event2/buffer.h> #include <stdio.h> @@ -254,116 +255,135 @@ http_client_cookie_transmit (http *s, struct evbuffer *source, conn_t *conn) { /* On the client side, we have to embed the data in a GET query somehow; the only plausible places to put it are the URL and cookies. This presently uses the URL. And it can't be binary. */ - // struct evbuffer *scratch; - struct evbuffer_iovec *iv; - int i, nv; - struct evbuffer *dest = conn_get_outbound(conn); - size_t sbuflen = evbuffer_get_length(source); - char buf[10000]; - unsigned char data[(int) sbuflen*2]; - // unsigned char outbuf[MAX_COOKIE_SIZE]; - unsigned char outbuf[(int) sbuflen*8]; - int datalen; - - - // size_t sofar = 0; - size_t cookie_len; - - - /* Convert all the data in 'source' to hexadecimal and write it to - 'scratch'. Data is padded to a multiple of four characters with - equals signs. */ + struct evbuffer *dest = conn_get_outbound(conn); + size_t sbuflen = evbuffer_get_length(source); + int bufsize = 10000; + char* buf = (char*) xmalloc(bufsize); - unsigned int len = 0; - unsigned int cnt = 0; + char* data; + char* data2 = (char*) xmalloc (sbuflen*4); + char* cookiebuf = (char*) xmalloc (sbuflen*8); + int payload_len = 0; + int cnt = 0; + int cookie_len = 0; + int rval; + int len = 0; + base64::encoder E; - datalen = 0; - cookie_len = 4 * sbuflen + rand() % 4; + data = (char*) evbuffer_pullup(source, sbuflen); + if (data == NULL) { + log_debug("evbuffer_pullup failed"); + goto err; + } - nv = evbuffer_peek(source, sbuflen, NULL, NULL, 0); - iv = (evbuffer_iovec*)xzalloc(sizeof(struct evbuffer_iovec) * nv); - if (evbuffer_peek(source, sbuflen, NULL, iv, nv) != nv) { - free(iv); - return -1; - } // retry up to 10 times - while (!len) { - len = find_client_payload(buf, sizeof(buf), TYPE_HTTP_REQUEST); - if (cnt++ == 10) return -1; + while (!payload_len) { + payload_len = find_client_payload(buf, bufsize, TYPE_HTTP_REQUEST); + if (cnt++ == 10) { + goto err; + } } - + buf[payload_len] = 0; if (has_peer_name == 0 && lookup_peer_name_from_ip((char*) conn->peername, peername)) has_peer_name = 1; - // if (find_uri_type(buf) != HTTP_CONTENT_SWF) { - // fprintf(stderr, "%s\n", buf); - // exit(-1); - // } + bzero(data2, sbuflen*4); + E.encode((char*) data, sbuflen, (char*) data2); + E.encode_end(data2+strlen((char*) data2)); + len = (int) strlen(data2) - 1; + // remove trailing newline + data2[len] = 0; + + // substitute / with _, + with ., = with - that maybe inserted anywhere in the middle + sanitize_b64(data2, len); - cnt = 0; + + cookie_len = gen_b64_cookie_field(cookiebuf, data2, len); + cookiebuf[cookie_len] = 0; - for (i = 0; i < nv; i++) { - const unsigned char *p = (const unsigned char *)iv[i].iov_base; - const unsigned char *limit = p + iv[i].iov_len; - char c; - while (p < limit && cnt < sbuflen) { - c = *p++; - data[datalen] = "0123456789abcdef"[(c & 0xF0) >> 4]; - data[datalen+1] = "0123456789abcdef"[(c & 0x0F) >> 0]; - datalen += 2; - cnt++; - } + + + if (cookie_len < 0) { + log_debug("cookie generation failed\n"); + return -1; } + - free(iv); + // add uri field + rval = evbuffer_add(dest, buf, strstr(buf, "\r\n") - buf + 2); + if (rval) { + log_warn("error adding uri field\n"); + goto err; + } - if (cookie_len < 4) cookie_len = 4; + rval = evbuffer_add(dest, "Host: ", 6); + if (rval) { + log_warn("error adding host field\n"); + goto err; + } - datalen = gen_cookie_field(outbuf, cookie_len, data, datalen); - log_debug("CLIENT: sending cookie of length = %d %d\n", datalen, (int) cookie_len); - // fprintf(stderr, "CLIENT: sending cookie of length = %d %d\n", datalen, (int) cookie_len); + rval = evbuffer_add(dest, peername, strlen(peername)); + if (rval) { + log_warn("error adding peername field\n"); + goto err; + } - if (datalen < 0) { - log_debug("cookie generation failed\n"); - return -1; + + rval = evbuffer_add(dest, strstr(buf, "\r\n"), payload_len - (unsigned int) (strstr(buf, "\r\n") - buf)); + if (rval) { + log_warn("error adding HTTP fields\n"); + goto err; } + rval = evbuffer_add(dest, "Cookie: ", 8); + if (rval) { + log_warn("error adding cookie fields\n"); + goto err; + } + rval = evbuffer_add(dest, cookiebuf, cookie_len); - if (evbuffer_add(dest, buf, strstr(buf, "\r\n") - buf + 2) || // add uri field - evbuffer_add(dest, "Host: ", 6) || - evbuffer_add(dest, peername, strlen(peername)) || - evbuffer_add(dest, strstr(buf, "\r\n"), len - (unsigned int) (strstr(buf, "\r\n") - buf)) || // add everything but first line - evbuffer_add(dest, "Cookie: ", 8) || - evbuffer_add(dest, outbuf, cookie_len) || - evbuffer_add(dest, "\r\n\r\n", 4)) { - log_debug("error ***********************"); - return -1; - } + if (rval) { + log_warn("error adding cookie buf\n"); + goto err; + } - // debug - // log_warn("CLIENT HTTP request header:"); - // buf_dump((unsigned char*)buf, len, stderr); - // sofar += datalen/2; - evbuffer_drain(source, datalen/2); + rval = evbuffer_add(dest, "\r\n\r\n", 4); + + if (rval) { + log_warn("error adding terminators \n"); + goto err; + } - log_debug("CLIENT TRANSMITTED payload %d\n", (int) sbuflen); + + evbuffer_drain(source, sbuflen); + log_debug("CLIENT TRANSMITTED payload %d\n", (int) sbuflen); conn_cease_transmission(conn); - s->type = find_uri_type(buf, sizeof(buf)); + s->type = find_uri_type(buf, bufsize); s->have_transmitted = true; + + + free(buf); + free(data2); return 0; + +err: + free(buf); + free(data2); + return -1; + } @@ -547,11 +567,13 @@ http::transmit(struct evbuffer *source, conn_t *conn) if (is_clientside) { /* On the client side, we have to embed the data in a GET query somehow; - the only plausible places to put it are the URL and cookies. This - presently uses the URL. And it can't be binary. */ + the only plausible places to put it are the URL and cookies. */ - if (evbuffer_get_length(source) < 72) - return http_client_uri_transmit(this, source, conn); //@@ + /* if (evbuffer_get_length(source) < 72) + return http_client_uri_transmit(this, source, conn); + */ + + //@@ return http_client_cookie_transmit(this, source, conn); //@@ } else { @@ -588,15 +610,16 @@ http::transmit(struct evbuffer *source, conn_t *conn) int http_server_receive(http *s, conn_t *conn, struct evbuffer *dest, struct evbuffer* source) { - int cnt = 0; - unsigned char* data; + char* data; int type; do { struct evbuffer_ptr s2 = evbuffer_search(source, "\r\n\r\n", sizeof ("\r\n\r\n") -1 , NULL); - unsigned char *p; - unsigned char c, h, secondhalf; + char *p; + char *pend; + char outbuf[MAX_COOKIE_SIZE]; + char outbuf2[MAX_COOKIE_SIZE]; int sofar = 0; int cookie_mode = 0; @@ -609,7 +632,7 @@ http_server_receive(http *s, conn_t *conn, struct evbuffer *dest, struct evbuffe log_debug("SERVER received request header of length %d", (int)s2.pos); - data = evbuffer_pullup(source, s2.pos+4); + data = (char*) evbuffer_pullup(source, s2.pos+4); if (data == NULL) { log_debug("SERVER evbuffer_pullup fails"); @@ -622,54 +645,45 @@ http_server_receive(http *s, conn_t *conn, struct evbuffer *dest, struct evbuffe type = find_uri_type((char *)data, s2.pos+4); if (strstr((char*) data, "Cookie") != NULL) { - p = (unsigned char*) strstr((char*) data, "Cookie:") + sizeof "Cookie: "-1; + p = strstr((char*) data, "Cookie:") + sizeof "Cookie: "-1; cookie_mode = 1; } else p = data + sizeof "GET /" -1; + pend = strstr(p, "\r\n"); - secondhalf = 0; - c = 0; + if (pend == NULL || (pend - p > MAX_COOKIE_SIZE)) { + fprintf(stderr, "incorrect cookie recovery \n"); + exit(-1); + + } - while (strncmp((char*) p, "\r\n", 2) != 0 && (cookie_mode != 0 || p[0] != '.') && sofar < MAX_COOKIE_SIZE) { - if (!secondhalf) - c = 0; - if ('0' <= *p && *p <= '9') - h = *p - '0'; - else if ('a' <= *p && *p <= 'f') - h = *p - 'a' + 10; - else { - p++; - continue; - } - c = (c << 4) + h; - if (secondhalf) { - outbuf[sofar++] = c; - cnt++; - } - secondhalf = !secondhalf; - p++; - } + bzero(outbuf, sizeof(outbuf)); + int cookielen = unwrap_b64_cookie((char*) p, (char*) outbuf, pend - p); - if (sofar >= MAX_COOKIE_SIZE) { - fprintf(stderr, "cookie buffer overflow\n"); - exit(-1); - } + desanitize_b64(outbuf, cookielen); + outbuf[cookielen] = '\n'; + bzero(outbuf2, sizeof(outbuf2)); - outbuf[sofar] = 0; + base64::decoder D; + sofar = D.decode(outbuf, cookielen+1, outbuf2); - if (secondhalf) { - fprintf(stderr, "incorrect cookie or uri recovery \n"); - exit(-1); - } + if (sofar <= 0) + log_warn("decode failed\n"); + + + if (sofar >= MAX_COOKIE_SIZE) { + log_warn("cookie buffer overflow????\n"); + exit(-1); + } - if (evbuffer_add(dest, outbuf, sofar)) { + if (evbuffer_add(dest, outbuf2, sofar)) { log_debug("Failed to transfer buffer"); return RECV_BAD; } @@ -679,7 +693,6 @@ http_server_receive(http *s, conn_t *conn, struct evbuffer *dest, struct evbuffe s->have_received = 1; s->type = type; - // fprintf(stderr, "SERVER RECEIVED payload %d %d\n", cnt, type); conn_transmit_soon(conn, 100); return RECV_GOOD;

1 0