commit 7740d4fdb9626c8b47e99e30e60839c7f9abb90d Author: David Fifield david@bamsoftware.com Date: Sun Oct 30 16:57:01 2011 -0700

Use a small custom HTTP downloader rather than Wget.

I had been timing downloads with "time wget" but that counts the time taken to establish (and possibly retry) the connection. In my graphs there was a clear effect where the first 20 connections would take x seconds to download, but the 21st and later would take x + 5 or so. --- experiments/throughput/httpget.py | 35 ++++++++++++++++++++++++++++++++++ experiments/throughput/throughput.sh | 17 +-------------- 2 files changed, 37 insertions(+), 15 deletions(-)

diff --git a/experiments/throughput/httpget.py b/experiments/throughput/httpget.py new file mode 100755 index 0000000..a08412b --- /dev/null +++ b/experiments/throughput/httpget.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python + +# A simple HTTP downloader that discards what it downloads and prints the time +# taken to download. We use this rather than "time wget" because the latter +# includes time taken to establish (and possibly retry) the connection. + +import getopt +import sys +import time +import urllib2 + +BLOCK_SIZE = 65536 + +label = None + +opts, args = getopt.gnu_getopt(sys.argv[1:], "l:") +for o, a in opts: + if o == "-l": + label = a + +try: + stream = urllib2.urlopen(args[0], timeout=10) + start_time = time.time() + while stream.read(BLOCK_SIZE): + pass + end_time = time.time() + if label: + print "%s %.3f" % (label, end_time - start_time) + else: + print "%.3f" % (end_time - start_time) +except: + if label: + print "%s error" % label + else: + print "error" diff --git a/experiments/throughput/throughput.sh b/experiments/throughput/throughput.sh index 5e2fd4b..7dcbad2 100755 --- a/experiments/throughput/throughput.sh +++ b/experiments/throughput/throughput.sh @@ -77,26 +77,13 @@ PIDS_TO_KILL+=($!) visible_sleep 1

-# Extract and parse the "real" part of "time" output and return as a count of -# seconds. -extract_real_time() { - perl -n -e '($m, $s) = ($_ =~ m/real\s+(\d+)m([\d.]+)s/); if (defined($m)) { print $m*60 + $s . "\n"; }' -} -

"$RESULTS_FILE_NAME"

declare -a WAIT_PIDS i=0 while [ $i -lt $NUM_CLIENTS ]; do - echo "Start Wget $((i + 1))." - ( - times=$((time wget http://localhost:2000/dump -q --timeout 30 -t 1 -O /dev/null) 2>&1) - if [ $? -eq 0 ]; then - echo "$times" | extract_real_time >> "$RESULTS_FILE_NAME" - else - echo "error" >> "$RESULTS_FILE_NAME" - fi - ) & + echo "Start downloader $((i + 1))." + ./httpget.py http://localhost:2000/dump >> "$RESULTS_FILE_NAME" & WAIT_PIDS+=($!) i=$((i + 1)) done