[tor-commits] [flashproxy/master] Use a small custom HTTP downloader rather than Wget.

[dcf at torproject.org]

commit 7740d4fdb9626c8b47e99e30e60839c7f9abb90d
Author: David Fifield <david at bamsoftware.com>
Date:   Sun Oct 30 16:57:01 2011 -0700

    Use a small custom HTTP downloader rather than Wget.
    
    I had been timing downloads with "time wget" but that counts the time
    taken to establish (and possibly retry) the connection. In my graphs
    there was a clear effect where the first 20 connections would take x
    seconds to download, but the 21st and later would take x + 5 or so.
---
 experiments/throughput/httpget.py    |   35 ++++++++++++++++++++++++++++++++++
 experiments/throughput/throughput.sh |   17 +--------------
 2 files changed, 37 insertions(+), 15 deletions(-)

diff --git a/experiments/throughput/httpget.py b/experiments/throughput/httpget.py
new file mode 100755
index 0000000..a08412b
--- /dev/null
+++ b/experiments/throughput/httpget.py
@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+
+# A simple HTTP downloader that discards what it downloads and prints the time
+# taken to download. We use this rather than "time wget" because the latter
+# includes time taken to establish (and possibly retry) the connection.
+
+import getopt
+import sys
+import time
+import urllib2
+
+BLOCK_SIZE = 65536
+
+label = None
+
+opts, args = getopt.gnu_getopt(sys.argv[1:], "l:")
+for o, a in opts:
+    if o == "-l":
+        label = a
+
+try:
+    stream = urllib2.urlopen(args[0], timeout=10)
+    start_time = time.time()
+    while stream.read(BLOCK_SIZE):
+        pass
+    end_time = time.time()
+    if label:
+        print "%s %.3f" % (label, end_time - start_time)
+    else:
+        print "%.3f" % (end_time - start_time)
+except:
+    if label:
+        print "%s error" % label
+    else:
+        print "error"
diff --git a/experiments/throughput/throughput.sh b/experiments/throughput/throughput.sh
index 5e2fd4b..7dcbad2 100755
--- a/experiments/throughput/throughput.sh
+++ b/experiments/throughput/throughput.sh
@@ -77,26 +77,13 @@ PIDS_TO_KILL+=($!)
 visible_sleep 1
 
 
-# Extract and parse the "real" part of "time" output and return as a count of
-# seconds.
-extract_real_time() {
-	perl -n -e '($m, $s) = ($_ =~ m/real\s+(\d+)m([\d\.]+)s/); if (defined($m)) { print $m*60 + $s . "\n"; }'
-}
-
 > "$RESULTS_FILE_NAME"
 
 declare -a WAIT_PIDS
 i=0
 while [ $i -lt $NUM_CLIENTS ]; do
-	echo "Start Wget $((i + 1))."
-	(
-		times=$((time wget http://localhost:2000/dump -q --timeout 30 -t 1 -O /dev/null) 2>&1)
-		if [ $? -eq 0 ]; then
-			echo "$times" | extract_real_time >> "$RESULTS_FILE_NAME"
-		else
-			echo "error" >> "$RESULTS_FILE_NAME"
-		fi
-	) &
+	echo "Start downloader $((i + 1))."
+	./httpget.py http://localhost:2000/dump >> "$RESULTS_FILE_NAME" &
 	WAIT_PIDS+=($!)
 	i=$((i + 1))
 done