[tor-commits] [torflow/master] Bugs 4431+2014: Randomized url support + variable file sizes.

[mikeperry at torproject.org]

commit 5fdb018fe760def96eca098bee723dea4b5e9049
Author: Mike Perry <mikeperry-git at fscked.org>
Date:   Mon Nov 7 21:16:07 2011 -0800

    Bugs 4431+2014: Randomized url support + variable file sizes.
    
    Scanner child piece.
---
 NetworkScanners/BwAuthority/bwauthority_child.py |   35 ++++++++++++---------
 1 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/NetworkScanners/BwAuthority/bwauthority_child.py b/NetworkScanners/BwAuthority/bwauthority_child.py
index 27dc0ba..1527a11 100755
--- a/NetworkScanners/BwAuthority/bwauthority_child.py
+++ b/NetworkScanners/BwAuthority/bwauthority_child.py
@@ -23,6 +23,7 @@ import ConfigParser
 import sqlalchemy
 import sets
 import re
+import random
 
 sys.path.append("../../")
 
@@ -46,18 +47,7 @@ user_agent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.0.37
 # Note these urls should be https due to caching considerations.
 # If you really must make them http, be sure to change exit_ports to [80]
 # below, or else the scan will not finish.
-# TODO: As the network balances, these can become more uniform in size
-# TODO: We'll also want to try to prefer pairing unmeasured nodes 
-# together then, and use a different url set for them.
-#          cutoff percent                URL
-urls =         [(5,          "https://38.229.70.2/8M"),     # fbw 2000k..451k
-                (10,          "https://38.229.70.2/4M"),    # fbw 451k..275k
-                (20,          "https://38.229.70.2/2M"),    # fbw 275k..150k
-                (30,          "https://38.229.70.2/1M"),    # fbw 150k..100k
-                (40,          "https://38.229.70.2/1M"),    # fbw 100k..75k
-                (50,          "https://38.229.70.2/512k"),  # fbw 75k..50k
-                (80,          "https://38.229.70.2/256k"),  # fbw 50k..25k
-                (100,         "https://38.229.70.2/128k")]  # fbw 25k..10k
+urls =         ["https://38.229.70.2/"] #, "https://38.229.70.19/"]
 
 
 # Do NOT modify this object directly after it is handed to PathBuilder
@@ -109,10 +99,25 @@ def read_config(filename):
             sleep_start,sleep_stop,min_streams,pid_file,db_url)
 
 def choose_url(percentile):
-  for (pct, url) in urls:
+  # TODO: Maybe we don't want to read the file *every* time?
+  # Maybe once per slice?
+  # Read in the bw auths file
+  f = file("./data/bwfiles", "r")
+  lines = []
+  valid = False
+  for l in f.readlines():
+    if l == ".\n":
+      valid = True
+      break
+    pair = l.split()
+    lines.append((int(pair[0]), pair[1]))
+
+  if not valid:
+    plog("ERROR", "File size list is invalid!")
+
+  for (pct, fname) in lines:
     if percentile < pct:
-      return url
-      #return "https://86.59.21.36/torbrowser/dist/tor-im-browser-1.2.0_ru_split/tor-im-browser-1.2.0_ru_split.part01.exe"
+      return random.choice(urls) + fname
   raise PathSupport.NoNodesRemain("No nodes left for url choice!")
 
 def http_request(address):