commit 5fdb018fe760def96eca098bee723dea4b5e9049 Author: Mike Perry <mikeperry-git@fscked.org> Date: Mon Nov 7 21:16:07 2011 -0800 Bugs 4431+2014: Randomized url support + variable file sizes. Scanner child piece. --- NetworkScanners/BwAuthority/bwauthority_child.py | 35 ++++++++++++--------- 1 files changed, 20 insertions(+), 15 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority_child.py b/NetworkScanners/BwAuthority/bwauthority_child.py index 27dc0ba..1527a11 100755 --- a/NetworkScanners/BwAuthority/bwauthority_child.py +++ b/NetworkScanners/BwAuthority/bwauthority_child.py @@ -23,6 +23,7 @@ import ConfigParser import sqlalchemy import sets import re +import random sys.path.append("../../") @@ -46,18 +47,7 @@ user_agent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.0.37 # Note these urls should be https due to caching considerations. # If you really must make them http, be sure to change exit_ports to [80] # below, or else the scan will not finish. -# TODO: As the network balances, these can become more uniform in size -# TODO: We'll also want to try to prefer pairing unmeasured nodes -# together then, and use a different url set for them. -# cutoff percent URL -urls = [(5, "https://38.229.70.2/8M"), # fbw 2000k..451k - (10, "https://38.229.70.2/4M"), # fbw 451k..275k - (20, "https://38.229.70.2/2M"), # fbw 275k..150k - (30, "https://38.229.70.2/1M"), # fbw 150k..100k - (40, "https://38.229.70.2/1M"), # fbw 100k..75k - (50, "https://38.229.70.2/512k"), # fbw 75k..50k - (80, "https://38.229.70.2/256k"), # fbw 50k..25k - (100, "https://38.229.70.2/128k")] # fbw 25k..10k +urls = ["https://38.229.70.2/"] #, "https://38.229.70.19/"] # Do NOT modify this object directly after it is handed to PathBuilder @@ -109,10 +99,25 @@ def read_config(filename): sleep_start,sleep_stop,min_streams,pid_file,db_url) def choose_url(percentile): - for (pct, url) in urls: + # TODO: Maybe we don't want to read the file *every* time? + # Maybe once per slice? + # Read in the bw auths file + f = file("./data/bwfiles", "r") + lines = [] + valid = False + for l in f.readlines(): + if l == ".\n": + valid = True + break + pair = l.split() + lines.append((int(pair[0]), pair[1])) + + if not valid: + plog("ERROR", "File size list is invalid!") + + for (pct, fname) in lines: if percentile < pct: - return url - #return "https://86.59.21.36/torbrowser/dist/tor-im-browser-1.2.0_ru_split/tor-im-bro..." + return random.choice(urls) + fname raise PathSupport.NoNodesRemain("No nodes left for url choice!") def http_request(address):