commit 7022ff30fc657e53be8ca1726ae73259174a322b Author: juga0 <juga@riseup.net> Date: Wed Dec 12 15:21:47 2018 +0000 scanner: refactor, move HTTP headers to constants --- sbws/core/scanner.py | 7 ++++--- sbws/globals.py | 5 +++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index 51e7076..f6e226a 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -9,7 +9,7 @@ from ..lib.relayprioritizer import RelayPrioritizer from ..lib.destination import DestinationList from ..util.timestamp import now_isodt_str from ..util.state import State -from sbws.globals import fail_hard +from sbws.globals import fail_hard, HTTP_GET_HEADERS import sbws.util.stem as stem_utils import sbws.util.requests as requests_utils from argparse import ArgumentDefaultsHelpFormatter @@ -31,14 +31,15 @@ def timed_recv_from_server(session, dest, byte_range): ''' Request the **byte_range** from the URL at **dest**. If successful, return True and the time it took to download. Otherwise return False and an exception. ''' - headers = {'Range': byte_range, 'Accept-Encoding': 'identity'} + start_time = time.time() + HTTP_GET_HEADERS['Range'] = byte_range # TODO: # - What other exceptions can this throw? # - Do we have to read the content, or did requests already do so? try: # headers are merged with the session ones, not overwritten. - session.get(dest.url, headers=headers, verify=dest.verify) + session.get(dest.url, headers=HTTP_GET_HEADERS, verify=dest.verify) except requests.exceptions.ConnectionError as e: return False, e except requests.exceptions.ReadTimeout as e: diff --git a/sbws/globals.py b/sbws/globals.py index 2277850..7196736 100644 --- a/sbws/globals.py +++ b/sbws/globals.py @@ -53,6 +53,11 @@ MAX_BW_DIFF_PERC = 50 BW_LINE_SIZE = 510 +HTTP_GET_HEADERS = { + 'Range': '{}', + 'Accept-Encoding': 'identity', +} + def fail_hard(*a, **kw): ''' Log something ... and then exit as fast as possible '''