commit 096c2bb23d79033be79503754e180145879d3449
Author: Ana C. Custura <ana(a)netstat.org.uk>
Date: Wed Jun 5 15:07:10 2019 +0100
Allows source to be given as a parameter on the commandline
---
onionperf/onionperf | 9 ++++++++-
onionperf/reprocessing.py | 42 ++++++++++++++++++++++++------------------
2 files changed, 32 insertions(+), 19 deletions(-)
diff --git a/onionperf/onionperf b/onionperf/onionperf
index fab805b..45ead3c 100755
--- a/onionperf/onionperf
+++ b/onionperf/onionperf
@@ -385,6 +385,13 @@ files generated by this script will be written""",
action="store", dest="date_filter",
default=None)
+ reprocess_parser.add_argument('-n', '--nickname',
+ help="""a nickname STRING that identifies the machine where the input logfiles were produced""",
+ metavar="STRING", type=str,
+ action="store", dest="nickname",
+ default=None)
+
+
# get args and call the command handler for the chosen mode
args = main_parser.parse_args()
args.func(args)
@@ -495,7 +502,7 @@ def reprocess(args):
torctl_logs = reprocessing.collect_logs(args.torctl_dirpath, '*torctl.log')
log_pairs = reprocessing.match(tgen_logs, torctl_logs, args.date_filter)
logging.info("Found {0} matching log pairs to be reprocessed".format(len(log_pairs)))
- reprocessing.multiprocess_logs(log_pairs, args.prefix)
+ reprocessing.multiprocess_logs(log_pairs, args.prefix, args.nickname)
def type_nonnegative_integer(value):
i = int(value)
diff --git a/onionperf/reprocessing.py b/onionperf/reprocessing.py
index 25111ba..66b82c7 100644
--- a/onionperf/reprocessing.py
+++ b/onionperf/reprocessing.py
@@ -7,6 +7,8 @@ import fnmatch
import logging
import os
import re
+import sys
+
def collect_logs(dirpath, pattern):
logs = []
@@ -14,7 +16,7 @@ def collect_logs(dirpath, pattern):
for filename in fnmatch.filter(filenames, pattern):
logs.append(os.path.join(root, filename))
return logs
-
+
def match(tgen_logs, tor_logs, date_filter):
log_pairs = []
@@ -24,41 +26,47 @@ def match(tgen_logs, tor_logs, date_filter):
date = m.group(0)
fdate = datetime.datetime.strptime(date, "%Y-%m-%d")
found = False
- if date_filter is None or util.do_dates_match(date_filter,fdate):
+ if date_filter is None or util.do_dates_match(date_filter, fdate):
for tor_log in tor_logs:
if date in tor_log:
log_pairs.append((tgen_log, tor_log, fdate))
found = True
- break
+ break
if not found:
- logging.warning('Skipping file {0}, could not find a match for it'.format(tgen_log))
-
+ logging.warning(
+ 'Skipping file {0}, could not find a match for it'.
+ format(tgen_log))
+
else:
- logging.warning('Filename {0} does not contain a date'.format(tgen_log))
+ logging.warning(
+ 'Filename {0} does not contain a date'.format(tgen_log))
if not log_pairs:
- logging.warning('Could not find any log matches. No analyses will be performed')
+ logging.warning(
+ 'Could not find any log matches. No analyses will be performed')
return log_pairs
-
-def analyze_func(prefix, pair):
- analysis = Analysis()
+
+def analyze_func(prefix, nick, pair):
+ analysis = Analysis(nickname=nick)
logging.info('Analysing pair for date {0}'.format(pair[2]))
analysis.add_tgen_file(pair[0])
analysis.add_torctl_file(pair[1])
analysis.analyze(do_simple=False, date_filter=pair[2])
analysis.save(output_prefix=prefix)
- analysis.export_torperf_version_1_1(output_prefix=prefix, do_compress=False)
+ analysis.export_torperf_version_1_1(
+ output_prefix=prefix, do_compress=False)
return 1
-
-
-def multiprocess_logs(log_pairs, prefix):
+
+
+def multiprocess_logs(log_pairs, prefix, nick):
pool = Pool(cpu_count())
analyses = None
try:
- func = partial(analyze_func, prefix)
+ func = partial(analyze_func, prefix, nick)
mr = pool.map_async(func, log_pairs)
pool.close()
- while not mr.ready(): mr.wait(1)
+ while not mr.ready():
+ mr.wait(1)
except KeyboardInterrupt:
logging.info("interrupted, terminating process pool")
pool.terminate()
@@ -66,5 +74,3 @@ def multiprocess_logs(log_pairs, prefix):
sys.exit()
except Exception as e:
logging.error(e)
-
-