[tor-commits] [onionperf/master] Allows source to be given as a parameter on the commandline

[irl at torproject.org]

commit 096c2bb23d79033be79503754e180145879d3449
Author: Ana C. Custura <ana at netstat.org.uk>
Date:   Wed Jun 5 15:07:10 2019 +0100

    Allows source to be given as a parameter on the commandline
---
 onionperf/onionperf       |  9 ++++++++-
 onionperf/reprocessing.py | 42 ++++++++++++++++++++++++------------------
 2 files changed, 32 insertions(+), 19 deletions(-)

diff --git a/onionperf/onionperf b/onionperf/onionperf
index fab805b..45ead3c 100755
--- a/onionperf/onionperf
+++ b/onionperf/onionperf
@@ -385,6 +385,13 @@ files generated by this script will be written""",
         action="store", dest="date_filter",
         default=None)
 
+    reprocess_parser.add_argument('-n', '--nickname',
+        help="""a nickname STRING that identifies the machine where the input logfiles were produced""",
+        metavar="STRING", type=str,
+        action="store", dest="nickname",
+        default=None)
+
+
     # get args and call the command handler for the chosen mode
     args = main_parser.parse_args()
     args.func(args)
@@ -495,7 +502,7 @@ def reprocess(args):
         torctl_logs = reprocessing.collect_logs(args.torctl_dirpath, '*torctl.log')
         log_pairs = reprocessing.match(tgen_logs, torctl_logs, args.date_filter)
         logging.info("Found {0} matching log pairs to be reprocessed".format(len(log_pairs)))
-        reprocessing.multiprocess_logs(log_pairs, args.prefix)
+        reprocessing.multiprocess_logs(log_pairs, args.prefix, args.nickname)
 
 def type_nonnegative_integer(value):
     i = int(value)
diff --git a/onionperf/reprocessing.py b/onionperf/reprocessing.py
index 25111ba..66b82c7 100644
--- a/onionperf/reprocessing.py
+++ b/onionperf/reprocessing.py
@@ -7,6 +7,8 @@ import fnmatch
 import logging
 import os
 import re
+import sys
+
 
 def collect_logs(dirpath, pattern):
     logs = []
@@ -14,7 +16,7 @@ def collect_logs(dirpath, pattern):
         for filename in fnmatch.filter(filenames, pattern):
             logs.append(os.path.join(root, filename))
     return logs
-    
+
 
 def match(tgen_logs, tor_logs, date_filter):
     log_pairs = []
@@ -24,41 +26,47 @@ def match(tgen_logs, tor_logs, date_filter):
             date = m.group(0)
             fdate = datetime.datetime.strptime(date, "%Y-%m-%d")
             found = False
-            if date_filter is None or util.do_dates_match(date_filter,fdate):
+            if date_filter is None or util.do_dates_match(date_filter, fdate):
                 for tor_log in tor_logs:
                     if date in tor_log:
                         log_pairs.append((tgen_log, tor_log, fdate))
                         found = True
-                        break                       
+                        break
                 if not found:
-                    logging.warning('Skipping file {0}, could not find a match for it'.format(tgen_log))
-                           
+                    logging.warning(
+                        'Skipping file {0}, could not find a match for it'.
+                        format(tgen_log))
+
         else:
-            logging.warning('Filename {0} does not contain a date'.format(tgen_log))
+            logging.warning(
+                'Filename {0} does not contain a date'.format(tgen_log))
     if not log_pairs:
-       logging.warning('Could not find any log matches. No analyses will be performed')
+        logging.warning(
+            'Could not find any log matches. No analyses will be performed')
     return log_pairs
- 
 
-def analyze_func(prefix, pair):
-    analysis = Analysis()
+
+def analyze_func(prefix, nick, pair):
+    analysis = Analysis(nickname=nick)
     logging.info('Analysing pair for date {0}'.format(pair[2]))
     analysis.add_tgen_file(pair[0])
     analysis.add_torctl_file(pair[1])
     analysis.analyze(do_simple=False, date_filter=pair[2])
     analysis.save(output_prefix=prefix)
-    analysis.export_torperf_version_1_1(output_prefix=prefix, do_compress=False)
+    analysis.export_torperf_version_1_1(
+        output_prefix=prefix, do_compress=False)
     return 1
- 
- 
-def multiprocess_logs(log_pairs, prefix):
+
+
+def multiprocess_logs(log_pairs, prefix, nick):
     pool = Pool(cpu_count())
     analyses = None
     try:
-        func = partial(analyze_func, prefix)
+        func = partial(analyze_func, prefix, nick)
         mr = pool.map_async(func, log_pairs)
         pool.close()
-        while not mr.ready(): mr.wait(1)
+        while not mr.ready():
+            mr.wait(1)
     except KeyboardInterrupt:
         logging.info("interrupted, terminating process pool")
         pool.terminate()
@@ -66,5 +74,3 @@ def multiprocess_logs(log_pairs, prefix):
         sys.exit()
     except Exception as e:
         logging.error(e)
-
-