[tor-commits] [onionperf/develop] Make models more configurable.
karsten at torproject.org
karsten at torproject.org
Thu Aug 27 09:07:15 UTC 2020
commit 7ef8c64833d41337d5c9cc5baaee2808092c9aad
Author: Philipp Winter <phw at nymity.ch>
Date: Fri Jun 26 10:00:29 2020 -0700
Make models more configurable.
This patch removes the --oneshot subcommand and replaces it with several
new subcommands for OnionPerf's "measure" command:
--tgen-start-pause (Initial pause before file transfers.)
--tgen-num-transfers (Number of file transfers.)
--tgen-intertransfer-pause (Pause in between file transfers.)
--tgen-transfer-size (Size of each file transfer.)
By default, OnionPerf continues to run in "continuous" mode. One can
simulate oneshot mode by running onionperf with the following flags:
onionperf measure --tgen-num-transfers=1
In addition to the above subcommands, this patch improves the code base
by 1) adding a TGenConf class to hold TGen's configuration and by 2)
adding a TGenModelConf class to hold TGen's traffic model.
This fixes tpo/metrics/onionperf#33432.
---
onionperf/measurement.py | 102 +++++++++++++++++++---------------
onionperf/model.py | 108 +++++++++++++++++++++++-------------
onionperf/onionperf | 60 +++++++++++++++-----
onionperf/tests/test_measurement.py | 12 ++--
4 files changed, 175 insertions(+), 107 deletions(-)
diff --git a/onionperf/measurement.py b/onionperf/measurement.py
index af1fa0d..e2d8d1c 100644
--- a/onionperf/measurement.py
+++ b/onionperf/measurement.py
@@ -15,6 +15,16 @@ from stem.control import Controller
from stem.version import Version, Requirement, get_system_tor_version
from stem import __version__ as stem_version
+class TGenConf(object):
+ """Represents a TGen configuration, for both client and server."""
+ def __init__(self, listen_port=None, connect_ip=None, connect_port=None, tor_ctl_port=None, tor_socks_port=None):
+ self.listen_port = str(listen_port)
+ self.tor_ctl_port = tor_ctl_port
+ self.tor_socks_port = tor_socks_port
+ # TGen clients use connect_ip and connect_port.
+ self.connect_ip = connect_ip
+ self.connect_port = connect_port
+
# onionperf imports
from . import analysis, monitor, model, util
@@ -173,12 +183,11 @@ def logrotate_thread_task(writables, tgen_writable, torctl_writable, docroot, ni
class Measurement(object):
- def __init__(self, tor_bin_path, tgen_bin_path, datadir_path, privatedir_path, nickname, oneshot, additional_client_conf=None, torclient_conf_file=None, torserver_conf_file=None, single_onion=False):
+ def __init__(self, tor_bin_path, tgen_bin_path, datadir_path, privatedir_path, nickname, additional_client_conf=None, torclient_conf_file=None, torserver_conf_file=None, single_onion=False):
self.tor_bin_path = tor_bin_path
self.tgen_bin_path = tgen_bin_path
self.datadir_path = datadir_path
self.privatedir_path = privatedir_path
- self.oneshot = oneshot
self.nickname = nickname
self.threads = None
self.done_event = None
@@ -190,20 +199,30 @@ class Measurement(object):
self.torserver_conf_file = torserver_conf_file
self.single_onion = single_onion
- def run(self, do_onion=True, do_inet=True, client_tgen_listen_port=58888, client_tgen_connect_ip='0.0.0.0', client_tgen_connect_port=8080, client_tor_ctl_port=59050, client_tor_socks_port=59000,
- server_tgen_listen_port=8080, server_tor_ctl_port=59051, server_tor_socks_port=59001):
+ def run(self, do_onion=True, do_inet=True, tgen_model=None, tgen_client_conf=None, tgen_server_conf=None):
'''
- only `server_tgen_listen_port` are "public" and need to be opened on the firewall.
- if `client_tgen_connect_port` != `server_tgen_listen_port`, then you should have installed a forwarding rule in the firewall.
+ only `tgen_server_conf.listen_port` are "public" and need to be opened on the firewall.
+ if `tgen_client_conf.connect_port` != `tgen_server_conf.listen_port`, then you should have installed a forwarding rule in the firewall.
all ports need to be unique though, and unique among multiple onionperf instances.
here are some sane defaults:
- client_tgen_listen_port=58888, client_tgen_connect_port=8080, client_tor_ctl_port=59050, client_tor_socks_port=59000,
- server_tgen_listen_port=8080, server_tor_ctl_port=59051, server_tor_socks_port=59001
+ tgen_client_conf.listen_port=58888, tgen_client_conf.connect_port=8080, tgen_client_conf.tor_ctl_port=59050, tgen_client_conf.tor_socks_port=59000,
+ tgen_server_conf.listen_port=8080, tgen_server_conf.tor_ctl_port=59051, tgen_server_conf.tor_socks_port=59001
'''
self.threads = []
self.done_event = threading.Event()
+ if tgen_client_conf is None:
+ tgen_client_conf = TGenConf(listen_port=58888,
+ connect_ip='0.0.0.0',
+ connect_port=8080,
+ tor_ctl_port=59050,
+ tor_socks_port=59000)
+ if tgen_server_conf is None:
+ tgen_server_conf = TGenConf(listen_port=8080,
+ tor_ctl_port=59051,
+ tor_socks_port=59001)
+
# if ctrl-c is pressed, shutdown child processes properly
try:
# make sure stem and Tor supports ephemeral HS (version >= 0.2.7.1-alpha)
@@ -225,52 +244,53 @@ class Measurement(object):
tgen_client_writable, torctl_client_writable = None, None
if do_onion or do_inet:
- general_writables.append(self.__start_tgen_server(server_tgen_listen_port))
+ tgen_model.port = tgen_server_conf.listen_port
+ general_writables.append(self.__start_tgen_server(tgen_model))
if do_onion:
logging.info("Onion Service private keys will be placed in {0}".format(self.privatedir_path))
# one must not have an open socks port when running a single
# onion service. see tor's man page for more information.
if self.single_onion:
- server_tor_socks_port = 0
- tor_writable, torctl_writable = self.__start_tor_server(server_tor_ctl_port,
- server_tor_socks_port,
- {client_tgen_connect_port:server_tgen_listen_port})
+ tgen_server_conf.tor_socks_port = 0
+ tor_writable, torctl_writable = self.__start_tor_server(tgen_server_conf.tor_ctl_port,
+ tgen_server_conf.tor_socks_port,
+ {tgen_client_conf.connect_port:tgen_server_conf.listen_port})
general_writables.append(tor_writable)
general_writables.append(torctl_writable)
if do_onion or do_inet:
- tor_writable, torctl_client_writable = self.__start_tor_client(client_tor_ctl_port, client_tor_socks_port)
+ tor_writable, torctl_client_writable = self.__start_tor_client(tgen_client_conf.tor_ctl_port, tgen_client_conf.tor_socks_port)
general_writables.append(tor_writable)
server_urls = []
if do_onion and self.hs_v3_service_id is not None:
- server_urls.append("{0}.onion:{1}".format(self.hs_v3_service_id, client_tgen_connect_port))
+ server_urls.append("{0}.onion:{1}".format(self.hs_v3_service_id, tgen_client_conf.connect_port))
if do_inet:
- connect_ip = client_tgen_connect_ip if client_tgen_connect_ip != '0.0.0.0' else util.get_ip_address()
- server_urls.append("{0}:{1}".format(connect_ip, client_tgen_connect_port))
+ connect_ip = tgen_client_conf.connect_ip if tgen_client_conf.connect_ip != '0.0.0.0' else util.get_ip_address()
+ server_urls.append("{0}:{1}".format(connect_ip, tgen_client_conf.connect_port))
+ tgen_model.servers = server_urls
if do_onion or do_inet:
assert len(server_urls) > 0
- tgen_client_writable = self.__start_tgen_client(server_urls, client_tgen_listen_port, client_tor_socks_port)
+ tgen_model.port = tgen_client_conf.listen_port
+ tgen_model.socks_port = tgen_client_conf.tor_socks_port
+ tgen_client_writable = self.__start_tgen_client(tgen_model)
self.__start_log_processors(general_writables, tgen_client_writable, torctl_client_writable)
logging.info("Bootstrapping finished, entering heartbeat loop")
time.sleep(1)
- if self.oneshot:
- logging.info("Onionperf is running in Oneshot mode. It will download a 5M file and shut down gracefully...")
while True:
- # TODO add status update of some kind? maybe the number of files in the www directory?
- # logging.info("Heartbeat: {0} downloads have completed successfully".format(self.__get_download_count(tgen_client_writable.filename)))
- if self.oneshot:
+ if tgen_model.num_transfers:
downloads = 0
while True:
downloads = self.__get_download_count(tgen_client_writable.filename)
- if downloads >= 1:
- logging.info("Onionperf has downloaded a 5M file in oneshot mode, and will now shut down.")
- break
+ time.sleep(1)
+ if downloads >= tgen_model.num_transfers:
+ logging.info("Onionperf has downloaded %d files and will now shut down." % tgen_model.num_transfers)
+ break
else:
continue
break
@@ -320,35 +340,25 @@ class Measurement(object):
logrotate.start()
self.threads.append(logrotate)
- def __start_tgen_client(self, server_urls, tgen_port, socks_port):
- return self.__start_tgen("client", tgen_port, socks_port, server_urls)
+ def __start_tgen_client(self, tgen_model_conf):
+ return self.__start_tgen("client", tgen_model_conf)
- def __start_tgen_server(self, tgen_port):
- return self.__start_tgen("server", tgen_port)
+ def __start_tgen_server(self, tgen_model_conf):
+ return self.__start_tgen("server", tgen_model_conf)
- def __start_tgen(self, name, tgen_port, socks_port=None, server_urls=None):
- logging.info("Starting TGen {0} process on port {1}...".format(name, tgen_port))
+ def __start_tgen(self, name, tgen_model_conf):
+ logging.info("Starting TGen {0} process on port {1}...".format(name, tgen_model_conf.port))
tgen_datadir = "{0}/tgen-{1}".format(self.datadir_path, name)
if not os.path.exists(tgen_datadir): os.makedirs(tgen_datadir)
tgen_confpath = "{0}/tgen.graphml.xml".format(tgen_datadir)
if os.path.exists(tgen_confpath): os.remove(tgen_confpath)
- if socks_port is None:
- model.ListenModel(tgen_port="{0}".format(tgen_port)).dump_to_file(tgen_confpath)
- logging.info("TGen server running at 0.0.0.0:{0}".format(tgen_port))
+ if tgen_model_conf.socks_port is None:
+ model.ListenModel(tgen_port="{0}".format(tgen_model_conf.port)).dump_to_file(tgen_confpath)
+ logging.info("TGen server running at 0.0.0.0:{0}".format(tgen_model_conf.port))
else:
-
- tgen_model_args = {
- "tgen_port": "{0}".format(tgen_port),
- "tgen_servers": server_urls,
- "socksproxy": "127.0.0.1:{0}".format(socks_port)
- }
- if self.oneshot:
- tgen_model = model.OneshotModel(**tgen_model_args)
- else:
- tgen_model = model.TorperfModel(**tgen_model_args)
-
+ tgen_model = model.TorperfModel(tgen_model_conf)
tgen_model.dump_to_file(tgen_confpath)
tgen_logpath = "{0}/onionperf.tgen.log".format(tgen_datadir)
diff --git a/onionperf/model.py b/onionperf/model.py
index cb45f51..a4af2fc 100644
--- a/onionperf/model.py
+++ b/onionperf/model.py
@@ -41,6 +41,21 @@ class TGenLoadableModel(TGenModel):
model_instance = cls(graph)
return model_instance
+class TGenModelConf(object):
+ """Represents a TGen traffic model configuration."""
+ def __init__(self, initial_pause=0, num_transfers=1, transfer_size="5 MiB",
+ continuous_transfers=False, inter_transfer_pause=5, port=None, servers=[],
+ socks_port=None):
+ self.initial_pause = initial_pause
+ self.num_transfers = num_transfers
+ self.transfer_size = transfer_size
+ self.continuous_transfers = continuous_transfers
+ self.inter_transfer_pause = inter_transfer_pause
+ self.port = port
+ self.servers = servers
+ self.socks_port = socks_port
+
+
class GeneratableTGenModel(TGenModel, metaclass=ABCMeta):
@abstractmethod
@@ -58,61 +73,74 @@ class ListenModel(GeneratableTGenModel):
g.add_node("start", serverport=self.tgen_port, loglevel="info", heartbeat="1 minute")
return g
+
class TorperfModel(GeneratableTGenModel):
- def __init__(self, tgen_port="8889", tgen_servers=["127.0.0.1:8888"], socksproxy=None):
- self.tgen_port = tgen_port
- self.tgen_servers = tgen_servers
- self.socksproxy = socksproxy
+ def __init__(self, config):
+ self.config = config
self.graph = self.generate()
def generate(self):
- server_str = ','.join(self.tgen_servers)
+ server_str = ','.join(self.config.servers)
g = DiGraph()
- if self.socksproxy is not None:
- g.add_node("start", serverport=self.tgen_port, peers=server_str, loglevel="info", heartbeat="1 minute", socksproxy=self.socksproxy)
+ if self.config.socks_port is not None:
+ g.add_node("start",
+ serverport=self.config.port,
+ peers=server_str,
+ loglevel="info",
+ heartbeat="1 minute",
+ socksproxy="127.0.0.1:{0}".format(self.config.socks_port))
else:
- g.add_node("start", serverport=self.tgen_port, peers=server_str, loglevel="info", heartbeat="1 minute")
- g.add_node("pause", time="5 minutes")
- g.add_node("stream5m", sendsize="0", recvsize="5 mib", timeout="270 seconds", stallout="0 seconds")
+ g.add_node("start",
+ serverport=self.config.port,
+ peers=server_str,
+ loglevel="info",
+ heartbeat="1 minute")
+ g.add_node("pause", time="%d seconds" % self.config.initial_pause)
g.add_edge("start", "pause")
- # after the pause, we start another pause timer while *at the same time* choosing one of
- # the file sizes and downloading it from one of the servers in the server pool
- g.add_edge("pause", "pause")
-
- # these are chosen with weighted probability, change edge 'weight' attributes to adjust probability
- g.add_edge("pause", "stream5m")
-
- return g
-
-class OneshotModel(GeneratableTGenModel):
-
- def __init__(self, tgen_port="8889", tgen_servers=["127.0.0.1:8888"], socksproxy=None):
- self.tgen_port = tgen_port
- self.tgen_servers = tgen_servers
- self.socksproxy = socksproxy
- self.graph = self.generate()
-
- def generate(self):
- server_str = ','.join(self.tgen_servers)
- g = DiGraph()
-
- if self.socksproxy is not None:
- g.add_node("start", serverport=self.tgen_port, peers=server_str, loglevel="info", heartbeat="1 minute", socksproxy=self.socksproxy)
- else:
- g.add_node("start", serverport=self.tgen_port, peers=server_str, loglevel="info", heartbeat="1 minute")
- g.add_node("stream5m", sendsize="0", recvsize="5 mib", timeout="270 seconds", stallout="0 seconds")
-
- g.add_edge("start", "stream5m")
- g.add_edge("stream5m", "start")
+ # "One-shot mode," i.e., onionperf will stop after the given number of
+ # iterations. The idea is:
+ # start -> pause -> stream-1 -> pause-1 -> ... -> stream-n -> pause-n -> end
+ if self.config.num_transfers > 0:
+ for i in range(self.config.num_transfers):
+ g.add_node("stream-%d" % i,
+ sendsize="0",
+ recvsize=self.config.transfer_size,
+ timeout="15 seconds",
+ stallout="10 seconds")
+ g.add_node("pause-%d" % i,
+ time="%d seconds" % self.config.inter_transfer_pause)
+
+ g.add_edge("stream-%d" % i, "pause-%d" % i)
+ if i > 0:
+ g.add_edge("pause-%d" % (i-1), "stream-%d" % i)
+
+ g.add_node("end")
+ g.add_edge("pause", "stream-0")
+ g.add_edge("pause-%d" % (self.config.num_transfers - 1), "end")
+
+ # Continuous mode, i.e., onionperf will not stop. The idea is:
+ # start -> pause -> stream -> pause
+ # ^ |
+ # +-------+
+ elif self.config.continuous_transfers:
+ g.add_node("stream",
+ sendsize="0",
+ recvsize=self.config.transfer_size,
+ timeout="15 seconds",
+ stallout="10 seconds")
+ g.add_node("pause",
+ time="%d seconds" % self.config.inter_transfer_pause)
+ g.add_edge("pause", "stream")
+ g.add_edge("stream", "pause")
+ g.add_edge("pause", "stream")
return g
-
def dump_example_tgen_torperf_model(domain_name, onion_name):
# the server listens on 8888, the client uses Tor to come back directly, and using a hidden serv
server = ListenModel(tgen_port="8888")
diff --git a/onionperf/onionperf b/onionperf/onionperf
index e8024ce..d95e691 100755
--- a/onionperf/onionperf
+++ b/onionperf/onionperf
@@ -154,11 +154,6 @@ def main():
action="store", dest="tgenpath",
default=util.which("tgen"))
- measure_parser.add_argument('--oneshot',
- help="""Enables oneshot mode, onionperf closes on successfully downloading a file""",
- action="store_true", dest="oneshot",
- default=False)
-
measure_parser.add_argument('--additional-client-conf',
help="""Additional configuration lines for the Tor client, for example bridge lines""",
metavar="CONFIG", type=str,
@@ -195,6 +190,30 @@ def main():
action="store", dest="tgenconnectport",
default=8080)
+ measure_parser.add_argument('--tgen-start-pause',
+ help="""the number of seconds TGen should wait before walking through its action graph""",
+ metavar="N", type=int,
+ action="store", dest="tgenstartpause",
+ default=5)
+
+ measure_parser.add_argument('--tgen-intertransfer-pause',
+ help="""the number of seconds TGen should wait in between two transfers""",
+ metavar="N", type=int,
+ action="store", dest="tgenintertransferpause",
+ default=300)
+
+ measure_parser.add_argument('--tgen-transfer-size',
+ help="""the size of the file transfer that TGen will perform (e.g., '5 MiB' or '10 KiB')""",
+ metavar="STRING", type=str,
+ action="store", dest="tgentransfersize",
+ default="5 MiB")
+
+ measure_parser.add_argument('--tgen-num-transfers',
+ help="""the number of file transfers that TGen will perform""",
+ metavar="N", type=int,
+ action="store", dest="tgennumtransfers",
+ default=0)
+
onion_or_inet_only_group = measure_parser.add_mutually_exclusive_group()
onion_or_inet_only_group.add_argument('-o', '--onion-only',
@@ -327,7 +346,8 @@ def monitor(args):
writer.close()
def measure(args):
- from onionperf.measurement import Measurement
+ from onionperf.measurement import Measurement, TGenConf
+ from onionperf.model import TGenModelConf
# check paths
args.torpath = util.find_path(args.torpath, "tor")
@@ -347,12 +367,27 @@ def measure(args):
server_tor_ctl_port = util.get_random_free_port()
server_tor_socks_port = util.get_random_free_port()
+ tgen_client_conf = TGenConf(listen_port=client_tgen_port,
+ connect_ip=client_connect_ip,
+ connect_port=client_connect_port,
+ tor_ctl_port=client_tor_ctl_port,
+ tor_socks_port=client_tor_socks_port)
+
+ tgen_server_conf = TGenConf(listen_port=server_tgen_port,
+ tor_ctl_port=server_tor_ctl_port,
+ tor_socks_port=server_tor_socks_port)
+
+ tgen_model = TGenModelConf(initial_pause=args.tgenstartpause,
+ transfer_size=args.tgentransfersize,
+ num_transfers=args.tgennumtransfers,
+ continuous_transfers=args.tgennumtransfers == 0,
+ inter_transfer_pause=args.tgenintertransferpause)
+
meas = Measurement(args.torpath,
args.tgenpath,
args.prefix,
args.private_prefix,
args.nickname,
- args.oneshot,
args.additional_client_conf,
args.torclient_conf_file,
args.torserver_conf_file,
@@ -360,14 +395,9 @@ def measure(args):
meas.run(do_onion=not args.inet_only,
do_inet=not args.onion_only,
- client_tgen_listen_port=client_tgen_port,
- client_tgen_connect_ip=client_connect_ip,
- client_tgen_connect_port=client_connect_port,
- client_tor_ctl_port=client_tor_ctl_port,
- client_tor_socks_port=client_tor_socks_port,
- server_tgen_listen_port=server_tgen_port,
- server_tor_ctl_port=server_tor_ctl_port,
- server_tor_socks_port=server_tor_socks_port)
+ tgen_model=tgen_model,
+ tgen_client_conf=tgen_client_conf,
+ tgen_server_conf=tgen_server_conf)
else:
logging.info("Please fix path errors to continue")
diff --git a/onionperf/tests/test_measurement.py b/onionperf/tests/test_measurement.py
index e5010fa..6bca8ab 100644
--- a/onionperf/tests/test_measurement.py
+++ b/onionperf/tests/test_measurement.py
@@ -57,8 +57,8 @@ WarnUnsafeSocks 0\nSafeLogging 0\nMaxCircuitDirtiness 60 seconds\nDataDirectory
known_config_server = "RunAsDaemon 0\nORPort 0\nDirPort 0\nControlPort 9001\nSocksPort 9050\nSocksListenAddress 127.0.0.1\nClientOnly 1\n\
WarnUnsafeSocks 0\nSafeLogging 0\nMaxCircuitDirtiness 60 seconds\nDataDirectory /tmp/\nDataDirectoryGroupReadable 1\nLog INFO stdout\nUseEntryGuards 0\n"
- meas = measurement.Measurement(None, None, None, None, None, None,
- "UseBridges 1\n", None, None)
+ meas = measurement.Measurement(None, None, None, None, None,
+ "UseBridges 1\n", None, None, False)
config_client = meas.create_tor_config(9001, 9050, "/tmp/", "client")
config_server = meas.create_tor_config(9001, 9050, "/tmp/", "server")
assert_equals(config_client, known_config)
@@ -80,8 +80,8 @@ WarnUnsafeSocks 0\nSafeLogging 0\nMaxCircuitDirtiness 60 seconds\nDataDirectory
known_config = "RunAsDaemon 0\nORPort 0\nDirPort 0\nControlPort 9001\nSocksPort 9050\nSocksListenAddress 127.0.0.1\nClientOnly 1\n\
WarnUnsafeSocks 0\nSafeLogging 0\nMaxCircuitDirtiness 60 seconds\nDataDirectory /tmp/\nDataDirectoryGroupReadable 1\nLog INFO stdout\nUseBridges 1\n"
- meas = measurement.Measurement(None, None, None, None, None, None, None,
- absolute_data_path("config"), None)
+ meas = measurement.Measurement(None, None, None, None, None, None,
+ absolute_data_path("config"), None, False)
config_client = meas.create_tor_config(9001, 9050, "/tmp/", "client")
config_server = meas.create_tor_config(9001, 9050, "/tmp/", "server")
assert_equals(config_client, known_config)
@@ -103,8 +103,8 @@ WarnUnsafeSocks 0\nSafeLogging 0\nMaxCircuitDirtiness 60 seconds\nDataDirectory
known_config = "RunAsDaemon 0\nORPort 0\nDirPort 0\nControlPort 9001\nSocksPort 9050\nSocksListenAddress 127.0.0.1\nClientOnly 1\n\
WarnUnsafeSocks 0\nSafeLogging 0\nMaxCircuitDirtiness 60 seconds\nDataDirectory /tmp/\nDataDirectoryGroupReadable 1\nLog INFO stdout\nUseEntryGuards 0\n"
- meas = measurement.Measurement(None, None, None, None, None, None, None, None,
- absolute_data_path("config"))
+ meas = measurement.Measurement(None, None, None, None, None, None, None,
+ absolute_data_path("config"), False)
config_client = meas.create_tor_config(9001, 9050, "/tmp/", "client")
config_server = meas.create_tor_config(9001, 9050, "/tmp/", "server")
assert_equals(config_client, known_config)
More information about the tor-commits
mailing list