Skip to content

GitLab

"vscode:/vscode.git/clone" did not exist on "dd5d4dd875193dc66830cf966d59b24270e58341"
Commit 7549fa9e authored by Jonas Kaufmann's avatar Jonas Kaufmann Committed by Antoine Kaufmann
Browse files

pre-commit run -a

parent aac98df8
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 985 additions and 422 deletions
experiments/pyexps/netperf.py
View file @ 7549fa9e
......@@ -19,16 +19,20 @@
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""Experiment, which simulates two hosts, one running a netperf server and the
"""
Experiment, which simulates two hosts, one running a netperf server and the
other a client with the goal of measuring latency and throughput between them.
The goal is to compare different simulators for host, NIC, and the network in
terms of simulated network throughput and latency."""
terms of simulated network throughput and latency.
"""
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.nodeconfig as node
import simbricks.simulators as sim
from simbricks.simulator_utils import create_basic_hosts
import simbricks.experiments as exp
host_types = ['qemu', 'gem5', 'qt']
nic_types = ['i40e', 'cd_bm', 'cd_verilator']
net_types = ['switch', 'ns3']
......@@ -39,7 +43,9 @@ experiments = []
for host_type in host_types:
for nic_type in nic_types:
for net_type in net_types:
e = exp.Experiment('netperf-' + host_type + '-' + net_type + '-' + nic_type)
e = exp.Experiment(
'netperf-' + host_type + '-' + net_type + '-' + nic_type
)
# network
if net_type == 'switch':
......@@ -54,10 +60,12 @@ for host_type in host_types:
if host_type == 'qemu':
host_class = sim.QemuHost
elif host_type == 'qt':
def qemu_timing():
h = sim.QemuHost()
h.sync = True
return h
host_class = qemu_timing
elif host_type == 'gem5':
host_class = sim.Gem5Host
......@@ -79,11 +87,28 @@ for host_type in host_types:
raise NameError(nic_type)
# create servers and clients
servers = create_basic_hosts(e, 1, 'server', net, nic_class, host_class,
nc_class, node.NetperfServer)
servers = create_basic_hosts(
e,
1,
'server',
net,
nic_class,
host_class,
nc_class,
node.NetperfServer
)
clients = create_basic_hosts(e, 1, 'client', net, nic_class, host_class,
nc_class, node.NetperfClient, ip_start = 2)
clients = create_basic_hosts(
e,
1,
'client',
net,
nic_class,
host_class,
nc_class,
node.NetperfClient,
ip_start=2
)
for c in clients:
c.wait = True
......
experiments/pyexps/no-simbricks.py
View file @ 7549fa9e
......@@ -20,16 +20,15 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.nodeconfig as node
import simbricks.simulators as sim
import simbricks.experiments as exp
app_types = ['sleep', 'busy']
experiments = []
for app_type in app_types:
e = exp.Experiment('no_simb-gt-' + app_type)
......@@ -37,11 +36,10 @@ for app_type in app_types:
host_class = sim.Gem5Host
e.checkpoint = True
e.no_simbricks = True
nc_class = node.I40eLinuxNode
# create servers and clients
host = host_class()
host.name = 'host.0'
node_config = nc_class()
......@@ -49,22 +47,17 @@ for app_type in app_types:
node_config.app = node.NoTraffic()
node_config.cores = 1
# is busy
# is busy
if app_type == 'sleep':
node_config.app.is_sleep = 1
else:
node_config.app.is_sleep = 0
host.set_config(node_config)
e.add_host(host)
host.wait = True
print(e.name)
# add to experiments
experiments.append(e)
experiments/pyexps/no_traffic.py
View file @ 7549fa9e
......@@ -20,11 +20,11 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.nodeconfig as node
import simbricks.simulators as sim
from simbricks.simulator_utils import create_basic_hosts
import simbricks.experiments as exp
# iperf TCP_single test
# naming convention following host-nic-net-app
......@@ -34,11 +34,10 @@ from simbricks.simulator_utils import create_basic_hosts
# app: UDPs
host_types = ['gt', 'qt', 'qemu']
nic_types = ['cv','cb','ib']
nic_types = ['cv', 'cb', 'ib']
net_types = ['sw', 'br']
app_types = ['sleep', 'busy']
num_cores = 1
n_client = 1
......@@ -49,7 +48,10 @@ for host_type in host_types:
for net_type in net_types:
for app_type in app_types:
e = exp.Experiment('noTraf-' + host_type + '-' + nic_type + '-' + net_type + '-' + app_type)
e = exp.Experiment(
'noTraf-' + host_type + '-' + nic_type + '-' + net_type +
'-' + app_type
)
#e.no_simbricks = False
# network
if net_type == 'sw':
......@@ -64,10 +66,12 @@ for host_type in host_types:
if host_type == 'qemu':
host_class = sim.QemuHost
elif host_type == 'qt':
def qemu_timing():
h = sim.QemuHost()
h.sync = True
return h
host_class = qemu_timing
elif host_type == 'gt':
host_class = sim.Gem5Host
......@@ -92,35 +96,38 @@ for host_type in host_types:
"""
servers = create_basic_hosts(e, 1, 'server', net, nic_class, host_class,
nc_class, node.NoTraffic)
for s in servers:
s.node_config.cores = num_cores
s.node_config.app.is_sleep = 0
s.node_config.app.is_server = 1
"""
clients = create_basic_hosts(e, n_client, 'client', net, nic_class, host_class,
nc_class, node.NoTraffic, ip_start=2)
clients = create_basic_hosts(
e,
n_client,
'client',
net,
nic_class,
host_class,
nc_class,
node.NoTraffic,
ip_start=2
)
clients[n_client-1].wait = True
clients[n_client - 1].wait = True
for c in clients:
c.node_config.cores = num_cores
# is busy
# is busy
if app_type == 'sleep':
c.node_config.app.is_sleep = 1
else:
c.node_config.app.is_sleep = 0
#c.wait = True
print(e.name)
# add to experiments
experiments.append(e)
experiments/pyexps/nopaxos.py
View file @ 7549fa9e
......@@ -20,11 +20,12 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.nodeconfig as node
import simbricks.simulators as sim
from simbricks.simulator_utils import create_basic_hosts
import simbricks.experiments as exp
host_configs = ['qemu', 'gt', 'qt']
seq_configs = ['swseq', 'ehseq', 'tofino']
nic_configs = ['ib', 'cb', 'cv']
......@@ -33,7 +34,7 @@ num_client_configs = [1, 2, 3, 4, 5, 6, 7, 8, 10, 12]
experiments = []
sync_period = 200
link_rate_opt = '--LinkRate=100Gb/s ' # don't forget space at the end
link_rate_opt = '--LinkRate=100Gb/s ' # don't forget space at the end
link_latency_opt = '--LinkLatency=500ns '
for proto_config in proto_configs:
......@@ -41,7 +42,10 @@ for proto_config in proto_configs:
for host_config in host_configs:
for seq_config in seq_configs:
for nic_config in nic_configs:
e = exp.Experiment(proto_config + '-' + host_config + '-' + nic_config + '-' + seq_config + f'-{num_c}')
e = exp.Experiment(
proto_config + '-' + host_config + '-' + nic_config +
'-' + seq_config + f'-{num_c}'
)
if seq_config == 'tofino':
net = sim.TofinoNet()
else:
......@@ -58,10 +62,12 @@ for proto_config in proto_configs:
host_class = sim.Gem5Host
e.checkpoint = True
elif host_config == 'qt':
def qemu_timing():
h = sim.QemuHost()
h.sync = True
return h
host_class = qemu_timing
else:
raise NameError(host_config)
......@@ -79,7 +85,6 @@ for proto_config in proto_configs:
else:
raise NameError(nic_config)
# app
if proto_config == 'vr':
replica_class = node.VRReplica
......@@ -92,25 +97,53 @@ for proto_config in proto_configs:
# endhost sequencer
if seq_config == 'ehseq' and proto_config == 'nopaxos':
sequencer = create_basic_hosts(e, 1, 'sequencer', net, nic_class,
host_class, nc_class, node.NOPaxosSequencer, ip_start = 100)
sequencer = create_basic_hosts(
e,
1,
'sequencer',
net,
nic_class,
host_class,
nc_class,
node.NOPaxosSequencer,
ip_start=100
)
sequencer[0].node_config.disk_image = 'nopaxos'
sequencer[0].pcidevs[0].sync_period = sync_period
sequencer[0].sync_period = sync_period
replicas = create_basic_hosts(e, 3, 'replica', net, nic_class,
host_class, nc_class, replica_class)
replicas = create_basic_hosts(
e,
3,
'replica',
net,
nic_class,
host_class,
nc_class,
replica_class
)
for i in range(len(replicas)):
replicas[i].node_config.app.index = i
replicas[i].node_config.disk_image = 'nopaxos'
replicas[i].pcidevs[0].sync_period = sync_period
replicas[i].sync_period = sync_period
clients = create_basic_hosts(e, num_c, 'client', net, nic_class,
host_class, nc_class, client_class, ip_start = 4)
clients = create_basic_hosts(
e,
num_c,
'client',
net,
nic_class,
host_class,
nc_class,
client_class,
ip_start=4
)
for c in clients:
c.node_config.app.server_ips = ['10.0.0.1', '10.0.0.2', '10.0.0.3']
c.node_config.app.server_ips = [
'10.0.0.1', '10.0.0.2', '10.0.0.3'
]
if seq_config == 'ehseq':
c.node_config.app.server_ips.append('10.0.0.100')
c.node_config.app.use_ehseq = True
......@@ -124,4 +157,3 @@ for proto_config in proto_configs:
#print(e.name)
experiments.append(e)
experiments/pyexps/pci_validation.py
View file @ 7549fa9e
......@@ -23,12 +23,12 @@
e1000 NIC, and then with the extracted gem5 e1000 NIC connected through
SimBricks."""
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.nodeconfig as node
import simbricks.simulators as sim
experiments = []
import simbricks.experiments as exp
experiments = []
for internal in [True, False]:
if internal:
......@@ -66,8 +66,10 @@ for internal in [True, False]:
for h in [client, server]:
h.cpu_type = h.cpu_type_cp = 'TimingSimpleCPU'
h.variant = 'opt' # need opt gem5 variant with debug support
h.extra_main_args.append('--debug-flags=SimBricksEthernet,SimBricksPci,EthernetAll,EthernetDesc')
h.variant = 'opt' # need opt gem5 variant with debug support
h.extra_main_args.append(
'--debug-flags=SimBricksEthernet,SimBricksPci,EthernetAll,EthernetDesc'
)
if internal:
h.add_netdirect(net)
else:
......
experiments/pyexps/qemu_e1000_pair.py
View file @ 7549fa9e
......@@ -20,25 +20,41 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.nodeconfig as node
import simbricks.simulators as sim
from simbricks.simulator_utils import create_basic_hosts
import simbricks.experiments as exp
e = exp.Experiment('qemu-e1000-pair')
net = sim.SwitchNet()
e.add_network(net)
servers = create_basic_hosts(e, 1, 'server', net, sim.E1000NIC, sim.QemuHost,
node.E1000LinuxNode, node.IperfTCPServer)
clients = create_basic_hosts(e, 1, 'client', net, sim.E1000NIC, sim.QemuHost,
node.E1000LinuxNode, node.IperfTCPClient, ip_start = 2)
servers = create_basic_hosts(
e,
1,
'server',
net,
sim.E1000NIC,
sim.QemuHost,
node.E1000LinuxNode,
node.IperfTCPServer
)
clients = create_basic_hosts(
e,
1,
'client',
net,
sim.E1000NIC,
sim.QemuHost,
node.E1000LinuxNode,
node.IperfTCPClient,
ip_start=2
)
for c in clients:
c.wait = True
c.node_config.app.server_ip = servers[0].node_config.ip
experiments = [e]
experiments/pyexps/qemu_i40e_pair.py
View file @ 7549fa9e
......@@ -20,25 +20,41 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.nodeconfig as node
import simbricks.simulators as sim
from simbricks.simulator_utils import create_basic_hosts
import simbricks.experiments as exp
e = exp.Experiment('qemu-i40e-pair')
net = sim.SwitchNet()
e.add_network(net)
servers = create_basic_hosts(e, 1, 'server', net, sim.I40eNIC, sim.QemuHost,
node.I40eLinuxNode, node.IperfTCPServer)
clients = create_basic_hosts(e, 2, 'client', net, sim.I40eNIC, sim.QemuHost,
node.I40eLinuxNode, node.IperfTCPClient, ip_start = 2)
servers = create_basic_hosts(
e,
1,
'server',
net,
sim.I40eNIC,
sim.QemuHost,
node.I40eLinuxNode,
node.IperfTCPServer
)
clients = create_basic_hosts(
e,
2,
'client',
net,
sim.I40eNIC,
sim.QemuHost,
node.I40eLinuxNode,
node.IperfTCPClient,
ip_start=2
)
for c in clients:
c.wait = True
c.node_config.app.server_ip = servers[0].node_config.ip
experiments = [e]
experiments/pyexps/qemu_tcp_multi.py
View file @ 7549fa9e
......@@ -20,11 +20,11 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.nodeconfig as node
import simbricks.simulators as sim
from simbricks.simulator_utils import create_basic_hosts
import simbricks.experiments as exp
# iperf TCP_multi_client test
# naming convention following host-nic-net-app
......@@ -34,7 +34,7 @@ from simbricks.simulator_utils import create_basic_hosts
# app: TCPm
kinds_of_host = ['qemu']
kinds_of_nic = ['cv','cb','ib']
kinds_of_nic = ['cv', 'cb', 'ib']
kinds_of_net = ['switch', 'dumbbell', 'bridge']
kinds_of_app = ['TCPm']
num_client = 4
......@@ -51,39 +51,84 @@ for n in kinds_of_net:
if n == 'bridge':
net_class = sim.NS3BridgeNet
# set nic sim
for c in kinds_of_nic:
net = net_class()
e = exp.Experiment('qemu-' + c + '-' + n + '-' + 'TCPm')
e = exp.Experiment('qemu-' + c + '-' + n + '-' + 'TCPm')
e.add_network(net)
if c == 'cv':
servers = create_basic_hosts(e, 1, 'server', net, sim.CorundumVerilatorNIC, sim.QemuHost,
node.CorundumLinuxNode, node.IperfTCPServer)
clients = create_basic_hosts(e, num_client, 'client', net, sim.CorundumVerilatorNIC, sim.QemuHost,
node.CorundumLinuxNode, node.IperfTCPClient, ip_start = 2)
servers = create_basic_hosts(
e,
1,
'server',
net,
sim.CorundumVerilatorNIC,
sim.QemuHost,
node.CorundumLinuxNode,
node.IperfTCPServer
)
clients = create_basic_hosts(
e,
num_client,
'client',
net,
sim.CorundumVerilatorNIC,
sim.QemuHost,
node.CorundumLinuxNode,
node.IperfTCPClient,
ip_start=2
)
if c == 'cb':
servers = create_basic_hosts(e, 1, 'server', net, sim.CorundumBMNIC, sim.QemuHost,
node.CorundumLinuxNode, node.IperfTCPServer)
clients = create_basic_hosts(e, num_client, 'client', net, sim.CorundumBMNIC, sim.QemuHost,
node.CorundumLinuxNode, node.IperfTCPClient, ip_start = 2)
servers = create_basic_hosts(
e,
1,
'server',
net,
sim.CorundumBMNIC,
sim.QemuHost,
node.CorundumLinuxNode,
node.IperfTCPServer
)
clients = create_basic_hosts(
e,
num_client,
'client',
net,
sim.CorundumBMNIC,
sim.QemuHost,
node.CorundumLinuxNode,
node.IperfTCPClient,
ip_start=2
)
if c == 'ib':
servers = create_basic_hosts(e, 1, 'server', net, sim.I40eNIC, sim.QemuHost,
node.I40eLinuxNode, node.IperfTCPServer)
clients = create_basic_hosts(e, num_client, 'client', net, sim.I40eNIC, sim.QemuHost,
node.I40eLinuxNode, node.IperfTCPClient, ip_start = 2)
servers = create_basic_hosts(
e,
1,
'server',
net,
sim.I40eNIC,
sim.QemuHost,
node.I40eLinuxNode,
node.IperfTCPServer
)
clients = create_basic_hosts(
e,
num_client,
'client',
net,
sim.I40eNIC,
sim.QemuHost,
node.I40eLinuxNode,
node.IperfTCPClient,
ip_start=2
)
for cl in clients:
cl.wait = True
cl.node_config.app.server_ip = servers[0].node_config.ip
print(e.name)
experiments.append(e)
experiments/pyexps/qemu_tcp_single.py
View file @ 7549fa9e
......@@ -20,11 +20,11 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.nodeconfig as node
import simbricks.simulators as sim
from simbricks.simulator_utils import create_basic_hosts
import simbricks.experiments as exp
# iperf TCP_single test
# naming convention following host-nic-net-app
......@@ -34,7 +34,7 @@ from simbricks.simulator_utils import create_basic_hosts
# app: TCPs
kinds_of_host = ['qemu']
kinds_of_nic = ['cv','cb','ib']
kinds_of_nic = ['cv', 'cb', 'ib']
kinds_of_net = ['wire', 'switch', 'dumbbell', 'bridge', 'tofino']
kinds_of_app = ['TCPs']
......@@ -53,37 +53,83 @@ for n in kinds_of_net:
if n == 'tofino':
net_class = sim.TofinoNet
# set nic sim
for c in kinds_of_nic:
net = net_class()
e = exp.Experiment('qemu-' + c + '-' + n + '-' + 'TCPs')
e = exp.Experiment('qemu-' + c + '-' + n + '-' + 'TCPs')
e.add_network(net)
if c == 'cv':
servers = create_basic_hosts(e, 1, 'server', net, sim.CorundumVerilatorNIC, sim.QemuHost,
node.CorundumLinuxNode, node.IperfTCPServer)
clients = create_basic_hosts(e, 1, 'client', net, sim.CorundumVerilatorNIC, sim.QemuHost,
node.CorundumLinuxNode, node.IperfTCPClient, ip_start = 2)
servers = create_basic_hosts(
e,
1,
'server',
net,
sim.CorundumVerilatorNIC,
sim.QemuHost,
node.CorundumLinuxNode,
node.IperfTCPServer
)
clients = create_basic_hosts(
e,
1,
'client',
net,
sim.CorundumVerilatorNIC,
sim.QemuHost,
node.CorundumLinuxNode,
node.IperfTCPClient,
ip_start=2
)
if c == 'cb':
servers = create_basic_hosts(e, 1, 'server', net, sim.CorundumBMNIC, sim.QemuHost,
node.CorundumLinuxNode, node.IperfTCPServer)
clients = create_basic_hosts(e, 1, 'client', net, sim.CorundumBMNIC, sim.QemuHost,
node.CorundumLinuxNode, node.IperfTCPClient, ip_start = 2)
servers = create_basic_hosts(
e,
1,
'server',
net,
sim.CorundumBMNIC,
sim.QemuHost,
node.CorundumLinuxNode,
node.IperfTCPServer
)
clients = create_basic_hosts(
e,
1,
'client',
net,
sim.CorundumBMNIC,
sim.QemuHost,
node.CorundumLinuxNode,
node.IperfTCPClient,
ip_start=2
)
if c == 'ib':
servers = create_basic_hosts(e, 1, 'server', net, sim.I40eNIC, sim.QemuHost,
node.I40eLinuxNode, node.IperfTCPServer)
clients = create_basic_hosts(e, 1, 'client', net, sim.I40eNIC, sim.QemuHost,
node.I40eLinuxNode, node.IperfTCPClient, ip_start = 2)
servers = create_basic_hosts(
e,
1,
'server',
net,
sim.I40eNIC,
sim.QemuHost,
node.I40eLinuxNode,
node.IperfTCPServer
)
clients = create_basic_hosts(
e,
1,
'client',
net,
sim.I40eNIC,
sim.QemuHost,
node.I40eLinuxNode,
node.IperfTCPClient,
ip_start=2
)
clients[0].wait = True
clients[0].node_config.app.server_ip = servers[0].node_config.ip
print(e.name)
experiments.append(e)
experiments/pyexps/qemu_udp_multi.py
View file @ 7549fa9e
......@@ -20,11 +20,11 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.nodeconfig as node
import simbricks.simulators as sim
from simbricks.simulator_utils import create_basic_hosts
import simbricks.experiments as exp
# iperf TCP_multi_client test
# naming convention following host-nic-net-app
......@@ -34,7 +34,7 @@ from simbricks.simulator_utils import create_basic_hosts
# app: TCPm
kinds_of_host = ['qemu']
kinds_of_nic = ['cv','cb','ib']
kinds_of_nic = ['cv', 'cb', 'ib']
kinds_of_net = ['switch', 'dumbbell', 'bridge']
kinds_of_app = ['UDPm']
......@@ -53,35 +53,81 @@ for n in kinds_of_net:
if n == 'bridge':
net_class = sim.NS3BridgeNet
# set nic sim
for c in kinds_of_nic:
net = net_class()
e = exp.Experiment('qemu-' + c + '-' + n + '-' + 'UDPm')
e = exp.Experiment('qemu-' + c + '-' + n + '-' + 'UDPm')
e.add_network(net)
if c == 'cv':
servers = create_basic_hosts(e, 1, 'server', net, sim.CorundumVerilatorNIC, sim.QemuHost,
node.CorundumLinuxNode, node.IperfUDPServer)
clients = create_basic_hosts(e, num_client, 'client', net, sim.CorundumVerilatorNIC, sim.QemuHost,
node.CorundumLinuxNode, node.IperfUDPClient, ip_start = 2)
servers = create_basic_hosts(
e,
1,
'server',
net,
sim.CorundumVerilatorNIC,
sim.QemuHost,
node.CorundumLinuxNode,
node.IperfUDPServer
)
clients = create_basic_hosts(
e,
num_client,
'client',
net,
sim.CorundumVerilatorNIC,
sim.QemuHost,
node.CorundumLinuxNode,
node.IperfUDPClient,
ip_start=2
)
if c == 'cb':
servers = create_basic_hosts(e, 1, 'server', net, sim.CorundumBMNIC, sim.QemuHost,
node.CorundumLinuxNode, node.IperfUDPServer)
clients = create_basic_hosts(e, num_client, 'client', net, sim.CorundumBMNIC, sim.QemuHost,
node.CorundumLinuxNode, node.IperfUDPClient, ip_start = 2)
servers = create_basic_hosts(
e,
1,
'server',
net,
sim.CorundumBMNIC,
sim.QemuHost,
node.CorundumLinuxNode,
node.IperfUDPServer
)
clients = create_basic_hosts(
e,
num_client,
'client',
net,
sim.CorundumBMNIC,
sim.QemuHost,
node.CorundumLinuxNode,
node.IperfUDPClient,
ip_start=2
)
if c == 'ib':
servers = create_basic_hosts(e, 1, 'server', net, sim.I40eNIC, sim.QemuHost,
node.I40eLinuxNode, node.IperfUDPServer)
clients = create_basic_hosts(e, num_client, 'client', net, sim.I40eNIC, sim.QemuHost,
node.I40eLinuxNode, node.IperfUDPClient, ip_start = 2)
servers = create_basic_hosts(
e,
1,
'server',
net,
sim.I40eNIC,
sim.QemuHost,
node.I40eLinuxNode,
node.IperfUDPServer
)
clients = create_basic_hosts(
e,
num_client,
'client',
net,
sim.I40eNIC,
sim.QemuHost,
node.I40eLinuxNode,
node.IperfUDPClient,
ip_start=2
)
for cl in clients:
cl.wait = True
cl.node_config.app.server_ip = servers[0].node_config.ip
......@@ -89,4 +135,3 @@ for n in kinds_of_net:
print(e.name)
experiments.append(e)
experiments/pyexps/qemu_udp_single.py
View file @ 7549fa9e
......@@ -20,11 +20,11 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.nodeconfig as node
import simbricks.simulators as sim
from simbricks.simulator_utils import create_basic_hosts
import simbricks.experiments as exp
# iperf TCP_single test
# naming convention following host-nic-net-app
......@@ -34,7 +34,7 @@ from simbricks.simulator_utils import create_basic_hosts
# app: UDPs
kinds_of_host = ['qemu']
kinds_of_nic = ['cv','cb','ib']
kinds_of_nic = ['cv', 'cb', 'ib']
kinds_of_net = ['wire', 'switch', 'dumbbell', 'bridge', 'tofino']
kinds_of_app = ['UDPs']
......@@ -55,33 +55,80 @@ for n in kinds_of_net:
if n == 'tofino':
net_class = sim.TofinoNet
# set nic sim
for c in kinds_of_nic:
net = net_class()
e = exp.Experiment('qemu-' + c + '-' + n + '-' + 'UDPs')
e = exp.Experiment('qemu-' + c + '-' + n + '-' + 'UDPs')
e.add_network(net)
if c == 'cv':
servers = create_basic_hosts(e, 1, 'server', net, sim.CorundumVerilatorNIC, sim.QemuHost,
node.CorundumLinuxNode, node.IperfUDPServer)
clients = create_basic_hosts(e, 1, 'client', net, sim.CorundumVerilatorNIC, sim.QemuHost,
node.CorundumLinuxNode, node.IperfUDPClient, ip_start = 2)
servers = create_basic_hosts(
e,
1,
'server',
net,
sim.CorundumVerilatorNIC,
sim.QemuHost,
node.CorundumLinuxNode,
node.IperfUDPServer
)
clients = create_basic_hosts(
e,
1,
'client',
net,
sim.CorundumVerilatorNIC,
sim.QemuHost,
node.CorundumLinuxNode,
node.IperfUDPClient,
ip_start=2
)
if c == 'cb':
servers = create_basic_hosts(e, 1, 'server', net, sim.CorundumBMNIC, sim.QemuHost,
node.CorundumLinuxNode, node.IperfUDPServer)
clients = create_basic_hosts(e, 1, 'client', net, sim.CorundumBMNIC, sim.QemuHost,
node.CorundumLinuxNode, node.IperfUDPClient, ip_start = 2)
servers = create_basic_hosts(
e,
1,
'server',
net,
sim.CorundumBMNIC,
sim.QemuHost,
node.CorundumLinuxNode,
node.IperfUDPServer
)
clients = create_basic_hosts(
e,
1,
'client',
net,
sim.CorundumBMNIC,
sim.QemuHost,
node.CorundumLinuxNode,
node.IperfUDPClient,
ip_start=2
)
if c == 'ib':
servers = create_basic_hosts(e, 1, 'server', net, sim.I40eNIC, sim.QemuHost,
node.I40eLinuxNode, node.IperfUDPServer)
clients = create_basic_hosts(e, 1, 'client', net, sim.I40eNIC, sim.QemuHost,
node.I40eLinuxNode, node.IperfUDPClient, ip_start = 2)
servers = create_basic_hosts(
e,
1,
'server',
net,
sim.I40eNIC,
sim.QemuHost,
node.I40eLinuxNode,
node.IperfUDPServer
)
clients = create_basic_hosts(
e,
1,
'client',
net,
sim.I40eNIC,
sim.QemuHost,
node.I40eLinuxNode,
node.IperfUDPClient,
ip_start=2
)
clients[0].wait = True
clients[0].node_config.app.server_ip = servers[0].node_config.ip
......@@ -89,4 +136,3 @@ for n in kinds_of_net:
print(e.name)
experiments.append(e)
experiments/pyexps/rpc_singlecore.py
View file @ 7549fa9e
......@@ -20,11 +20,11 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.nodeconfig as node
import simbricks.simulators as sim
from simbricks.simulator_utils import create_basic_hosts
import simbricks.experiments as exp
msg_sizes = [64, 1024, 8092]
stacks = ['mtcp', 'tas', 'linux']
......@@ -32,38 +32,49 @@ num_clients = 1
experiments = []
for msg_size in msg_sizes:
for stack in stacks:
e = exp.Experiment('qemu-ib-switch-rpc-%s-1t-1fpc-%db-0mpc' % (stack,msg_size))
net = sim.SwitchNet()
e.add_network(net)
if stack == 'tas':
n = node.TASNode
elif stack == 'mtcp':
n = node.MtcpNode
else:
n = node.I40eLinuxNode
for stack in stacks:
e = exp.Experiment(
'qemu-ib-switch-rpc-%s-1t-1fpc-%db-0mpc' % (stack, msg_size)
)
net = sim.SwitchNet()
e.add_network(net)
servers = create_basic_hosts(e, 1, 'server', net, sim.I40eNIC, sim.QemuHost,
n, node.RPCServer)
if stack == 'tas':
n = node.TASNode
elif stack == 'mtcp':
n = node.MtcpNode
else:
n = node.I40eLinuxNode
clients = create_basic_hosts(e, num_clients, 'client', net, sim.I40eNIC,
sim.QemuHost, n, node.RPCClient, ip_start = 2)
servers = create_basic_hosts(
e, 1, 'server', net, sim.I40eNIC, sim.QemuHost, n, node.RPCServer
)
for h in servers + clients:
h.node_config.cores = 1 if stack != 'tas' else 3
h.node_config.fp_cores = 1
h.node_config.app.threads = 1
h.node_config.app.max_bytes = msg_size
clients = create_basic_hosts(
e,
num_clients,
'client',
net,
sim.I40eNIC,
sim.QemuHost,
n,
node.RPCClient,
ip_start=2
)
if stack == 'linux':
h.node_config.disk_image = 'tas'
for h in servers + clients:
h.node_config.cores = 1 if stack != 'tas' else 3
h.node_config.fp_cores = 1
h.node_config.app.threads = 1
h.node_config.app.max_bytes = msg_size
servers[0].sleep = 5
if stack == 'linux':
h.node_config.disk_image = 'tas'
for c in clients:
c.wait = True
c.node_config.app.server_ip = servers[0].node_config.ip
servers[0].sleep = 5
experiments.append(e)
for c in clients:
c.wait = True
c.node_config.app.server_ip = servers[0].node_config.ip
experiments.append(e)
experiments/pyexps/scalability.py
View file @ 7549fa9e
......@@ -20,11 +20,12 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.nodeconfig as node
import simbricks.simulators as sim
from simbricks.simulator_utils import create_basic_hosts
import simbricks.experiments as exp
host_configs = ['bm', 'cycle']
n_clients = [1, 2, 4, 8]
target_bandwidth = 100
......@@ -50,15 +51,31 @@ for host_config in host_configs:
e.add_network(net)
servers = create_basic_hosts(e, 1, 'server', net, nic_class, host_class,
nc_class, node.IperfUDPServer)
servers = create_basic_hosts(
e,
1,
'server',
net,
nic_class,
host_class,
nc_class,
node.IperfUDPServer
)
clients = create_basic_hosts(e, nc, 'client', net, nic_class, host_class,
nc_class, node.IperfUDPClient)
clients = create_basic_hosts(
e,
nc,
'client',
net,
nic_class,
host_class,
nc_class,
node.IperfUDPClient
)
for c in clients:
c.wait = True
c.node_config.app.server_ip = servers[0].node_config.ip
c.node_config.app.rate = str(target_bandwidth/nc)+'m'
c.node_config.app.rate = str(target_bandwidth / nc) + 'm'
experiments.append(e)
experiments/run.py
View file @ 7549fa9e
......@@ -21,95 +21,199 @@
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import argparse
import sys
import os
import fnmatch
import importlib
import importlib.util
import json
import os
import pickle
import fnmatch
import sys
import typing as tp
import simbricks.exectools as exectools
from simbricks.runtime.common import *
from simbricks.runtime.distributed import *
from simbricks.runtime.local import *
from simbricks.runtime.slurm import *
import simbricks.exectools as exectools
import simbricks.experiments as exp
def mkdir_if_not_exists(path):
if not os.path.exists(path):
os.mkdir(path)
parser = argparse.ArgumentParser()
parser.add_argument('experiments', metavar='EXP', type=str, nargs='+',
help='An experiment file to run')
parser.add_argument('--list', action='store_const', const=True, default=False,
help='Only list available experiment names')
parser.add_argument('--filter', metavar='PATTERN', type=str, nargs='+',
help='Pattern to match experiment names against')
parser.add_argument('--pickled', action='store_const', const=True,
default=False,
help='Read exp files as pickled runs instead of exp.py files')
parser.add_argument('--runs', metavar='N', type=int, default=1,
help='Number of repetition for each experiment')
parser.add_argument('--firstrun', metavar='N', type=int, default=1,
help='ID for first run')
parser.add_argument('--force', action='store_const', const=True, default=False,
help='Run experiments even if output already exists')
parser.add_argument('--verbose', action='store_const', const=True,
default=False,
help='Verbose output')
parser.add_argument('--pcap', action='store_const', const=True, default=False,
help='Dump pcap file (if supported by simulator)')
parser.add_argument(
'experiments',
metavar='EXP',
type=str,
nargs='+',
help='An experiment file to run'
)
parser.add_argument(
'--list',
action='store_const',
const=True,
default=False,
help='Only list available experiment names'
)
parser.add_argument(
'--filter',
metavar='PATTERN',
type=str,
nargs='+',
help='Pattern to match experiment names against'
)
parser.add_argument(
'--pickled',
action='store_const',
const=True,
default=False,
help='Read exp files as pickled runs instead of exp.py files'
)
parser.add_argument(
'--runs',
metavar='N',
type=int,
default=1,
help='Number of repetition for each experiment'
)
parser.add_argument(
'--firstrun', metavar='N', type=int, default=1, help='ID for first run'
)
parser.add_argument(
'--force',
action='store_const',
const=True,
default=False,
help='Run experiments even if output already exists'
)
parser.add_argument(
'--verbose',
action='store_const',
const=True,
default=False,
help='Verbose output'
)
parser.add_argument(
'--pcap',
action='store_const',
const=True,
default=False,
help='Dump pcap file (if supported by simulator)'
)
g_env = parser.add_argument_group('Environment')
g_env.add_argument('--repo', metavar='DIR', type=str,
default='..', help='Repo directory')
g_env.add_argument('--workdir', metavar='DIR', type=str,
default='./out/', help='Work directory base')
g_env.add_argument('--outdir', metavar='DIR', type=str,
default='./out/', help='Output directory base')
g_env.add_argument('--cpdir', metavar='DIR', type=str,
default='./out/', help='Checkpoint directory base')
g_env.add_argument('--hosts', metavar='JSON_FILE', type=str,
default=None, help='List of hosts to use (json)')
g_env.add_argument('--shmdir', metavar='DIR', type=str,
default=None, help='Shared memory directory base (workdir if not set)')
g_env.add_argument(
'--repo', metavar='DIR', type=str, default='..', help='Repo directory'
)
g_env.add_argument(
'--workdir',
metavar='DIR',
type=str,
default='./out/',
help='Work directory base'
)
g_env.add_argument(
'--outdir',
metavar='DIR',
type=str,
default='./out/',
help='Output directory base'
)
g_env.add_argument(
'--cpdir',
metavar='DIR',
type=str,
default='./out/',
help='Checkpoint directory base'
)
g_env.add_argument(
'--hosts',
metavar='JSON_FILE',
type=str,
default=None,
help='List of hosts to use (json)'
)
g_env.add_argument(
'--shmdir',
metavar='DIR',
type=str,
default=None,
help='Shared memory directory base (workdir if not set)'
)
g_par = parser.add_argument_group('Parallel Runtime')
g_par.add_argument('--parallel', dest='runtime', action='store_const',
const='parallel', default='sequential',
help='Use parallel instead of sequential runtime')
g_par.add_argument('--cores', metavar='N', type=int,
default=len(os.sched_getaffinity(0)),
help='Number of cores to use for parallel runs')
g_par.add_argument('--mem', metavar='N', type=int, default=None,
help='Memory limit for parallel runs (in MB)')
g_par.add_argument(
'--parallel',
dest='runtime',
action='store_const',
const='parallel',
default='sequential',
help='Use parallel instead of sequential runtime'
)
g_par.add_argument(
'--cores',
metavar='N',
type=int,
default=len(os.sched_getaffinity(0)),
help='Number of cores to use for parallel runs'
)
g_par.add_argument(
'--mem',
metavar='N',
type=int,
default=None,
help='Memory limit for parallel runs (in MB)'
)
g_slurm = parser.add_argument_group('Slurm Runtime')
g_slurm.add_argument('--slurm', dest='runtime', action='store_const',
const='slurm', default='sequential',
help='Use slurm instead of sequential runtime')
g_slurm.add_argument('--slurmdir', metavar='DIR', type=str,
default='./slurm/', help='Slurm communication directory')
g_slurm.add_argument(
'--slurm',
dest='runtime',
action='store_const',
const='slurm',
default='sequential',
help='Use slurm instead of sequential runtime'
)
g_slurm.add_argument(
'--slurmdir',
metavar='DIR',
type=str,
default='./slurm/',
help='Slurm communication directory'
)
g_dist = parser.add_argument_group('Distributed Runtime')
g_dist.add_argument('--dist', dest='runtime', action='store_const',
const='dist', default='sequential',
help='Use sequential distributed runtime instead of local')
g_dist.add_argument('--auto-dist', action='store_const', const=True,
default=False,
help='Automatically distribute non-distributed experiments')
g_dist.add_argument('--proxy-type', metavar='TYPE', type=str,
default='sockets',
help='Proxy type to use (sockets,rdma) for auto distribution')
g_dist.add_argument(
'--dist',
dest='runtime',
action='store_const',
const='dist',
default='sequential',
help='Use sequential distributed runtime instead of local'
)
g_dist.add_argument(
'--auto-dist',
action='store_const',
const=True,
default=False,
help='Automatically distribute non-distributed experiments'
)
g_dist.add_argument(
'--proxy-type',
metavar='TYPE',
type=str,
default='sockets',
help='Proxy type to use (sockets,rdma) for auto distribution'
)
args = parser.parse_args()
def load_executors(path):
""" Load hosts list from json file and return list of executors. """
"""Load hosts list from json file and return list of executors."""
with open(path, 'r') as f:
hosts = json.load(f)
......@@ -129,21 +233,27 @@ def load_executors(path):
exs.append(ex)
return exs
if args.hosts is None:
executors = [exectools.LocalExecutor()]
else:
executors = load_executors(args.hosts)
def warn_multi_exec():
if len(executors) > 1:
print('Warning: multiple hosts specified, only using first one for now',
file=sys.stderr)
print(
'Warning: multiple hosts specified, only using first one for now',
file=sys.stderr
)
# initialize runtime
if args.runtime == 'parallel':
warn_multi_exec()
rt = LocalParallelRuntime(cores=args.cores, mem=args.mem,
verbose=args.verbose, exec=executors[0])
rt = LocalParallelRuntime(
cores=args.cores, mem=args.mem, verbose=args.verbose, exec=executors[0]
)
elif args.runtime == 'slurm':
rt = SlurmRuntime(args.slurmdir, args, verbose=args.verbose)
elif args.runtime == 'dist':
......@@ -154,8 +264,12 @@ else:
def add_exp(
e: exp.Experiment, run: int, prereq: tp.Optional[Run],
create_cp: bool, restore_cp: bool, no_simbricks: bool
e: exp.Experiment,
run: int,
prereq: tp.Optional[Run],
create_cp: bool,
restore_cp: bool,
no_simbricks: bool
):
outpath = '%s/%s-%d.json' % (args.outdir, e.name, run)
if os.path.exists(outpath) and not args.force:
......@@ -170,10 +284,10 @@ def add_exp(
env = exp.ExpEnv(args.repo, workdir, cpdir)
env.create_cp = create_cp
env.restore_cp = restore_cp
env.no_simbricks=no_simbricks
env.no_simbricks = no_simbricks
env.pcap_file = ''
if args.pcap:
env.pcap_file = workdir+'/pcap'
env.pcap_file = workdir + '/pcap'
if args.shmdir is not None:
env.shm_base = os.path.abspath(shmdir)
......@@ -181,6 +295,7 @@ def add_exp(
rt.add_run(run)
return run
# load experiments
if not args.pickled:
# default: load python modules with experiments
......@@ -213,9 +328,9 @@ if not args.pickled:
# if this is an experiment with a checkpoint we might have to create it
if e.no_simbricks:
no_simbricks = True
no_simbricks = True
else:
no_simbricks = False
no_simbricks = False
if e.checkpoint:
prereq = add_exp(e, 0, None, True, False, no_simbricks)
else:
......
experiments/simbricks/exectools.py
View file @ 7549fa9e
......@@ -21,15 +21,17 @@
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import asyncio
from asyncio.subprocess import Process
import os
import pathlib
import re
import shlex
import shutil
import signal
from asyncio.subprocess import Process
class HostConfig(object):
def __init__(self, name, ip, mac, sudopwd, other={}):
self.name = name
self.ip = ip
......@@ -38,6 +40,7 @@ class HostConfig(object):
self.sudo_pwd = sudopwd
self.other = other.copy()
class Component(object):
proc: Process
terminate_future: asyncio.Task[int]
......@@ -92,9 +95,12 @@ class Component(object):
return
async def _waiter(self):
out_handlers = asyncio.ensure_future(asyncio.wait([
self._read_stream(self.proc.stdout, self._consume_out),
self._read_stream(self.proc.stderr, self._consume_err)]))
out_handlers = asyncio.ensure_future(
asyncio.wait([
self._read_stream(self.proc.stdout, self._consume_out),
self._read_stream(self.proc.stderr, self._consume_err)
])
)
rc = await self.proc.wait()
await out_handlers
await self.terminated(rc)
......@@ -111,11 +117,12 @@ class Component(object):
else:
stdin = None
self.proc = await asyncio.create_subprocess_exec(*self.cmd_parts,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
stdin=stdin,
)
self.proc = await asyncio.create_subprocess_exec(
*self.cmd_parts,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
stdin=stdin,
)
self.terminate_future = asyncio.ensure_future(self._waiter())
await self.started()
......@@ -137,7 +144,7 @@ class Component(object):
async def int_term_kill(self, delay=5):
await self.interrupt()
_,pending = await asyncio.wait([self.terminate_future], timeout=delay)
_, pending = await asyncio.wait([self.terminate_future], timeout=delay)
if len(pending) != 0:
print('terminating')
await self.terminate()
......@@ -161,8 +168,10 @@ class Component(object):
class SimpleComponent(Component):
def __init__(self, label, cmd_parts, verbose=True, canfail=False,
*args, **kwargs):
def __init__(
self, label, cmd_parts, verbose=True, canfail=False, *args, **kwargs
):
self.label = label
self.verbose = verbose
self.canfail = canfail
......@@ -185,10 +194,20 @@ class SimpleComponent(Component):
if not self.canfail and rc != 0:
raise Exception('Command Failed: ' + str(self.cmd_parts))
class SimpleRemoteComponent(SimpleComponent):
pid_fut: asyncio.Future
def __init__(self, host_name, label, cmd_parts, cwd=None, ssh_extra_args=[], *args, **kwargs):
def __init__(
self,
host_name,
label,
cmd_parts,
cwd=None,
ssh_extra_args=[],
*args,
**kwargs
):
self.host_name = host_name
self.extra_flags = ssh_extra_args
# add a wrapper to print the PID
......@@ -209,25 +228,23 @@ class SimpleRemoteComponent(SimpleComponent):
super().__init__(label, parts, *args, **kwargs)
def _ssh_cmd(self, parts):
""" SSH invocation of command for this host. """
"""SSH invocation of command for this host."""
return [
'ssh',
'-o',
'UserKnownHostsFile=/dev/null',
'-o',
'StrictHostKeyChecking=no'
] + self.extra_flags + [
self.host_name,
'--'] + parts
] + self.extra_flags + [self.host_name, '--'] + parts
async def start(self):
""" Start this command (includes waiting for its pid. """
"""Start this command (includes waiting for its pid."""
self.pid_fut = asyncio.get_running_loop().create_future()
await super().start()
await self.pid_fut
async def process_out(self, lines, eof):
""" Scans output and set PID future once PID line found. """
"""Scans output and set PID future once PID line found."""
if not self.pid_fut.done():
newlines = []
pid_re = re.compile(r'^PID\s+(\d+)\s*$')
......@@ -247,9 +264,10 @@ class SimpleRemoteComponent(SimpleComponent):
await super().process_out(lines, eof)
async def _kill_cmd(self, sig):
""" Send signal to command by running ssh kill -$sig $PID. """
cmd_parts = self._ssh_cmd(['kill', '-' + sig,
str(self.pid_fut.result())])
"""Send signal to command by running ssh kill -$sig $PID."""
cmd_parts = self._ssh_cmd([
'kill', '-' + sig, str(self.pid_fut.result())
])
proc = await asyncio.create_subprocess_exec(*cmd_parts)
await proc.wait()
......@@ -262,30 +280,32 @@ class SimpleRemoteComponent(SimpleComponent):
async def kill(self):
await self._kill_cmd('KILL')
class Executor(object):
ip = None
def create_component(self, label, parts, **kwargs) -> SimpleComponent:
raise NotImplementedError("Please Implement this method")
raise NotImplementedError('Please Implement this method')
async def await_file(self, path, delay=0.05, verbose=False):
raise NotImplementedError("Please Implement this method")
raise NotImplementedError('Please Implement this method')
async def send_file(self, path, verbose=False):
raise NotImplementedError("Please Implement this method")
raise NotImplementedError('Please Implement this method')
async def mkdir(self, path, verbose=False):
raise NotImplementedError("Please Implement this method")
raise NotImplementedError('Please Implement this method')
async def rmtree(self, path, verbose=False):
raise NotImplementedError("Please Implement this method")
raise NotImplementedError('Please Implement this method')
# runs the list of commands as strings sequentially
async def run_cmdlist(self, label, cmds, verbose=True, host=None):
i = 0
for cmd in cmds:
cmdC = self.create_component(label + '.' + str(i), shlex.split(cmd),
verbose=verbose)
cmdC = self.create_component(
label + '.' + str(i), shlex.split(cmd), verbose=verbose
)
await cmdC.start()
await cmdC.wait()
......@@ -295,7 +315,9 @@ class Executor(object):
xs.append(self.await_file(p, *args, **kwargs))
await asyncio.wait(xs)
class LocalExecutor(Executor):
def create_component(self, label, parts, **kwargs):
return SimpleComponent(label, parts, **kwargs)
......@@ -322,7 +344,9 @@ class LocalExecutor(Executor):
elif os.path.exists(path):
os.unlink(path)
class RemoteExecutor(Executor):
def __init__(self, host_name, workdir):
self.host_name = host_name
self.cwd = workdir
......@@ -330,21 +354,33 @@ class RemoteExecutor(Executor):
self.scp_extra_args = []
def create_component(self, label, parts, **kwargs):
return SimpleRemoteComponent(self.host_name, label, parts,
cwd=self.cwd, ssh_extra_args=self.ssh_extra_args, **kwargs)
return SimpleRemoteComponent(
self.host_name,
label,
parts,
cwd=self.cwd,
ssh_extra_args=self.ssh_extra_args,
**kwargs
)
async def await_file(self, path, delay=0.05, verbose=False, timeout=30):
if verbose:
print('%s.await_file(%s) started' % (self.host_name, path))
to_its = timeout / delay
loop_cmd = ('i=0 ; while [ ! -e %s ] ; do '
'if [ $i -ge %u ] ; then exit 1 ; fi ; '
'sleep %f ; '
'i=$(($i+1)) ; done; exit 0') % (path, to_its, delay)
loop_cmd = (
'i=0 ; while [ ! -e %s ] ; do '
'if [ $i -ge %u ] ; then exit 1 ; fi ; '
'sleep %f ; '
'i=$(($i+1)) ; done; exit 0'
) % (path, to_its, delay)
parts = ['/bin/sh', '-c', loop_cmd]
sc = self.create_component("%s.await_file('%s')" % (self.host_name,
path), parts, canfail=False, verbose=verbose)
sc = self.create_component(
"%s.await_file('%s')" % (self.host_name, path),
parts,
canfail=False,
verbose=verbose
)
await sc.start()
await sc.wait()
......@@ -357,22 +393,30 @@ class RemoteExecutor(Executor):
'UserKnownHostsFile=/dev/null',
'-o',
'StrictHostKeyChecking=no'
] + self.scp_extra_args + [
path,
'%s:%s' % (self.host_name, path)]
sc = SimpleComponent("%s.send_file('%s')" % (
self.host_name, path), parts, canfail=False, verbose=verbose)
] + self.scp_extra_args + [path, '%s:%s' % (self.host_name, path)]
sc = SimpleComponent(
"%s.send_file('%s')" % (self.host_name, path),
parts,
canfail=False,
verbose=verbose
)
await sc.start()
await sc.wait()
async def mkdir(self, path, verbose=False):
sc = self.create_component("%s.mkdir('%s')" % (self.host_name, path),
['mkdir', '-p', path], canfail=False, verbose=verbose)
sc = self.create_component(
"%s.mkdir('%s')" % (self.host_name, path), ['mkdir', '-p', path],
canfail=False,
verbose=verbose
)
await sc.start()
await sc.wait()
async def rmtree(self, path, verbose=False):
sc = self.create_component("%s.rmtree('%s')" % (self.host_name, path),
['rm', '-rf', path], canfail=False, verbose=verbose)
sc = self.create_component(
"%s.rmtree('%s')" % (self.host_name, path), ['rm', '-rf', path],
canfail=False,
verbose=verbose
)
await sc.start()
await sc.wait()
experiments/simbricks/experiment/experiment_output.py
View file @ 7549fa9e
......@@ -20,7 +20,6 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import json
import time
......@@ -59,4 +58,4 @@ class ExpOutput(object):
def loads(self, json_s):
for k, v in json.loads(json_s).items():
self.__dict__[k] = v
\ No newline at end of file
self.__dict__[k] = v
experiments/simbricks/experiments.py
View file @ 7549fa9e
This diff is collapsed.
experiments/simbricks/nodeconfig.py
View file @ 7549fa9e
This diff is collapsed.
experiments/simbricks/proxy.py
View file @ 7549fa9e
This diff is collapsed.
experiments/simbricks/runtime/__init__.py
View file @ 7549fa9e
......@@ -20,7 +20,7 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
from simbricks.runtime.common import (Run, Runtime)
from simbricks.runtime.local import (LocalSimpleRuntime, LocalParallelRuntime)
from simbricks.runtime.common import Run, Runtime
from simbricks.runtime.distributed import DistributedSimpleRuntime, auto_dist
from simbricks.runtime.local import LocalParallelRuntime, LocalSimpleRuntime
from simbricks.runtime.slurm import SlurmRuntime
from simbricks.runtime.distributed import (DistributedSimpleRuntime, auto_dist)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment