Skip to content

GitLab

Commit 7549fa9e authored by Jonas Kaufmann's avatar Jonas Kaufmann Committed by Antoine Kaufmann
Browse files

pre-commit run -a

parent aac98df8
Show whitespace changes
Inline Side-by-side
Showing with 909 additions and 546 deletions
experiments/pyexps/ae/nopaxos.py
View file @ 7549fa9e
......@@ -20,11 +20,12 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.nodeconfig as node
import simbricks.simulators as sim
from simbricks.simulator_utils import create_basic_hosts
import simbricks.experiments as exp
host_configs = ['qt']
seq_configs = ['swseq', 'ehseq', 'tofino']
nic_configs = ['ib']
......@@ -41,7 +42,10 @@ for proto_config in proto_configs:
for host_config in host_configs:
for seq_config in seq_configs:
for nic_config in nic_configs:
e = exp.Experiment(proto_config + '-' + host_config + '-' + nic_config + '-' + seq_config + f'-{num_c}')
e = exp.Experiment(
proto_config + '-' + host_config + '-' + nic_config +
'-' + seq_config + f'-{num_c}'
)
if seq_config == 'tofino':
net = sim.TofinoNet()
else:
......@@ -58,10 +62,12 @@ for proto_config in proto_configs:
host_class = sim.Gem5Host
e.checkpoint = True
elif host_config == 'qt':
def qemu_timing():
h = sim.QemuHost()
h.sync = True
return h
host_class = qemu_timing
else:
raise NameError(host_config)
......@@ -79,7 +85,6 @@ for proto_config in proto_configs:
else:
raise NameError(nic_config)
# app
if proto_config == 'vr':
replica_class = node.VRReplica
......@@ -92,25 +97,53 @@ for proto_config in proto_configs:
# endhost sequencer
if seq_config == 'ehseq' and proto_config == 'nopaxos':
sequencer = create_basic_hosts(e, 1, 'sequencer', net, nic_class,
host_class, nc_class, node.NOPaxosSequencer, ip_start = 100)
sequencer = create_basic_hosts(
e,
1,
'sequencer',
net,
nic_class,
host_class,
nc_class,
node.NOPaxosSequencer,
ip_start=100
)
sequencer[0].node_config.disk_image = 'nopaxos'
sequencer[0].pcidevs[0].sync_period = sync_period
sequencer[0].sync_period = sync_period
replicas = create_basic_hosts(e, 3, 'replica', net, nic_class,
host_class, nc_class, replica_class)
replicas = create_basic_hosts(
e,
3,
'replica',
net,
nic_class,
host_class,
nc_class,
replica_class
)
for i in range(len(replicas)):
replicas[i].node_config.app.index = i
replicas[i].node_config.disk_image = 'nopaxos'
replicas[i].pcidevs[0].sync_period = sync_period
replicas[i].sync_period = sync_period
clients = create_basic_hosts(e, num_c, 'client', net, nic_class,
host_class, nc_class, client_class, ip_start = 4)
clients = create_basic_hosts(
e,
num_c,
'client',
net,
nic_class,
host_class,
nc_class,
client_class,
ip_start=4
)
for c in clients:
c.node_config.app.server_ips = ['10.0.0.1', '10.0.0.2', '10.0.0.3']
c.node_config.app.server_ips = [
'10.0.0.1', '10.0.0.2', '10.0.0.3'
]
if seq_config == 'ehseq':
c.node_config.app.server_ips.append('10.0.0.100')
c.node_config.app.use_ehseq = True
......
experiments/pyexps/ae/t1_netperf.py
View file @ 7549fa9e
......@@ -39,12 +39,12 @@
# $: python3 run.py pyexps/ae/t1_combination.py --filter nf-* --verbose
########################################################################
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.nodeconfig as node
import simbricks.simulators as sim
from simbricks.simulator_utils import create_basic_hosts
import simbricks.experiments as exp
host_types = ['qemu', 'gt', 'qt']
nic_types = ['ib', 'cb', 'cv']
net_types = ['sw', 'ns3']
......@@ -55,7 +55,9 @@ experiments = []
for host_type in host_types:
for nic_type in nic_types:
for net_type in net_types:
e = exp.Experiment('nf-' + host_type + '-' + net_type + '-' + nic_type)
e = exp.Experiment(
'nf-' + host_type + '-' + net_type + '-' + nic_type
)
# network
if net_type == 'sw':
......@@ -70,10 +72,12 @@ for host_type in host_types:
if host_type == 'qemu':
host_class = sim.QemuHost
elif host_type == 'qt':
def qemu_timing():
h = sim.QemuHost()
h.sync = True
return h
host_class = qemu_timing
elif host_type == 'gt':
host_class = sim.Gem5Host
......@@ -95,11 +99,28 @@ for host_type in host_types:
raise NameError(nic_type)
# create servers and clients
servers = create_basic_hosts(e, 1, 'server', net, nic_class, host_class,
nc_class, node.NetperfServer)
servers = create_basic_hosts(
e,
1,
'server',
net,
nic_class,
host_class,
nc_class,
node.NetperfServer
)
clients = create_basic_hosts(e, 1, 'client', net, nic_class, host_class,
nc_class, node.NetperfClient, ip_start = 2)
clients = create_basic_hosts(
e,
1,
'client',
net,
nic_class,
host_class,
nc_class,
node.NetperfClient,
ip_start=2
)
for c in clients:
c.wait = True
......
experiments/pyexps/ae/utils/iperf.py
View file @ 7549fa9e
......@@ -20,16 +20,19 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import fnmatch
import glob
import itertools
import json
import os
import fnmatch
import re
import itertools
import sys
def parse_iperf_run(data, skip=1, use=8):
tp_pat = re.compile(r'\[ *\d*\] *([0-9\.]*)- *([0-9\.]*) sec.*Bytes *([0-9\.]*) ([GM])bits.*')
tp_pat = re.compile(
r'\[ *\d*\] *([0-9\.]*)- *([0-9\.]*) sec.*Bytes *([0-9\.]*) ([GM])bits.*'
)
tps_time = {}
for hn in fnmatch.filter(data['sims'].keys(), 'host.client.*'):
sim = data['sims'][hn]
......@@ -50,7 +53,7 @@ def parse_iperf_run(data, skip=1, use=8):
if m.group(4) == 'G':
tps_time[time].append(float(m.group(3)))
elif m.group(4) == 'M':
m_tps = float(m.group(3))/1000
m_tps = float(m.group(3)) / 1000
tps_time[time].append(m_tps)
tps = []
......@@ -58,11 +61,11 @@ def parse_iperf_run(data, skip=1, use=8):
x = sum(tps_time[t])
tps.append(x)
if len(tps) == 0:
return None
return sum(tps) / len(tps)
def parse_iperf(basename, skip=1, use=8):
runs = []
for path in glob.glob(basename + '-*.json'):
......@@ -79,6 +82,7 @@ def parse_iperf(basename, skip=1, use=8):
if not runs:
return {'avg': None, 'min': None, 'max': None}
else:
return {'avg': sum(runs) / len(runs), 'min': min(runs),
'max': max(runs)}
return {
'avg': sum(runs) / len(runs), 'min': min(runs), 'max': max(runs)
}
result = {}
experiments/pyexps/ae/utils/netperf.py
View file @ 7549fa9e
......@@ -21,8 +21,9 @@
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import json
import re
import os
import re
def parse_netperf_run(path):
ret = {}
......@@ -50,8 +51,6 @@ def parse_netperf_run(path):
m = tp_pat.match(tp_line)
ret['throughput'] = float(m.group(1))
lath_pat = re.compile(r'\s*Mean Latency.*')
start = None
i = 0
......@@ -64,7 +63,9 @@ def parse_netperf_run(path):
if start is not None:
lat_line = lines[start + 1]
lat_pat = re.compile(r'\s*([-0-9\.]*),([-0-9\.]*),([-0-9\.]*),([-0-9\.]*).*')
lat_pat = re.compile(
r'\s*([-0-9\.]*),([-0-9\.]*),([-0-9\.]*),([-0-9\.]*).*'
)
m = lat_pat.match(lat_line)
ret['latenyMean'] = float(m.group(1))
ret['latenyTail'] = float(m.group(4))
......
experiments/pyexps/ae/utils/parse_nopaxos.py
View file @ 7549fa9e
......@@ -21,8 +21,9 @@
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import json
import re
import os
import re
def parse_nopaxos_run(num_c, path):
......@@ -45,7 +46,7 @@ def parse_nopaxos_run(num_c, path):
sim_name = f'host.client.{i}'
# in this host log stdout
for j in log["sims"][sim_name]["stdout"]:
for j in log['sims'][sim_name]['stdout']:
m_t = tp_pat.match(j)
m_l = lat_pat.match(j)
if m_l:
......@@ -53,8 +54,7 @@ def parse_nopaxos_run(num_c, path):
if m_t:
total_tput += int(m_t.group(2))
avg_lat = total_lat/num_c
avg_lat = total_lat / num_c
ret['throughput'] = total_tput
ret['latency'] = int(avg_lat)
......
experiments/pyexps/dctcp.py
View file @ 7549fa9e
......@@ -20,11 +20,11 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.nodeconfig as node
import simbricks.simulators as sim
from simbricks.simulator_utils import create_dctcp_hosts
import simbricks.experiments as exp
# iperf TCP_multi_client test
# naming convention following host-nic-net-app
......@@ -33,8 +33,8 @@ from simbricks.simulator_utils import create_dctcp_hosts
# net: switch/dumbbell/bridge
# app: DCTCPm
types_of_host = ['qemu', 'qt','gt', 'gO3']
types_of_nic = ['cv','cb','ib']
types_of_host = ['qemu', 'qt', 'gt', 'gO3']
types_of_nic = ['cv', 'cb', 'ib']
types_of_net = ['dumbbell']
types_of_app = ['DCTCPm']
types_of_mtu = [1500, 4000, 9000]
......@@ -65,7 +65,10 @@ for mtu in types_of_mtu:
net = net_class()
net.opt = link_rate_opt + link_latency_opt + f'--EcnTh={k_val}'
e = exp.Experiment( h + '-' + c + '-' + 'dumbbell' + '-' + 'DCTCPm' + f'{k_val}' + f'-{mtu}')
e = exp.Experiment(
h + '-' + c + '-' + 'dumbbell' + '-' + 'DCTCPm' +
f'{k_val}' + f'-{mtu}'
)
e.add_network(net)
freq = cpu_freq
......@@ -74,24 +77,30 @@ for mtu in types_of_mtu:
host_class = sim.QemuHost
elif h == 'qt':
freq = cpu_freq_qemu
def qemu_timing():
h = sim.QemuHost()
h.sync = True
return h
host_class = qemu_timing
elif h == 'gt':
def gem5_timing():
h = sim.Gem5Host()
#h.sys_clock = sys_clock
return h
host_class = gem5_timing
e.checkpoint = True
elif h == 'gO3':
def gem5_o3():
h = sim.Gem5Host()
h.cpu_type = 'DerivO3CPU'
h.sys_clock = sys_clock
return h
host_class = gem5_o3
e.checkpoint = True
else:
......@@ -110,12 +119,31 @@ for mtu in types_of_mtu:
else:
raise NameError(c)
servers = create_dctcp_hosts(e, num_pairs, 'server', net, nic_class, host_class,
nc_class, node.DctcpServer, freq, mtu)
clients = create_dctcp_hosts(e, num_pairs, 'client', net, nic_class, host_class,
nc_class, node.DctcpClient, freq, mtu, ip_start=num_pairs+1)
servers = create_dctcp_hosts(
e,
num_pairs,
'server',
net,
nic_class,
host_class,
nc_class,
node.DctcpServer,
freq,
mtu
)
clients = create_dctcp_hosts(
e,
num_pairs,
'client',
net,
nic_class,
host_class,
nc_class,
node.DctcpClient,
freq,
mtu,
ip_start=num_pairs + 1
)
i = 0
for cl in clients:
......@@ -126,9 +154,8 @@ for mtu in types_of_mtu:
# This is to prevent the simulation gets stuck when one of host exits.
# The last client waits for the output printed in other hosts, then cleanup
clients[num_pairs-1].node_config.app.is_last = True
clients[num_pairs-1].wait = True
clients[num_pairs - 1].node_config.app.is_last = True
clients[num_pairs - 1].wait = True
print(e.name)
experiments.append(e)
experiments/pyexps/dist_memcache.py
View file @ 7549fa9e
......@@ -22,12 +22,14 @@
import math
import random
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.proxy as proxy
import simbricks.nodeconfig as node
import simbricks.proxy as proxy
import simbricks.simulators as sim
from simbricks.simulator_utils import create_multinic_hosts
import simbricks.experiments as exp
host_types = ['qemu', 'gem5', 'qt']
n_nets = [1, 2, 3, 4, 8, 16, 32]
n_hosts = [2, 10, 20, 30, 35, 40, 50, 60, 70, 80]
......@@ -36,6 +38,7 @@ separate_net = True
nets_per_host = 1
def select_servers(i, j, racks, n, n_host):
nc = int(n_host / 2)
......@@ -56,6 +59,7 @@ def select_servers(i, j, racks, n, n_host):
servers_other = random.sample(all_other_servers, k=n_remote)
return servers_local + servers_other
for host_type in host_types:
for n in n_nets:
for n_host in n_hosts:
......@@ -65,16 +69,20 @@ for host_type in host_types:
if separate_net:
nh += 1
e = exp.DistributedExperiment(f'dist_memcache-{host_type}-{n}-{n_host}', nh)
e = exp.DistributedExperiment(
f'dist_memcache-{host_type}-{n}-{n_host}', nh
)
# host
if host_type == 'qemu':
host_class = sim.QemuHost
elif host_type == 'qt':
def qemu_timing():
h = sim.QemuHost()
h.sync = True
return h
host_class = qemu_timing
elif host_type == 'gem5':
host_class = sim.Gem5Host
......@@ -106,23 +114,43 @@ for host_type in host_types:
# create servers and clients
m = int(n_host / 2)
servers = create_multinic_hosts(e, m, 'server_%d' % (i,),
switch, host_class, node.I40eLinuxNode,
node.MemcachedServer, ip_start = i * n_host + 1,
ip_prefix=16)
servers = create_multinic_hosts(
e,
m,
'server_%d' % (i,),
switch,
host_class,
node.I40eLinuxNode,
node.MemcachedServer,
ip_start=i * n_host + 1,
ip_prefix=16
)
for s in servers:
e.assign_sim_host(s, h_i)
s_multisubnic = next(pcidev for pcidev in s.pcidevs if isinstance(pcidev, sim.MultiSubNIC))
s_multisubnic = next(
pcidev for pcidev in s.pcidevs
if isinstance(pcidev, sim.MultiSubNIC)
)
e.assign_sim_host(s_multisubnic.multinic, h_i)
clients = create_multinic_hosts(e, m, 'client_%d' % (i,),
switch, host_class, node.I40eLinuxNode,
node.MemcachedClient, ip_start = i * n_host + 1 + m,
ip_prefix=16)
clients = create_multinic_hosts(
e,
m,
'client_%d' % (i,),
switch,
host_class,
node.I40eLinuxNode,
node.MemcachedClient,
ip_start=i * n_host + 1 + m,
ip_prefix=16
)
for c in clients:
c.wait = True
e.assign_sim_host(c, h_i)
c_multisubnic = next(pcidev for pcidev in c.pcidevs if isinstance(pcidev, sim.MultiSubNIC))
c_multisubnic = next(
pcidev for pcidev in c.pcidevs
if isinstance(pcidev, sim.MultiSubNIC)
)
e.assign_sim_host(c_multisubnic.multinic, h_i)
racks.append((servers, clients))
......@@ -147,7 +175,7 @@ for host_type in host_types:
all_servers = []
all_clients = []
for (s,c) in racks:
for (s, c) in racks:
all_servers += s
all_clients += c
......
experiments/pyexps/dist_multinet.py
View file @ 7549fa9e
......@@ -20,12 +20,13 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.proxy as proxy
import simbricks.nodeconfig as node
import simbricks.proxy as proxy
import simbricks.simulators as sim
from simbricks.simulator_utils import create_basic_hosts
import simbricks.experiments as exp
host_types = ['qemu', 'gem5', 'qt']
n_nets = [1, 2, 3, 4]
n_clients = [1, 10, 20, 30, 40, 50]
......@@ -37,16 +38,20 @@ for host_type in host_types:
for n in n_nets:
for n_client in n_clients:
nh = n if not separate_net else n + 1
e = exp.DistributedExperiment(f'dist_multinet-{host_type}-{n}-{n_client}', nh)
e = exp.DistributedExperiment(
f'dist_multinet-{host_type}-{n}-{n_client}', nh
)
# host
if host_type == 'qemu':
host_class = sim.QemuHost
elif host_type == 'qt':
def qemu_timing():
h = sim.QemuHost()
h.sync = True
return h
host_class = qemu_timing
elif host_type == 'gem5':
host_class = sim.Gem5Host
......@@ -75,18 +80,34 @@ for host_type in host_types:
# create servers and clients
m = n_client
if i == 0 or separate_server:
servers = create_basic_hosts(e, 1, 'server_%d' % (i,),
switch, sim.I40eNIC, host_class, node.I40eLinuxNode,
node.NetperfServer, ip_start = i * (n_client + 1) + 1)
servers = create_basic_hosts(
e,
1,
'server_%d' % (i,),
switch,
sim.I40eNIC,
host_class,
node.I40eLinuxNode,
node.NetperfServer,
ip_start=i * (n_client + 1) + 1
)
if not separate_server:
m = m - 1
e.assign_sim_host(servers[0], h_i)
e.assign_sim_host(servers[0].pcidevs[0], h_i)
clients = create_basic_hosts(e, m, 'client_%d' % (i,),
switch, sim.I40eNIC, host_class, node.I40eLinuxNode,
node.NetperfClient, ip_start = i * (n_client + 1) + 2)
clients = create_basic_hosts(
e,
m,
'client_%d' % (i,),
switch,
sim.I40eNIC,
host_class,
node.I40eLinuxNode,
node.NetperfClient,
ip_start=i * (n_client + 1) + 2
)
for c in clients:
c.wait = True
......
experiments/pyexps/dist_netperf.py
View file @ 7549fa9e
......@@ -20,12 +20,13 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.proxy as proxy
import simbricks.nodeconfig as node
import simbricks.proxy as proxy
import simbricks.simulators as sim
from simbricks.simulator_utils import create_basic_hosts
import simbricks.experiments as exp
host_types = ['qemu', 'gem5', 'qt']
nic_types = ['i40e', 'cd_bm', 'cd_verilator']
n_clients = [1, 4, 8, 16, 32]
......@@ -34,7 +35,9 @@ experiments = []
for host_type in host_types:
for nic_type in nic_types:
for n in n_clients:
e = exp.DistributedExperiment(f'dist_netperf-{host_type}-{nic_type}-{n}', 2)
e = exp.DistributedExperiment(
f'dist_netperf-{host_type}-{nic_type}-{n}', 2
)
net = sim.SwitchNet()
e.add_network(net)
......@@ -44,10 +47,12 @@ for host_type in host_types:
host_class = sim.QemuHost
net.sync = False
elif host_type == 'qt':
def qemu_timing():
h = sim.QemuHost()
h.sync = True
return h
host_class = qemu_timing
elif host_type == 'gem5':
host_class = sim.Gem5Host
......@@ -69,11 +74,28 @@ for host_type in host_types:
raise NameError(nic_type)
# create servers and clients
servers = create_basic_hosts(e, 1, 'server', net, nic_class, host_class,
nc_class, node.NetperfServer)
servers = create_basic_hosts(
e,
1,
'server',
net,
nic_class,
host_class,
nc_class,
node.NetperfServer
)
clients = create_basic_hosts(e, n, 'client', net, nic_class, host_class,
nc_class, node.NetperfClient, ip_start = 2)
clients = create_basic_hosts(
e,
n,
'client',
net,
nic_class,
host_class,
nc_class,
node.NetperfClient,
ip_start=2
)
for c in clients:
c.wait = True
......
experiments/pyexps/femutest.py
View file @ 7549fa9e
......@@ -20,9 +20,10 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.nodeconfig as node
import simbricks.simulators as sim
import simbricks.experiments as exp
experiments = []
......@@ -45,7 +46,6 @@ for h in ['qk', 'gk']:
e.add_host(host)
host.wait = True
femu = sim.FEMUDev()
femu.name = 'femu0'
e.add_pcidev(femu)
......
experiments/pyexps/gem5_i40e_pair.py
View file @ 7549fa9e
......@@ -20,22 +20,40 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.nodeconfig as node
import simbricks.simulators as sim
from simbricks.simulator_utils import create_basic_hosts
import simbricks.experiments as exp
e = exp.Experiment('gem5-i40e-pair')
#e.timeout = 5 * 60
e.checkpoint = True
net = sim.SwitchNet()
e.add_network(net)
servers = create_basic_hosts(e, 1, 'server', net, sim.I40eNIC, sim.Gem5Host,
node.I40eLinuxNode, node.IperfTCPServer)
servers = create_basic_hosts(
e,
1,
'server',
net,
sim.I40eNIC,
sim.Gem5Host,
node.I40eLinuxNode,
node.IperfTCPServer
)
clients = create_basic_hosts(e, 2, 'client', net, sim.I40eNIC, sim.Gem5Host,
node.I40eLinuxNode, node.IperfTCPClient, ip_start = 2)
clients = create_basic_hosts(
e,
2,
'client',
net,
sim.I40eNIC,
sim.Gem5Host,
node.I40eLinuxNode,
node.IperfTCPClient,
ip_start=2
)
for h in servers + clients:
h.cpu_type = 'TimingSimpleCPU'
......
experiments/pyexps/gt_tcp_multi.py
View file @ 7549fa9e
......@@ -20,11 +20,11 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.nodeconfig as node
import simbricks.simulators as sim
from simbricks.simulator_utils import create_basic_hosts
import simbricks.experiments as exp
# iperf TCP_multi_client test
# naming convention following host-nic-net-app
......@@ -34,7 +34,7 @@ from simbricks.simulator_utils import create_basic_hosts
# app: TCPm
kinds_of_host = ['gem5-timing']
kinds_of_nic = ['cv','cb','ib']
kinds_of_nic = ['cv', 'cb', 'ib']
kinds_of_net = ['switch', 'dumbbell', 'bridge']
kinds_of_app = ['TCPm']
......@@ -52,7 +52,6 @@ for n in kinds_of_net:
if n == 'bridge':
net_class = sim.NS3BridgeNet
# set nic sim
for c in kinds_of_nic:
net = net_class()
......@@ -61,26 +60,73 @@ for n in kinds_of_net:
e.checkpoint = True
if c == 'cv':
servers = create_basic_hosts(e, 1, 'server', net, sim.CorundumVerilatorNIC, sim.Gem5Host,
node.CorundumLinuxNode, node.IperfTCPServer)
clients = create_basic_hosts(e, num_client, 'client', net, sim.CorundumVerilatorNIC, sim.Gem5Host,
node.CorundumLinuxNode, node.IperfTCPClient, ip_start = 2)
servers = create_basic_hosts(
e,
1,
'server',
net,
sim.CorundumVerilatorNIC,
sim.Gem5Host,
node.CorundumLinuxNode,
node.IperfTCPServer
)
clients = create_basic_hosts(
e,
num_client,
'client',
net,
sim.CorundumVerilatorNIC,
sim.Gem5Host,
node.CorundumLinuxNode,
node.IperfTCPClient,
ip_start=2
)
if c == 'cb':
servers = create_basic_hosts(e, 1, 'server', net, sim.CorundumBMNIC, sim.Gem5Host,
node.CorundumLinuxNode, node.IperfTCPServer)
clients = create_basic_hosts(e, num_client, 'client', net, sim.CorundumBMNIC, sim.Gem5Host,
node.CorundumLinuxNode, node.IperfTCPClient, ip_start = 2)
servers = create_basic_hosts(
e,
1,
'server',
net,
sim.CorundumBMNIC,
sim.Gem5Host,
node.CorundumLinuxNode,
node.IperfTCPServer
)
clients = create_basic_hosts(
e,
num_client,
'client',
net,
sim.CorundumBMNIC,
sim.Gem5Host,
node.CorundumLinuxNode,
node.IperfTCPClient,
ip_start=2
)
if c == 'ib':
servers = create_basic_hosts(e, 1, 'server', net, sim.I40eNIC, sim.Gem5Host,
node.I40eLinuxNode, node.IperfTCPServer)
clients = create_basic_hosts(e, num_client, 'client', net, sim.I40eNIC, sim.Gem5Host,
node.I40eLinuxNode, node.IperfTCPClient, ip_start = 2)
servers = create_basic_hosts(
e,
1,
'server',
net,
sim.I40eNIC,
sim.Gem5Host,
node.I40eLinuxNode,
node.IperfTCPServer
)
clients = create_basic_hosts(
e,
num_client,
'client',
net,
sim.I40eNIC,
sim.Gem5Host,
node.I40eLinuxNode,
node.IperfTCPClient,
ip_start=2
)
for cl in clients:
cl.wait = True
......@@ -88,4 +134,3 @@ for n in kinds_of_net:
print(e.name)
experiments.append(e)
experiments/pyexps/gt_tcp_single.py
View file @ 7549fa9e
......@@ -20,11 +20,11 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.nodeconfig as node
import simbricks.simulators as sim
from simbricks.simulator_utils import create_basic_hosts
import simbricks.experiments as exp
# iperf TCP_single test
# naming convention following host-nic-net-app
......@@ -34,7 +34,7 @@ from simbricks.simulator_utils import create_basic_hosts
# app: TCPs
kinds_of_host = ['gem5-timing']
kinds_of_nic = ['cv','cb','ib']
kinds_of_nic = ['cv', 'cb', 'ib']
kinds_of_net = ['wire', 'switch', 'dumbbell', 'bridge']
kinds_of_app = ['TCPs']
......@@ -51,7 +51,6 @@ for n in kinds_of_net:
if n == 'bridge':
net_class = sim.NS3BridgeNet
# set nic sim
for c in kinds_of_nic:
net = net_class()
......@@ -60,29 +59,76 @@ for n in kinds_of_net:
e.add_network(net)
if c == 'cv':
servers = create_basic_hosts(e, 1, 'server', net, sim.CorundumVerilatorNIC, sim.Gem5Host,
node.CorundumLinuxNode, node.IperfTCPServer)
clients = create_basic_hosts(e, 1, 'client', net, sim.CorundumVerilatorNIC, sim.Gem5Host,
node.CorundumLinuxNode, node.IperfTCPClient, ip_start = 2)
servers = create_basic_hosts(
e,
1,
'server',
net,
sim.CorundumVerilatorNIC,
sim.Gem5Host,
node.CorundumLinuxNode,
node.IperfTCPServer
)
clients = create_basic_hosts(
e,
1,
'client',
net,
sim.CorundumVerilatorNIC,
sim.Gem5Host,
node.CorundumLinuxNode,
node.IperfTCPClient,
ip_start=2
)
if c == 'cb':
servers = create_basic_hosts(e, 1, 'server', net, sim.CorundumBMNIC, sim.Gem5Host,
node.CorundumLinuxNode, node.IperfTCPServer)
clients = create_basic_hosts(e, 1, 'client', net, sim.CorundumBMNIC, sim.Gem5Host,
node.CorundumLinuxNode, node.IperfTCPClient, ip_start = 2)
servers = create_basic_hosts(
e,
1,
'server',
net,
sim.CorundumBMNIC,
sim.Gem5Host,
node.CorundumLinuxNode,
node.IperfTCPServer
)
clients = create_basic_hosts(
e,
1,
'client',
net,
sim.CorundumBMNIC,
sim.Gem5Host,
node.CorundumLinuxNode,
node.IperfTCPClient,
ip_start=2
)
if c == 'ib':
servers = create_basic_hosts(e, 1, 'server', net, sim.I40eNIC, sim.Gem5Host,
node.I40eLinuxNode, node.IperfTCPServer)
clients = create_basic_hosts(e, 1, 'client', net, sim.I40eNIC, sim.Gem5Host,
node.I40eLinuxNode, node.IperfTCPClient, ip_start = 2)
servers = create_basic_hosts(
e,
1,
'server',
net,
sim.I40eNIC,
sim.Gem5Host,
node.I40eLinuxNode,
node.IperfTCPServer
)
clients = create_basic_hosts(
e,
1,
'client',
net,
sim.I40eNIC,
sim.Gem5Host,
node.I40eLinuxNode,
node.IperfTCPClient,
ip_start=2
)
clients[0].wait = True
clients[0].node_config.app.server_ip = servers[0].node_config.ip
print(e.name)
experiments.append(e)
experiments/pyexps/gt_udp_multi.py
View file @ 7549fa9e
......@@ -20,11 +20,11 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.nodeconfig as node
import simbricks.simulators as sim
from simbricks.simulator_utils import create_basic_hosts
import simbricks.experiments as exp
# iperf TCP_multi_client test
# naming convention following host-nic-net-app
......@@ -34,7 +34,7 @@ from simbricks.simulator_utils import create_basic_hosts
# app: UDPm
kinds_of_host = ['gem5-timing']
kinds_of_nic = ['cv','cb','ib']
kinds_of_nic = ['cv', 'cb', 'ib']
kinds_of_net = ['switch', 'dumbbell', 'bridge']
kinds_of_app = ['UDPm']
......@@ -53,7 +53,6 @@ for n in kinds_of_net:
if n == 'bridge':
net_class = sim.NS3BridgeNet
# set nic sim
for c in kinds_of_nic:
net = net_class()
......@@ -62,26 +61,73 @@ for n in kinds_of_net:
e.add_network(net)
if c == 'cv':
servers = create_basic_hosts(e, 1, 'server', net, sim.CorundumVerilatorNIC, sim.Gem5Host,
node.CorundumLinuxNode, node.IperfUDPServer)
clients = create_basic_hosts(e, num_client, 'client', net, sim.CorundumVerilatorNIC, sim.Gem5Host,
node.CorundumLinuxNode, node.IperfUDPClient, ip_start = 2)
servers = create_basic_hosts(
e,
1,
'server',
net,
sim.CorundumVerilatorNIC,
sim.Gem5Host,
node.CorundumLinuxNode,
node.IperfUDPServer
)
clients = create_basic_hosts(
e,
num_client,
'client',
net,
sim.CorundumVerilatorNIC,
sim.Gem5Host,
node.CorundumLinuxNode,
node.IperfUDPClient,
ip_start=2
)
if c == 'cb':
servers = create_basic_hosts(e, 1, 'server', net, sim.CorundumBMNIC, sim.Gem5Host,
node.CorundumLinuxNode, node.IperfUDPServer)
clients = create_basic_hosts(e, num_client, 'client', net, sim.CorundumBMNIC, sim.Gem5Host,
node.CorundumLinuxNode, node.IperfUDPClient, ip_start = 2)
servers = create_basic_hosts(
e,
1,
'server',
net,
sim.CorundumBMNIC,
sim.Gem5Host,
node.CorundumLinuxNode,
node.IperfUDPServer
)
clients = create_basic_hosts(
e,
num_client,
'client',
net,
sim.CorundumBMNIC,
sim.Gem5Host,
node.CorundumLinuxNode,
node.IperfUDPClient,
ip_start=2
)
if c == 'ib':
servers = create_basic_hosts(e, 1, 'server', net, sim.I40eNIC, sim.Gem5Host,
node.I40eLinuxNode, node.IperfUDPServer)
clients = create_basic_hosts(e, num_client, 'client', net, sim.I40eNIC, sim.Gem5Host,
node.I40eLinuxNode, node.IperfUDPClient, ip_start = 2)
servers = create_basic_hosts(
e,
1,
'server',
net,
sim.I40eNIC,
sim.Gem5Host,
node.I40eLinuxNode,
node.IperfUDPServer
)
clients = create_basic_hosts(
e,
num_client,
'client',
net,
sim.I40eNIC,
sim.Gem5Host,
node.I40eLinuxNode,
node.IperfUDPClient,
ip_start=2
)
for cl in clients:
cl.wait = True
......@@ -90,4 +136,3 @@ for n in kinds_of_net:
print(e.name)
experiments.append(e)
experiments/pyexps/log_parser.py
View file @ 7549fa9e
......@@ -20,12 +20,12 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import sys
import json
import os
import pathlib
import shutil
import json
import re
import shutil
import sys
# How to use
# $ python3 pyexps/parser.py out/qemu-wire-ib-TCPs-1.json
......@@ -44,30 +44,30 @@ tooutdir = f'/{exp_name}/{run_num}'
outdir = '/'.join(list(p.parts)[0:-1]) + tooutdir
if not os.path.exists(outdir):
raise Exception("no such directory")
raise Exception('no such directory')
start_end_path = os.path.join(outdir, 'start_end.txt')
start_end_file = open(start_end_path, 'w')
start_end_file.write('start time: ' + str(exp_log["start_time"]) + '\n')
start_end_file.write('end time: ' + str(exp_log["end_time"]) + '\n')
start_end_file.write('success: ' + str(exp_log["success"]))
start_end_file.write('start time: ' + str(exp_log['start_time']) + '\n')
start_end_file.write('end time: ' + str(exp_log['end_time']) + '\n')
start_end_file.write('success: ' + str(exp_log['success']))
start_end_file.close()
for i in exp_log["sims"]:
for i in exp_log['sims']:
#print(i)
simdir = os.path.join(outdir, i)
sim_out_file = open(simdir, 'w')
for j in exp_log["sims"][i]:
for j in exp_log['sims'][i]:
#print(j)
sim_out_file.write( j + '\n')
sim_out_file.write(j + '\n')
if (j == 'class' ):
sim_out_file.write(exp_log["sims"][i][j] + '\n')
if (j == 'class'):
sim_out_file.write(exp_log['sims'][i][j] + '\n')
else:
for k in exp_log["sims"][i][j]:
for k in exp_log['sims'][i][j]:
sim_out_file.write(k + '\n')
sim_out_file.close()
experiments/pyexps/modetcp.py
View file @ 7549fa9e
......@@ -20,11 +20,11 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.nodeconfig as node
import simbricks.simulators as sim
from simbricks.simulator_utils import create_basic_hosts
import simbricks.experiments as exp
# iperf TCP_multi_client test
# naming convention following host-nic-net-app
......@@ -33,16 +33,14 @@ from simbricks.simulator_utils import create_basic_hosts
# net: switch/dumbbell/bridge
# app: DCTCPm
types_of_host = ['qemu', 'qt','gt']
types_of_nic = ['ib', 'cv','cb']
types_of_host = ['qemu', 'qt', 'gt']
types_of_nic = ['ib', 'cv', 'cb']
types_of_net = ['switch']
types_of_app = ['TCPm']
types_of_num_pairs = [1, 4]
types_of_mode = [0, 1]
experiments = []
for mode in types_of_mode:
for num_pairs in types_of_num_pairs:
......@@ -54,22 +52,29 @@ for mode in types_of_mode:
net.sync_mode = mode
#net.opt = link_rate_opt + link_latency_opt
e = exp.Experiment( f'mode-{mode}-' + h + '-' + c + '-' + 'switch' + f'-{num_pairs}')
e = exp.Experiment(
f'mode-{mode}-' + h + '-' + c + '-' + 'switch' +
f'-{num_pairs}'
)
e.add_network(net)
# host
if h == 'qemu':
host_class = sim.QemuHost
elif h == 'qt':
def qemu_timing():
h = sim.QemuHost()
h.sync = True
return h
host_class = qemu_timing
elif h == 'gt':
def gem5_timing():
h = sim.Gem5Host()
return h
host_class = gem5_timing
e.checkpoint = True
else:
......@@ -89,19 +94,32 @@ for mode in types_of_mode:
else:
raise NameError(c)
servers = create_basic_hosts(e, num_pairs, 'server', net, nic_class, host_class,
nc_class, node.IperfTCPServer)
clients = create_basic_hosts(e, num_pairs, 'client', net, nic_class, host_class,
nc_class, node.IperfTCPClient, ip_start=num_pairs+1)
servers = create_basic_hosts(
e,
num_pairs,
'server',
net,
nic_class,
host_class,
nc_class,
node.IperfTCPServer
)
clients = create_basic_hosts(
e,
num_pairs,
'client',
net,
nic_class,
host_class,
nc_class,
node.IperfTCPClient,
ip_start=num_pairs + 1
)
for se in servers:
se.sync_mode = mode
se.pcidevs[0].sync_mode = mode
i = 0
for cl in clients:
cl.sync_mode = mode
......@@ -115,9 +133,8 @@ for mode in types_of_mode:
# This is to prevent the simulation gets stuck when one of host exits.
# The last client waits for the output printed in other hosts, then cleanup
clients[num_pairs-1].node_config.app.is_last = True
clients[num_pairs-1].wait = True
clients[num_pairs - 1].node_config.app.is_last = True
clients[num_pairs - 1].wait = True
print(e.name)
experiments.append(e)
experiments/pyexps/mtcp_cores.py
View file @ 7549fa9e
......@@ -20,11 +20,11 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.nodeconfig as node
import simbricks.simulators as sim
from simbricks.simulator_utils import create_basic_hosts
import simbricks.experiments as exp
server_cores_configs = [1, 2, 4, 8]
stacks = ['linux', 'mtcp']
......@@ -36,8 +36,10 @@ msg_size = 64
experiments = []
for server_cores in server_cores_configs:
for stack in stacks:
e = exp.Experiment('qemu-ib-switch-mtcp_cores-%s-%d' % (stack,server_cores))
e.timeout = 5* 60
e = exp.Experiment(
'qemu-ib-switch-mtcp_cores-%s-%d' % (stack, server_cores)
)
e.timeout = 5 * 60
# add meta data for output file
e.metadata['msg_size'] = msg_size
e.metadata['stack'] = stack
......@@ -52,11 +54,21 @@ for server_cores in server_cores_configs:
else:
n = node.I40eLinuxNode
servers = create_basic_hosts(e, 1, 'server', net, sim.I40eNIC, sim.QemuHost,
n, node.RPCServer)
clients = create_basic_hosts(e, num_clients, 'client', net, sim.I40eNIC,
sim.QemuHost, n, node.RPCClient, ip_start = 2)
servers = create_basic_hosts(
e, 1, 'server', net, sim.I40eNIC, sim.QemuHost, n, node.RPCServer
)
clients = create_basic_hosts(
e,
num_clients,
'client',
net,
sim.I40eNIC,
sim.QemuHost,
n,
node.RPCClient,
ip_start=2
)
for h in servers:
h.node_config.cores = server_cores
......@@ -64,8 +76,6 @@ for server_cores in server_cores_configs:
h.node_config.app.max_flows = connections * 4
h.sleep = 5
for c in clients:
c.wait = True
c.node_config.cores = client_cores
......@@ -85,4 +95,3 @@ for server_cores in server_cores_configs:
c.node_config.cores += 2
c.node_config.fp_cores = 1
experiments.append(e)
experiments/pyexps/mtcp_httpd.py
View file @ 7549fa9e
......@@ -20,16 +20,17 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.nodeconfig as node
import simbricks.simulators as sim
from simbricks.simulator_utils import create_basic_hosts
import simbricks.experiments as exp
configs = [
(node.I40eLinuxNode, node.HTTPDLinux, node.HTTPCLinux, 'linux'),
(node.I40eLinuxNode, node.HTTPDLinuxRPO, node.HTTPCLinux, 'linuxrpo'),
(node.MtcpNode, node.HTTPDMtcp, node.HTTPCMtcp, 'mtcp')
]
]
server_cores = 1
client_cores = 1
......@@ -41,16 +42,26 @@ experiments = []
for (nodec, appc, clientc, label) in configs:
e = exp.Experiment('qemu-ib-switch-mtcp_httpd-%s' % (label))
e.timeout = 5* 60
e.timeout = 5 * 60
net = sim.SwitchNet()
e.add_network(net)
servers = create_basic_hosts(e, 1, 'server', net, sim.I40eNIC,
sim.QemuHost, nodec, appc)
servers = create_basic_hosts(
e, 1, 'server', net, sim.I40eNIC, sim.QemuHost, nodec, appc
)
clients = create_basic_hosts(e, num_clients, 'client', net, sim.I40eNIC,
sim.QemuHost, nodec, clientc, ip_start = 2)
clients = create_basic_hosts(
e,
num_clients,
'client',
net,
sim.I40eNIC,
sim.QemuHost,
nodec,
clientc,
ip_start=2
)
for h in servers:
h.node_config.cores = server_cores
......@@ -70,4 +81,3 @@ for (nodec, appc, clientc, label) in configs:
for h in servers + clients:
h.node_config.disk_image = 'mtcp'
experiments.append(e)
experiments/pyexps/mtcp_mpcs.py
View file @ 7549fa9e
......@@ -20,11 +20,11 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.nodeconfig as node
import simbricks.simulators as sim
from simbricks.simulator_utils import create_basic_hosts
import simbricks.experiments as exp
mpcs = [1, 8, 128]
stacks = ['linux', 'mtcp']
......@@ -37,8 +37,8 @@ msg_size = 64
experiments = []
for mpc in mpcs:
for stack in stacks:
e = exp.Experiment('qemu-ib-switch-mtcp_mpc-%s-%d' % (stack,mpc))
e.timeout = 5* 60
e = exp.Experiment('qemu-ib-switch-mtcp_mpc-%s-%d' % (stack, mpc))
e.timeout = 5 * 60
# add meta data for output file
e.metadata['mpc'] = mpc
e.metadata['stack'] = stack
......@@ -53,11 +53,21 @@ for mpc in mpcs:
else:
n = node.I40eLinuxNode
servers = create_basic_hosts(e, 1, 'server', net, sim.I40eNIC, sim.QemuHost,
n, node.RPCServer)
clients = create_basic_hosts(e, num_clients, 'client', net, sim.I40eNIC,
sim.QemuHost, n, node.RPCClient, ip_start = 2)
servers = create_basic_hosts(
e, 1, 'server', net, sim.I40eNIC, sim.QemuHost, n, node.RPCServer
)
clients = create_basic_hosts(
e,
num_clients,
'client',
net,
sim.I40eNIC,
sim.QemuHost,
n,
node.RPCClient,
ip_start=2
)
for h in servers:
h.node_config.cores = server_cores
......@@ -65,8 +75,6 @@ for mpc in mpcs:
h.node_config.app.max_flows = connections * 4
h.sleep = 5
for c in clients:
c.wait = True
c.node_config.cores = client_cores
......@@ -86,4 +94,3 @@ for mpc in mpcs:
c.node_config.cores += 2
c.node_config.fp_cores = 1
experiments.append(e)
experiments/pyexps/mtcp_msgsz.py
View file @ 7549fa9e
......@@ -20,11 +20,11 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import simbricks.experiments as exp
import simbricks.simulators as sim
import simbricks.nodeconfig as node
import simbricks.simulators as sim
from simbricks.simulator_utils import create_basic_hosts
import simbricks.experiments as exp
msg_sizes = [64, 1024, 8092]
stacks = ['linux', 'mtcp']
......@@ -36,8 +36,10 @@ connections = 512
experiments = []
for msg_size in msg_sizes:
for stack in stacks:
e = exp.Experiment('qemu-ib-switch-mtcp_msgsz-%s-%d' % (stack,msg_size))
e.timeout = 5* 60
e = exp.Experiment(
'qemu-ib-switch-mtcp_msgsz-%s-%d' % (stack, msg_size)
)
e.timeout = 5 * 60
# add meta data for output file
e.metadata['msg_size'] = msg_size
e.metadata['stack'] = stack
......@@ -52,11 +54,21 @@ for msg_size in msg_sizes:
else:
n = node.I40eLinuxNode
servers = create_basic_hosts(e, 1, 'server', net, sim.I40eNIC, sim.QemuHost,
n, node.RPCServer)
clients = create_basic_hosts(e, num_clients, 'client', net, sim.I40eNIC,
sim.QemuHost, n, node.RPCClient, ip_start = 2)
servers = create_basic_hosts(
e, 1, 'server', net, sim.I40eNIC, sim.QemuHost, n, node.RPCServer
)
clients = create_basic_hosts(
e,
num_clients,
'client',
net,
sim.I40eNIC,
sim.QemuHost,
n,
node.RPCClient,
ip_start=2
)
for h in servers:
h.node_config.cores = server_cores
......@@ -64,8 +76,6 @@ for msg_size in msg_sizes:
h.node_config.app.max_flows = connections * 4
h.sleep = 5
for c in clients:
c.wait = True
c.node_config.cores = client_cores
......@@ -85,4 +95,3 @@ for msg_size in msg_sizes:
c.node_config.cores += 2
c.node_config.fp_cores = 1
experiments.append(e)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment