Skip to content

GitLab

Commit 78d4c701 authored by Jonas Kaufmann's avatar Jonas Kaufmann Committed by Antoine Kaufmann
Browse files

fix all pylint issues except missing doc strings in experiments/

parent 7549fa9e
Hide whitespace changes
Inline Side-by-side
Showing with 265 additions and 262 deletions
experiments/pyexps/ae/utils/iperf.py
View file @ 78d4c701
...@@ -22,17 +22,15 @@ ...@@ -22,17 +22,15 @@
import fnmatch import fnmatch
import glob import glob
import itertools
import json import json
import os
import re import re
import sys
def parse_iperf_run(data, skip=1, use=8): def parse_iperf_run(data, skip=1, use=8):
tp_pat = re.compile( tp_pat = re.compile((
r'\[ *\d*\] *([0-9\.]*)- *([0-9\.]*) sec.*Bytes *([0-9\.]*) ([GM])bits.*' r'\[ *\d*\] *([0-9\.]*)'
) r'- *([0-9\.]*) sec.*Bytes *([0-9\.]*) ([GM])bits.*'
))
tps_time = {} tps_time = {}
for hn in fnmatch.filter(data['sims'].keys(), 'host.client.*'): for hn in fnmatch.filter(data['sims'].keys(), 'host.client.*'):
sim = data['sims'][hn] sim = data['sims'][hn]
...@@ -47,7 +45,7 @@ def parse_iperf_run(data, skip=1, use=8): ...@@ -47,7 +45,7 @@ def parse_iperf_run(data, skip=1, use=8):
if time >= skip + use: if time >= skip + use:
continue continue
if not time in tps_time: if time not in tps_time:
tps_time[time] = [] tps_time[time] = []
if m.group(4) == 'G': if m.group(4) == 'G':
...@@ -73,7 +71,7 @@ def parse_iperf(basename, skip=1, use=8): ...@@ -73,7 +71,7 @@ def parse_iperf(basename, skip=1, use=8):
# skip checkpoints # skip checkpoints
continue continue
with open(path, 'r') as f: with open(path, 'r', encoding='utf-8') as f:
data = json.load(f) data = json.load(f)
result = parse_iperf_run(data, skip, use) result = parse_iperf_run(data, skip, use)
if result is not None: if result is not None:
......
experiments/pyexps/ae/utils/netperf.py
View file @ 78d4c701
...@@ -30,7 +30,7 @@ def parse_netperf_run(path): ...@@ -30,7 +30,7 @@ def parse_netperf_run(path):
if not os.path.exists(path): if not os.path.exists(path):
return ret return ret
with open(path, 'r') as f: with open(path, 'r', encoding='utf-8') as f:
data = json.load(f) data = json.load(f)
ret['simtime'] = data['end_time'] - data['start_time'] ret['simtime'] = data['end_time'] - data['start_time']
......
experiments/pyexps/ae/utils/parse_nopaxos.py
View file @ 78d4c701
...@@ -37,25 +37,25 @@ def parse_nopaxos_run(num_c, path): ...@@ -37,25 +37,25 @@ def parse_nopaxos_run(num_c, path):
tp_pat = re.compile(r'(.*)Total throughput is *([0-9\.]*) ops/sec(.*)') tp_pat = re.compile(r'(.*)Total throughput is *([0-9\.]*) ops/sec(.*)')
lat_pat = re.compile(r'(.*)Median latency is *([0-9\.]*) us(.*)') lat_pat = re.compile(r'(.*)Median latency is *([0-9\.]*) us(.*)')
f_log = open(path, 'r') with open(path, 'r', encoding='utf-8') as f_log:
log = json.load(f_log) log = json.load(f_log)
total_tput = 0 total_tput = 0
total_lat = 0 total_lat = 0
for i in range(num_c): for i in range(num_c):
sim_name = f'host.client.{i}' sim_name = f'host.client.{i}'
# in this host log stdout # in this host log stdout
for j in log['sims'][sim_name]['stdout']: for j in log['sims'][sim_name]['stdout']:
m_t = tp_pat.match(j) m_t = tp_pat.match(j)
m_l = lat_pat.match(j) m_l = lat_pat.match(j)
if m_l: if m_l:
total_lat += float(m_l.group(2)) total_lat += float(m_l.group(2))
if m_t: if m_t:
total_tput += int(m_t.group(2)) total_tput += int(m_t.group(2))
avg_lat = total_lat / num_c avg_lat = total_lat / num_c
ret['throughput'] = total_tput ret['throughput'] = total_tput
ret['latency'] = int(avg_lat) ret['latency'] = int(avg_lat)
return ret return ret
experiments/pyexps/dctcp.py
View file @ 78d4c701
...@@ -55,27 +55,27 @@ ip_start = '192.168.64.1' ...@@ -55,27 +55,27 @@ ip_start = '192.168.64.1'
experiments = [] experiments = []
# set network sim # set network sim
net_class = sim.NS3DumbbellNet NetClass = sim.NS3DumbbellNet
for mtu in types_of_mtu: for mtu in types_of_mtu:
for h in types_of_host: for host in types_of_host:
for c in types_of_nic: for nic in types_of_nic:
for k_val in range(0, max_k + 1, k_step): for k_val in range(0, max_k + 1, k_step):
net = net_class() net = NetClass()
net.opt = link_rate_opt + link_latency_opt + f'--EcnTh={k_val}' net.opt = link_rate_opt + link_latency_opt + f'--EcnTh={k_val}'
e = exp.Experiment( e = exp.Experiment(
h + '-' + c + '-' + 'dumbbell' + '-' + 'DCTCPm' + host + '-' + nic + '-' + 'dumbbell' + '-' + 'DCTCPm' +
f'{k_val}' + f'-{mtu}' f'{k_val}' + f'-{mtu}'
) )
e.add_network(net) e.add_network(net)
freq = cpu_freq freq = cpu_freq
# host # host
if h == 'qemu': if host == 'qemu':
host_class = sim.QemuHost HostClass = sim.QemuHost
elif h == 'qt': elif host == 'qt':
freq = cpu_freq_qemu freq = cpu_freq_qemu
def qemu_timing(): def qemu_timing():
...@@ -83,17 +83,17 @@ for mtu in types_of_mtu: ...@@ -83,17 +83,17 @@ for mtu in types_of_mtu:
h.sync = True h.sync = True
return h return h
host_class = qemu_timing HostClass = qemu_timing
elif h == 'gt': elif host == 'gt':
def gem5_timing(): def gem5_timing():
h = sim.Gem5Host() h = sim.Gem5Host()
#h.sys_clock = sys_clock #h.sys_clock = sys_clock
return h return h
host_class = gem5_timing HostClass = gem5_timing
e.checkpoint = True e.checkpoint = True
elif h == 'gO3': elif host == 'gO3':
def gem5_o3(): def gem5_o3():
h = sim.Gem5Host() h = sim.Gem5Host()
...@@ -101,32 +101,32 @@ for mtu in types_of_mtu: ...@@ -101,32 +101,32 @@ for mtu in types_of_mtu:
h.sys_clock = sys_clock h.sys_clock = sys_clock
return h return h
host_class = gem5_o3 HostClass = gem5_o3
e.checkpoint = True e.checkpoint = True
else: else:
raise NameError(h) raise NameError(host)
# nic # nic
if c == 'ib': if nic == 'ib':
nic_class = sim.I40eNIC NicClass = sim.I40eNIC
nc_class = node.I40eDCTCPNode NcClass = node.I40eDCTCPNode
elif c == 'cb': elif nic == 'cb':
nic_class = sim.CorundumBMNIC NicClass = sim.CorundumBMNIC
nc_class = node.CorundumDCTCPNode NcClass = node.CorundumDCTCPNode
elif c == 'cv': elif nic == 'cv':
nic_class = sim.CorundumVerilatorNIC NicClass = sim.CorundumVerilatorNIC
nc_class = node.CorundumDCTCPNode NcClass = node.CorundumDCTCPNode
else: else:
raise NameError(c) raise NameError(nic)
servers = create_dctcp_hosts( servers = create_dctcp_hosts(
e, e,
num_pairs, num_pairs,
'server', 'server',
net, net,
nic_class, NicClass,
host_class, HostClass,
nc_class, NcClass,
node.DctcpServer, node.DctcpServer,
freq, freq,
mtu mtu
...@@ -136,9 +136,9 @@ for mtu in types_of_mtu: ...@@ -136,9 +136,9 @@ for mtu in types_of_mtu:
num_pairs, num_pairs,
'client', 'client',
net, net,
nic_class, NicClass,
host_class, HostClass,
nc_class, NcClass,
node.DctcpClient, node.DctcpClient,
freq, freq,
mtu, mtu,
...@@ -150,10 +150,12 @@ for mtu in types_of_mtu: ...@@ -150,10 +150,12 @@ for mtu in types_of_mtu:
cl.node_config.app.server_ip = servers[i].node_config.ip cl.node_config.app.server_ip = servers[i].node_config.ip
i += 1 i += 1
# All the clients will not poweroff after finishing iperf test except the last one # All the clients will not poweroff after finishing iperf test
# This is to prevent the simulation gets stuck when one of host exits. # except the last one This is to prevent the simulation gets
# stuck when one of host exits.
# The last client waits for the output printed in other hosts, then cleanup # The last client waits for the output printed in other hosts,
# then cleanup
clients[num_pairs - 1].node_config.app.is_last = True clients[num_pairs - 1].node_config.app.is_last = True
clients[num_pairs - 1].wait = True clients[num_pairs - 1].wait = True
......
experiments/pyexps/dist_memcache.py
View file @ 78d4c701
...@@ -24,11 +24,11 @@ import math ...@@ -24,11 +24,11 @@ import math
import random import random
import simbricks.nodeconfig as node import simbricks.nodeconfig as node
import simbricks.proxy as proxy
import simbricks.simulators as sim import simbricks.simulators as sim
from simbricks.simulator_utils import create_multinic_hosts from simbricks.simulator_utils import create_multinic_hosts
import simbricks.experiments as exp import simbricks.experiments as exp
from simbricks import proxy
host_types = ['qemu', 'gem5', 'qt'] host_types = ['qemu', 'gem5', 'qt']
n_nets = [1, 2, 3, 4, 8, 16, 32] n_nets = [1, 2, 3, 4, 8, 16, 32]
...@@ -39,7 +39,8 @@ separate_net = True ...@@ -39,7 +39,8 @@ separate_net = True
nets_per_host = 1 nets_per_host = 1
def select_servers(i, j, racks, n, n_host): # pylint: disable=redefined-outer-name
def select_servers(i, racks, n, n_host):
nc = int(n_host / 2) nc = int(n_host / 2)
if n == 1: if n == 1:
...@@ -75,7 +76,7 @@ for host_type in host_types: ...@@ -75,7 +76,7 @@ for host_type in host_types:
# host # host
if host_type == 'qemu': if host_type == 'qemu':
host_class = sim.QemuHost HostClass = sim.QemuHost
elif host_type == 'qt': elif host_type == 'qt':
def qemu_timing(): def qemu_timing():
...@@ -83,9 +84,9 @@ for host_type in host_types: ...@@ -83,9 +84,9 @@ for host_type in host_types:
h.sync = True h.sync = True
return h return h
host_class = qemu_timing HostClass = qemu_timing
elif host_type == 'gem5': elif host_type == 'gem5':
host_class = sim.Gem5Host HostClass = sim.Gem5Host
e.checkpoint = False e.checkpoint = False
else: else:
raise NameError(host_type) raise NameError(host_type)
...@@ -104,7 +105,7 @@ for host_type in host_types: ...@@ -104,7 +105,7 @@ for host_type in host_types:
h_i += 1 h_i += 1
switch = sim.SwitchNet() switch = sim.SwitchNet()
switch.name = 'switch_%d' % (i,) switch.name = f'switch_{i}'
if host_type == 'qemu': if host_type == 'qemu':
switch.sync = False switch.sync = False
e.add_network(switch) e.add_network(switch)
...@@ -117,9 +118,9 @@ for host_type in host_types: ...@@ -117,9 +118,9 @@ for host_type in host_types:
servers = create_multinic_hosts( servers = create_multinic_hosts(
e, e,
m, m,
'server_%d' % (i,), f'server_{i}',
switch, switch,
host_class, HostClass,
node.I40eLinuxNode, node.I40eLinuxNode,
node.MemcachedServer, node.MemcachedServer,
ip_start=i * n_host + 1, ip_start=i * n_host + 1,
...@@ -136,9 +137,9 @@ for host_type in host_types: ...@@ -136,9 +137,9 @@ for host_type in host_types:
clients = create_multinic_hosts( clients = create_multinic_hosts(
e, e,
m, m,
'client_%d' % (i,), f'client_{i}',
switch, switch,
host_class, HostClass,
node.I40eLinuxNode, node.I40eLinuxNode,
node.MemcachedClient, node.MemcachedClient,
ip_start=i * n_host + 1 + m, ip_start=i * n_host + 1 + m,
...@@ -157,12 +158,12 @@ for host_type in host_types: ...@@ -157,12 +158,12 @@ for host_type in host_types:
if h_i != 0: if h_i != 0:
lp = proxy.SocketsNetProxyListener() lp = proxy.SocketsNetProxyListener()
lp.name = 'listener-%d' % (i,) lp.name = f'listener-{i}'
e.add_proxy(lp) e.add_proxy(lp)
e.assign_sim_host(lp, h_i) e.assign_sim_host(lp, h_i)
cp = proxy.SocketsNetProxyConnecter(lp) cp = proxy.SocketsNetProxyConnecter(lp)
cp.name = 'connecter-%d' % (i,) cp.name = f'connecter-{i}'
e.add_proxy(cp) e.add_proxy(cp)
e.assign_sim_host(cp, 0) e.assign_sim_host(cp, 0)
...@@ -183,7 +184,7 @@ for host_type in host_types: ...@@ -183,7 +184,7 @@ for host_type in host_types:
for i in range(0, n): for i in range(0, n):
for j in range(0, int(n_host / 2)): for j in range(0, int(n_host / 2)):
c = racks[i][1][j] c = racks[i][1][j]
servers = select_servers(i, j, racks, n, n_host) servers = select_servers(i, racks, n, n_host)
server_ips = [s.node_config.ip for s in servers] server_ips = [s.node_config.ip for s in servers]
c.node_config.app.server_ips = server_ips c.node_config.app.server_ips = server_ips
......
experiments/pyexps/dist_multinet.py
View file @ 78d4c701
...@@ -21,11 +21,11 @@ ...@@ -21,11 +21,11 @@
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import simbricks.nodeconfig as node import simbricks.nodeconfig as node
import simbricks.proxy as proxy
import simbricks.simulators as sim import simbricks.simulators as sim
from simbricks.simulator_utils import create_basic_hosts from simbricks.simulator_utils import create_basic_hosts
import simbricks.experiments as exp import simbricks.experiments as exp
from simbricks import proxy
host_types = ['qemu', 'gem5', 'qt'] host_types = ['qemu', 'gem5', 'qt']
n_nets = [1, 2, 3, 4] n_nets = [1, 2, 3, 4]
...@@ -44,7 +44,7 @@ for host_type in host_types: ...@@ -44,7 +44,7 @@ for host_type in host_types:
# host # host
if host_type == 'qemu': if host_type == 'qemu':
host_class = sim.QemuHost HostClass = sim.QemuHost
elif host_type == 'qt': elif host_type == 'qt':
def qemu_timing(): def qemu_timing():
...@@ -52,9 +52,9 @@ for host_type in host_types: ...@@ -52,9 +52,9 @@ for host_type in host_types:
h.sync = True h.sync = True
return h return h
host_class = qemu_timing HostClass = qemu_timing
elif host_type == 'gem5': elif host_type == 'gem5':
host_class = sim.Gem5Host HostClass = sim.Gem5Host
e.checkpoint = True e.checkpoint = True
else: else:
raise NameError(host_type) raise NameError(host_type)
...@@ -69,7 +69,7 @@ for host_type in host_types: ...@@ -69,7 +69,7 @@ for host_type in host_types:
for i in range(0, n): for i in range(0, n):
h_i = i if not separate_net else i + 1 h_i = i if not separate_net else i + 1
switch = sim.SwitchNet() switch = sim.SwitchNet()
switch.name = 'switch_%d' % (i,) switch.name = f'switch_{i}'
if host_type == 'qemu': if host_type == 'qemu':
switch.sync = False switch.sync = False
e.add_network(switch) e.add_network(switch)
...@@ -83,10 +83,10 @@ for host_type in host_types: ...@@ -83,10 +83,10 @@ for host_type in host_types:
servers = create_basic_hosts( servers = create_basic_hosts(
e, e,
1, 1,
'server_%d' % (i,), f'server_{i}',
switch, switch,
sim.I40eNIC, sim.I40eNIC,
host_class, HostClass,
node.I40eLinuxNode, node.I40eLinuxNode,
node.NetperfServer, node.NetperfServer,
ip_start=i * (n_client + 1) + 1 ip_start=i * (n_client + 1) + 1
...@@ -100,10 +100,10 @@ for host_type in host_types: ...@@ -100,10 +100,10 @@ for host_type in host_types:
clients = create_basic_hosts( clients = create_basic_hosts(
e, e,
m, m,
'client_%d' % (i,), f'client_{i}',
switch, switch,
sim.I40eNIC, sim.I40eNIC,
host_class, HostClass,
node.I40eLinuxNode, node.I40eLinuxNode,
node.NetperfClient, node.NetperfClient,
ip_start=i * (n_client + 1) + 2 ip_start=i * (n_client + 1) + 2
...@@ -120,12 +120,12 @@ for host_type in host_types: ...@@ -120,12 +120,12 @@ for host_type in host_types:
if h_i != 0: if h_i != 0:
lp = proxy.SocketsNetProxyListener() lp = proxy.SocketsNetProxyListener()
lp.name = 'listener-%d' % (i,) lp.name = f'listener-{i}'
e.add_proxy(lp) e.add_proxy(lp)
e.assign_sim_host(lp, h_i) e.assign_sim_host(lp, h_i)
cp = proxy.SocketsNetProxyConnecter(lp) cp = proxy.SocketsNetProxyConnecter(lp)
cp.name = 'connecter-%d' % (i,) cp.name = f'connecter-{i}'
e.add_proxy(cp) e.add_proxy(cp)
e.assign_sim_host(cp, 0) e.assign_sim_host(cp, 0)
......
experiments/pyexps/dist_netperf.py
View file @ 78d4c701
...@@ -21,11 +21,11 @@ ...@@ -21,11 +21,11 @@
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import simbricks.nodeconfig as node import simbricks.nodeconfig as node
import simbricks.proxy as proxy
import simbricks.simulators as sim import simbricks.simulators as sim
from simbricks.simulator_utils import create_basic_hosts from simbricks.simulator_utils import create_basic_hosts
import simbricks.experiments as exp import simbricks.experiments as exp
from simbricks import proxy
host_types = ['qemu', 'gem5', 'qt'] host_types = ['qemu', 'gem5', 'qt']
nic_types = ['i40e', 'cd_bm', 'cd_verilator'] nic_types = ['i40e', 'cd_bm', 'cd_verilator']
...@@ -44,7 +44,7 @@ for host_type in host_types: ...@@ -44,7 +44,7 @@ for host_type in host_types:
# host # host
if host_type == 'qemu': if host_type == 'qemu':
host_class = sim.QemuHost HostClass = sim.QemuHost
net.sync = False net.sync = False
elif host_type == 'qt': elif host_type == 'qt':
...@@ -53,23 +53,23 @@ for host_type in host_types: ...@@ -53,23 +53,23 @@ for host_type in host_types:
h.sync = True h.sync = True
return h return h
host_class = qemu_timing HostClass = qemu_timing
elif host_type == 'gem5': elif host_type == 'gem5':
host_class = sim.Gem5Host HostClass = sim.Gem5Host
e.checkpoint = True e.checkpoint = True
else: else:
raise NameError(host_type) raise NameError(host_type)
# nic # nic
if nic_type == 'i40e': if nic_type == 'i40e':
nic_class = sim.I40eNIC NicClass = sim.I40eNIC
nc_class = node.I40eLinuxNode NcClass = node.I40eLinuxNode
elif nic_type == 'cd_bm': elif nic_type == 'cd_bm':
nic_class = sim.CorundumBMNIC NicClass = sim.CorundumBMNIC
nc_class = node.CorundumLinuxNode NcClass = node.CorundumLinuxNode
elif nic_type == 'cd_verilator': elif nic_type == 'cd_verilator':
nic_class = sim.CorundumVerilatorNIC NicClass = sim.CorundumVerilatorNIC
nc_class = node.CorundumLinuxNode NcClass = node.CorundumLinuxNode
else: else:
raise NameError(nic_type) raise NameError(nic_type)
...@@ -79,9 +79,9 @@ for host_type in host_types: ...@@ -79,9 +79,9 @@ for host_type in host_types:
1, 1,
'server', 'server',
net, net,
nic_class, NicClass,
host_class, HostClass,
nc_class, NcClass,
node.NetperfServer node.NetperfServer
) )
...@@ -90,9 +90,9 @@ for host_type in host_types: ...@@ -90,9 +90,9 @@ for host_type in host_types:
n, n,
'client', 'client',
net, net,
nic_class, NicClass,
host_class, HostClass,
nc_class, NcClass,
node.NetperfClient, node.NetperfClient,
ip_start=2 ip_start=2
) )
......
experiments/pyexps/gt_tcp_multi.py
View file @ 78d4c701
...@@ -46,15 +46,15 @@ experiments = [] ...@@ -46,15 +46,15 @@ experiments = []
for n in kinds_of_net: for n in kinds_of_net:
if n == 'switch': if n == 'switch':
net_class = sim.SwitchNet NetClass = sim.SwitchNet
if n == 'dumbbell': if n == 'dumbbell':
net_class = sim.NS3DumbbellNet NetClass = sim.NS3DumbbellNet
if n == 'bridge': if n == 'bridge':
net_class = sim.NS3BridgeNet NetClass = sim.NS3BridgeNet
# set nic sim # set nic sim
for c in kinds_of_nic: for c in kinds_of_nic:
net = net_class() net = NetClass()
e = exp.Experiment('gt-' + c + '-' + n + '-' + 'TCPm') e = exp.Experiment('gt-' + c + '-' + n + '-' + 'TCPm')
e.add_network(net) e.add_network(net)
e.checkpoint = True e.checkpoint = True
......
experiments/pyexps/gt_tcp_single.py
View file @ 78d4c701
...@@ -43,17 +43,17 @@ experiments = [] ...@@ -43,17 +43,17 @@ experiments = []
# set network sim # set network sim
for n in kinds_of_net: for n in kinds_of_net:
if n == 'wire': if n == 'wire':
net_class = sim.WireNet NetClass = sim.WireNet
if n == 'switch': if n == 'switch':
net_class = sim.SwitchNet NetClass = sim.SwitchNet
if n == 'dumbbell': if n == 'dumbbell':
net_class = sim.NS3DumbbellNet NetClass = sim.NS3DumbbellNet
if n == 'bridge': if n == 'bridge':
net_class = sim.NS3BridgeNet NetClass = sim.NS3BridgeNet
# set nic sim # set nic sim
for c in kinds_of_nic: for c in kinds_of_nic:
net = net_class() net = NetClass()
e = exp.Experiment('gt-' + c + '-' + n + '-' + 'TCPs') e = exp.Experiment('gt-' + c + '-' + n + '-' + 'TCPs')
e.checkpoint = True e.checkpoint = True
e.add_network(net) e.add_network(net)
......
experiments/pyexps/gt_udp_multi.py
View file @ 78d4c701
...@@ -47,15 +47,15 @@ experiments = [] ...@@ -47,15 +47,15 @@ experiments = []
for n in kinds_of_net: for n in kinds_of_net:
if n == 'switch': if n == 'switch':
net_class = sim.SwitchNet NetClass = sim.SwitchNet
if n == 'dumbbell': if n == 'dumbbell':
net_class = sim.NS3DumbbellNet NetClass = sim.NS3DumbbellNet
if n == 'bridge': if n == 'bridge':
net_class = sim.NS3BridgeNet NetClass = sim.NS3BridgeNet
# set nic sim # set nic sim
for c in kinds_of_nic: for c in kinds_of_nic:
net = net_class() net = NetClass()
e = exp.Experiment('gt-' + c + '-' + n + '-' + 'UDPm') e = exp.Experiment('gt-' + c + '-' + n + '-' + 'UDPm')
e.checkpoint = True e.checkpoint = True
e.add_network(net) e.add_network(net)
......
experiments/pyexps/log_parser.py
View file @ 78d4c701
...@@ -24,7 +24,6 @@ import json ...@@ -24,7 +24,6 @@ import json
import os import os
import pathlib import pathlib
import re import re
import shutil
import sys import sys
# How to use # How to use
...@@ -32,42 +31,41 @@ import sys ...@@ -32,42 +31,41 @@ import sys
# #
log_file = sys.argv[1] log_file = sys.argv[1]
log = open(log_file, 'r')
exp_log = json.load(log) with open(log_file, 'r', encoding='utf-8') as log:
run_num = re.split('-|\.', log_file)[-2] exp_log = json.load(log)
#Name, starting & ending time run_num = re.split(r'-|\.', log_file)[-2]
exp_name = exp_log['exp_name'] #Name, starting & ending time
exp_name = exp_log['exp_name']
p = pathlib.Path(log_file) p = pathlib.Path(log_file)
tooutdir = f'/{exp_name}/{run_num}' tooutdir = f'/{exp_name}/{run_num}'
outdir = '/'.join(list(p.parts)[0:-1]) + tooutdir outdir = '/'.join(list(p.parts)[0:-1]) + tooutdir
if not os.path.exists(outdir): if not os.path.exists(outdir):
raise Exception('no such directory') raise Exception('no such directory')
start_end_path = os.path.join(outdir, 'start_end.txt') start_end_path = os.path.join(outdir, 'start_end.txt')
start_end_file = open(start_end_path, 'w')
start_end_file.write('start time: ' + str(exp_log['start_time']) + '\n')
start_end_file.write('end time: ' + str(exp_log['end_time']) + '\n')
start_end_file.write('success: ' + str(exp_log['success']))
start_end_file.close()
for i in exp_log['sims']: with open(start_end_path, 'w', encoding='utf-8') as start_end_file:
#print(i) start_end_file.write('start time: ' + str(exp_log['start_time']) + '\n')
simdir = os.path.join(outdir, i) start_end_file.write('end time: ' + str(exp_log['end_time']) + '\n')
sim_out_file = open(simdir, 'w') start_end_file.write('success: ' + str(exp_log['success']))
for j in exp_log['sims'][i]: for i in exp_log['sims']:
#print(j) #print(i)
sim_out_file.write(j + '\n') simdir = os.path.join(outdir, i)
if (j == 'class'): with open(simdir, 'w', encoding='utf-8') as sim_out_file:
sim_out_file.write(exp_log['sims'][i][j] + '\n')
else: for j in exp_log['sims'][i]:
#print(j)
sim_out_file.write(j + '\n')
for k in exp_log['sims'][i][j]: if j == 'class':
sim_out_file.write(k + '\n') sim_out_file.write(exp_log['sims'][i][j] + '\n')
sim_out_file.close() else:
for k in exp_log['sims'][i][j]:
sim_out_file.write(k + '\n')
experiments/pyexps/modetcp.py
View file @ 78d4c701
...@@ -45,7 +45,7 @@ experiments = [] ...@@ -45,7 +45,7 @@ experiments = []
for mode in types_of_mode: for mode in types_of_mode:
for num_pairs in types_of_num_pairs: for num_pairs in types_of_num_pairs:
for h in types_of_host: for host in types_of_host:
for c in types_of_nic: for c in types_of_nic:
net = sim.SwitchNet() net = sim.SwitchNet()
...@@ -53,44 +53,44 @@ for mode in types_of_mode: ...@@ -53,44 +53,44 @@ for mode in types_of_mode:
#net.opt = link_rate_opt + link_latency_opt #net.opt = link_rate_opt + link_latency_opt
e = exp.Experiment( e = exp.Experiment(
f'mode-{mode}-' + h + '-' + c + '-' + 'switch' + f'mode-{mode}-' + host + '-' + c + '-' + 'switch' +
f'-{num_pairs}' f'-{num_pairs}'
) )
e.add_network(net) e.add_network(net)
# host # host
if h == 'qemu': if host == 'qemu':
host_class = sim.QemuHost HostClass = sim.QemuHost
elif h == 'qt': elif host == 'qt':
def qemu_timing(): def qemu_timing():
h = sim.QemuHost() h = sim.QemuHost()
h.sync = True h.sync = True
return h return h
host_class = qemu_timing HostClass = qemu_timing
elif h == 'gt': elif host == 'gt':
def gem5_timing(): def gem5_timing():
h = sim.Gem5Host() h = sim.Gem5Host()
return h return h
host_class = gem5_timing HostClass = gem5_timing
e.checkpoint = True e.checkpoint = True
else: else:
raise NameError(h) raise NameError(host)
# nic # nic
if c == 'cb': if c == 'cb':
nic_class = sim.CorundumBMNIC NicClass = sim.CorundumBMNIC
nc_class = node.CorundumLinuxNode NcClass = node.CorundumLinuxNode
elif c == 'cv': elif c == 'cv':
nic_class = sim.CorundumVerilatorNIC NicClass = sim.CorundumVerilatorNIC
nc_class = node.CorundumLinuxNode NcClass = node.CorundumLinuxNode
elif c == 'ib': elif c == 'ib':
nic_class = sim.I40eNIC NicClass = sim.I40eNIC
nc_class = node.I40eDCTCPNode NcClass = node.I40eDCTCPNode
else: else:
raise NameError(c) raise NameError(c)
...@@ -99,9 +99,9 @@ for mode in types_of_mode: ...@@ -99,9 +99,9 @@ for mode in types_of_mode:
num_pairs, num_pairs,
'server', 'server',
net, net,
nic_class, NicClass,
host_class, HostClass,
nc_class, NcClass,
node.IperfTCPServer node.IperfTCPServer
) )
clients = create_basic_hosts( clients = create_basic_hosts(
...@@ -109,9 +109,9 @@ for mode in types_of_mode: ...@@ -109,9 +109,9 @@ for mode in types_of_mode:
num_pairs, num_pairs,
'client', 'client',
net, net,
nic_class, NicClass,
host_class, HostClass,
nc_class, NcClass,
node.IperfTCPClient, node.IperfTCPClient,
ip_start=num_pairs + 1 ip_start=num_pairs + 1
) )
...@@ -129,10 +129,12 @@ for mode in types_of_mode: ...@@ -129,10 +129,12 @@ for mode in types_of_mode:
i += 1 i += 1
#cl.wait = True #cl.wait = True
# All the clients will not poweroff after finishing iperf test except the last one # All the clients will not poweroff after finishing iperf test
# This is to prevent the simulation gets stuck when one of host exits. # except the last one. This is to prevent the simulation gets
# stuck when one of host exits.
# The last client waits for the output printed in other hosts, then cleanup # The last client waits for the output printed in other hosts,
# then cleanup.
clients[num_pairs - 1].node_config.app.is_last = True clients[num_pairs - 1].node_config.app.is_last = True
clients[num_pairs - 1].wait = True clients[num_pairs - 1].wait = True
......
experiments/pyexps/mtcp_cores.py
View file @ 78d4c701
...@@ -36,9 +36,7 @@ msg_size = 64 ...@@ -36,9 +36,7 @@ msg_size = 64
experiments = [] experiments = []
for server_cores in server_cores_configs: for server_cores in server_cores_configs:
for stack in stacks: for stack in stacks:
e = exp.Experiment( e = exp.Experiment(f'qemu-ib-switch-mtcp_cores-{stack}-{server_cores}')
'qemu-ib-switch-mtcp_cores-%s-%d' % (stack, server_cores)
)
e.timeout = 5 * 60 e.timeout = 5 * 60
# add meta data for output file # add meta data for output file
e.metadata['msg_size'] = msg_size e.metadata['msg_size'] = msg_size
...@@ -48,14 +46,14 @@ for server_cores in server_cores_configs: ...@@ -48,14 +46,14 @@ for server_cores in server_cores_configs:
e.add_network(net) e.add_network(net)
if stack == 'tas': if stack == 'tas':
n = node.TASNode N = node.TASNode
elif stack == 'mtcp': elif stack == 'mtcp':
n = node.MtcpNode N = node.MtcpNode
else: else:
n = node.I40eLinuxNode N = node.I40eLinuxNode
servers = create_basic_hosts( servers = create_basic_hosts(
e, 1, 'server', net, sim.I40eNIC, sim.QemuHost, n, node.RPCServer e, 1, 'server', net, sim.I40eNIC, sim.QemuHost, N, node.RPCServer
) )
clients = create_basic_hosts( clients = create_basic_hosts(
...@@ -65,7 +63,7 @@ for server_cores in server_cores_configs: ...@@ -65,7 +63,7 @@ for server_cores in server_cores_configs:
net, net,
sim.I40eNIC, sim.I40eNIC,
sim.QemuHost, sim.QemuHost,
n, N,
node.RPCClient, node.RPCClient,
ip_start=2 ip_start=2
) )
...@@ -92,6 +90,6 @@ for server_cores in server_cores_configs: ...@@ -92,6 +90,6 @@ for server_cores in server_cores_configs:
if stack == 'linux': if stack == 'linux':
h.node_config.disk_image = 'tas' h.node_config.disk_image = 'tas'
elif stack == 'tas': elif stack == 'tas':
c.node_config.cores += 2 h.node_config.cores += 2
c.node_config.fp_cores = 1 h.node_config.fp_cores = 1
experiments.append(e) experiments.append(e)
experiments/pyexps/mtcp_httpd.py
View file @ 78d4c701
...@@ -41,7 +41,7 @@ file_size = 64 ...@@ -41,7 +41,7 @@ file_size = 64
experiments = [] experiments = []
for (nodec, appc, clientc, label) in configs: for (nodec, appc, clientc, label) in configs:
e = exp.Experiment('qemu-ib-switch-mtcp_httpd-%s' % (label)) e = exp.Experiment(f'qemu-ib-switch-mtcp_httpd-{label}')
e.timeout = 5 * 60 e.timeout = 5 * 60
net = sim.SwitchNet() net = sim.SwitchNet()
......
experiments/pyexps/mtcp_mpcs.py
View file @ 78d4c701
...@@ -37,7 +37,7 @@ msg_size = 64 ...@@ -37,7 +37,7 @@ msg_size = 64
experiments = [] experiments = []
for mpc in mpcs: for mpc in mpcs:
for stack in stacks: for stack in stacks:
e = exp.Experiment('qemu-ib-switch-mtcp_mpc-%s-%d' % (stack, mpc)) e = exp.Experiment(f'qemu-ib-switch-mtcp_mpc-P{stack}-{mpc}')
e.timeout = 5 * 60 e.timeout = 5 * 60
# add meta data for output file # add meta data for output file
e.metadata['mpc'] = mpc e.metadata['mpc'] = mpc
...@@ -47,14 +47,14 @@ for mpc in mpcs: ...@@ -47,14 +47,14 @@ for mpc in mpcs:
e.add_network(net) e.add_network(net)
if stack == 'tas': if stack == 'tas':
n = node.TASNode N = node.TASNode
elif stack == 'mtcp': elif stack == 'mtcp':
n = node.MtcpNode N = node.MtcpNode
else: else:
n = node.I40eLinuxNode N = node.I40eLinuxNode
servers = create_basic_hosts( servers = create_basic_hosts(
e, 1, 'server', net, sim.I40eNIC, sim.QemuHost, n, node.RPCServer e, 1, 'server', net, sim.I40eNIC, sim.QemuHost, N, node.RPCServer
) )
clients = create_basic_hosts( clients = create_basic_hosts(
...@@ -64,7 +64,7 @@ for mpc in mpcs: ...@@ -64,7 +64,7 @@ for mpc in mpcs:
net, net,
sim.I40eNIC, sim.I40eNIC,
sim.QemuHost, sim.QemuHost,
n, N,
node.RPCClient, node.RPCClient,
ip_start=2 ip_start=2
) )
...@@ -91,6 +91,6 @@ for mpc in mpcs: ...@@ -91,6 +91,6 @@ for mpc in mpcs:
if stack == 'linux': if stack == 'linux':
h.node_config.disk_image = 'tas' h.node_config.disk_image = 'tas'
elif stack == 'tas': elif stack == 'tas':
c.node_config.cores += 2 h.node_config.cores += 2
c.node_config.fp_cores = 1 h.node_config.fp_cores = 1
experiments.append(e) experiments.append(e)
experiments/pyexps/mtcp_msgsz.py
View file @ 78d4c701
...@@ -36,9 +36,7 @@ connections = 512 ...@@ -36,9 +36,7 @@ connections = 512
experiments = [] experiments = []
for msg_size in msg_sizes: for msg_size in msg_sizes:
for stack in stacks: for stack in stacks:
e = exp.Experiment( e = exp.Experiment(f'qemu-ib-switch-mtcp_msgsz-{stack}-{msg_size}')
'qemu-ib-switch-mtcp_msgsz-%s-%d' % (stack, msg_size)
)
e.timeout = 5 * 60 e.timeout = 5 * 60
# add meta data for output file # add meta data for output file
e.metadata['msg_size'] = msg_size e.metadata['msg_size'] = msg_size
...@@ -48,14 +46,14 @@ for msg_size in msg_sizes: ...@@ -48,14 +46,14 @@ for msg_size in msg_sizes:
e.add_network(net) e.add_network(net)
if stack == 'tas': if stack == 'tas':
n = node.TASNode N = node.TASNode
elif stack == 'mtcp': elif stack == 'mtcp':
n = node.MtcpNode N = node.MtcpNode
else: else:
n = node.I40eLinuxNode N = node.I40eLinuxNode
servers = create_basic_hosts( servers = create_basic_hosts(
e, 1, 'server', net, sim.I40eNIC, sim.QemuHost, n, node.RPCServer e, 1, 'server', net, sim.I40eNIC, sim.QemuHost, N, node.RPCServer
) )
clients = create_basic_hosts( clients = create_basic_hosts(
...@@ -65,7 +63,7 @@ for msg_size in msg_sizes: ...@@ -65,7 +63,7 @@ for msg_size in msg_sizes:
net, net,
sim.I40eNIC, sim.I40eNIC,
sim.QemuHost, sim.QemuHost,
n, N,
node.RPCClient, node.RPCClient,
ip_start=2 ip_start=2
) )
...@@ -92,6 +90,6 @@ for msg_size in msg_sizes: ...@@ -92,6 +90,6 @@ for msg_size in msg_sizes:
if stack == 'linux': if stack == 'linux':
h.node_config.disk_image = 'tas' h.node_config.disk_image = 'tas'
elif stack == 'tas': elif stack == 'tas':
c.node_config.cores += 2 h.node_config.cores += 2
c.node_config.fp_cores = 1 h.node_config.fp_cores = 1
experiments.append(e) experiments.append(e)
experiments/pyexps/netperf.py
View file @ 78d4c701
...@@ -58,7 +58,7 @@ for host_type in host_types: ...@@ -58,7 +58,7 @@ for host_type in host_types:
# host # host
if host_type == 'qemu': if host_type == 'qemu':
host_class = sim.QemuHost HostClass = sim.QemuHost
elif host_type == 'qt': elif host_type == 'qt':
def qemu_timing(): def qemu_timing():
...@@ -66,23 +66,23 @@ for host_type in host_types: ...@@ -66,23 +66,23 @@ for host_type in host_types:
h.sync = True h.sync = True
return h return h
host_class = qemu_timing HostClass = qemu_timing
elif host_type == 'gem5': elif host_type == 'gem5':
host_class = sim.Gem5Host HostClass = sim.Gem5Host
e.checkpoint = True e.checkpoint = True
else: else:
raise NameError(host_type) raise NameError(host_type)
# nic # nic
if nic_type == 'i40e': if nic_type == 'i40e':
nic_class = sim.I40eNIC NicClass = sim.I40eNIC
nc_class = node.I40eLinuxNode NcClass = node.I40eLinuxNode
elif nic_type == 'cd_bm': elif nic_type == 'cd_bm':
nic_class = sim.CorundumBMNIC NicClass = sim.CorundumBMNIC
nc_class = node.CorundumLinuxNode NcClass = node.CorundumLinuxNode
elif nic_type == 'cd_verilator': elif nic_type == 'cd_verilator':
nic_class = sim.CorundumVerilatorNIC NicClass = sim.CorundumVerilatorNIC
nc_class = node.CorundumLinuxNode NcClass = node.CorundumLinuxNode
else: else:
raise NameError(nic_type) raise NameError(nic_type)
...@@ -92,9 +92,9 @@ for host_type in host_types: ...@@ -92,9 +92,9 @@ for host_type in host_types:
1, 1,
'server', 'server',
net, net,
nic_class, NicClass,
host_class, HostClass,
nc_class, NcClass,
node.NetperfServer node.NetperfServer
) )
...@@ -103,9 +103,9 @@ for host_type in host_types: ...@@ -103,9 +103,9 @@ for host_type in host_types:
1, 1,
'client', 'client',
net, net,
nic_class, NicClass,
host_class, HostClass,
nc_class, NcClass,
node.NetperfClient, node.NetperfClient,
ip_start=2 ip_start=2
) )
......
experiments/pyexps/no-simbricks.py → experiments/pyexps/no_simbricks.py
View file @ 78d4c701
...@@ -33,16 +33,16 @@ for app_type in app_types: ...@@ -33,16 +33,16 @@ for app_type in app_types:
e = exp.Experiment('no_simb-gt-' + app_type) e = exp.Experiment('no_simb-gt-' + app_type)
host_class = sim.Gem5Host HostClass = sim.Gem5Host
e.checkpoint = True e.checkpoint = True
e.no_simbricks = True e.no_simbricks = True
nc_class = node.I40eLinuxNode NcClass = node.I40eLinuxNode
# create servers and clients # create servers and clients
host = host_class() host = HostClass()
host.name = 'host.0' host.name = 'host.0'
node_config = nc_class() node_config = NcClass()
node_config.ip = '10.0.0.1' node_config.ip = '10.0.0.1'
node_config.app = node.NoTraffic() node_config.app = node.NoTraffic()
node_config.cores = 1 node_config.cores = 1
......
experiments/pyexps/no_traffic.py
View file @ 78d4c701
...@@ -64,7 +64,7 @@ for host_type in host_types: ...@@ -64,7 +64,7 @@ for host_type in host_types:
# host # host
if host_type == 'qemu': if host_type == 'qemu':
host_class = sim.QemuHost HostClass = sim.QemuHost
elif host_type == 'qt': elif host_type == 'qt':
def qemu_timing(): def qemu_timing():
...@@ -72,45 +72,51 @@ for host_type in host_types: ...@@ -72,45 +72,51 @@ for host_type in host_types:
h.sync = True h.sync = True
return h return h
host_class = qemu_timing HostClass = qemu_timing
elif host_type == 'gt': elif host_type == 'gt':
host_class = sim.Gem5Host HostClass = sim.Gem5Host
e.checkpoint = True e.checkpoint = True
else: else:
raise NameError(host_type) raise NameError(host_type)
# nic # nic
if nic_type == 'ib': if nic_type == 'ib':
nic_class = sim.I40eNIC NicClass = sim.I40eNIC
nc_class = node.I40eLinuxNode NcClass = node.I40eLinuxNode
elif nic_type == 'cb': elif nic_type == 'cb':
nic_class = sim.CorundumBMNIC NicClass = sim.CorundumBMNIC
nc_class = node.CorundumLinuxNode NcClass = node.CorundumLinuxNode
elif nic_type == 'cv': elif nic_type == 'cv':
nic_class = sim.CorundumVerilatorNIC NicClass = sim.CorundumVerilatorNIC
nc_class = node.CorundumLinuxNode NcClass = node.CorundumLinuxNode
else: else:
raise NameError(nic_type) raise NameError(nic_type)
# create servers and clients # create servers and clients
""" # servers = create_basic_hosts(
servers = create_basic_hosts(e, 1, 'server', net, nic_class, host_class, # e,
nc_class, node.NoTraffic) # 1,
# 'server',
for s in servers: # net,
s.node_config.cores = num_cores # NicClass,
s.node_config.app.is_sleep = 0 # HostClass,
s.node_config.app.is_server = 1 # NcClass,
""" # node.NoTraffic
# )
# for s in servers:
# s.node_config.cores = num_cores
# s.node_config.app.is_sleep = 0
# s.node_config.app.is_server = 1
clients = create_basic_hosts( clients = create_basic_hosts(
e, e,
n_client, n_client,
'client', 'client',
net, net,
nic_class, NicClass,
host_class, HostClass,
nc_class, NcClass,
node.NoTraffic, node.NoTraffic,
ip_start=2 ip_start=2
) )
......
experiments/pyexps/nopaxos.py
View file @ 78d4c701
...@@ -56,10 +56,10 @@ for proto_config in proto_configs: ...@@ -56,10 +56,10 @@ for proto_config in proto_configs:
# host # host
if host_config == 'qemu': if host_config == 'qemu':
host_class = sim.QemuHost HostClass = sim.QemuHost
net.sync = False net.sync = False
elif host_config == 'gt': elif host_config == 'gt':
host_class = sim.Gem5Host HostClass = sim.Gem5Host
e.checkpoint = True e.checkpoint = True
elif host_config == 'qt': elif host_config == 'qt':
...@@ -68,30 +68,30 @@ for proto_config in proto_configs: ...@@ -68,30 +68,30 @@ for proto_config in proto_configs:
h.sync = True h.sync = True
return h return h
host_class = qemu_timing HostClass = qemu_timing
else: else:
raise NameError(host_config) raise NameError(host_config)
# nic # nic
if nic_config == 'ib': if nic_config == 'ib':
nic_class = sim.I40eNIC NicClass = sim.I40eNIC
nc_class = node.I40eLinuxNode NcClass = node.I40eLinuxNode
elif nic_config == 'cb': elif nic_config == 'cb':
nic_class = sim.CorundumBMNIC NicClass = sim.CorundumBMNIC
nc_class = node.CorundumLinuxNode NcClass = node.CorundumLinuxNode
elif nic_config == 'cv': elif nic_config == 'cv':
nic_class = sim.CorundumVerilatorNIC NicClass = sim.CorundumVerilatorNIC
nc_class = node.CorundumLinuxNode NcClass = node.CorundumLinuxNode
else: else:
raise NameError(nic_config) raise NameError(nic_config)
# app # app
if proto_config == 'vr': if proto_config == 'vr':
replica_class = node.VRReplica ReplicaClass = node.VRReplica
client_class = node.VRClient ClientClass = node.VRClient
elif proto_config == 'nopaxos': elif proto_config == 'nopaxos':
replica_class = node.NOPaxosReplica ReplicaClass = node.NOPaxosReplica
client_class = node.NOPaxosClient ClientClass = node.NOPaxosClient
else: else:
raise NameError(proto_config) raise NameError(proto_config)
...@@ -102,9 +102,9 @@ for proto_config in proto_configs: ...@@ -102,9 +102,9 @@ for proto_config in proto_configs:
1, 1,
'sequencer', 'sequencer',
net, net,
nic_class, NicClass,
host_class, HostClass,
nc_class, NcClass,
node.NOPaxosSequencer, node.NOPaxosSequencer,
ip_start=100 ip_start=100
) )
...@@ -117,10 +117,10 @@ for proto_config in proto_configs: ...@@ -117,10 +117,10 @@ for proto_config in proto_configs:
3, 3,
'replica', 'replica',
net, net,
nic_class, NicClass,
host_class, HostClass,
nc_class, NcClass,
replica_class ReplicaClass
) )
for i in range(len(replicas)): for i in range(len(replicas)):
replicas[i].node_config.app.index = i replicas[i].node_config.app.index = i
...@@ -133,10 +133,10 @@ for proto_config in proto_configs: ...@@ -133,10 +133,10 @@ for proto_config in proto_configs:
num_c, num_c,
'client', 'client',
net, net,
nic_class, NicClass,
host_class, HostClass,
nc_class, NcClass,
client_class, ClientClass,
ip_start=4 ip_start=4
) )
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment