Skip to content

GitLab

Commit 7549fa9e authored by Jonas Kaufmann's avatar Jonas Kaufmann Committed by Antoine Kaufmann
Browse files

pre-commit run -a

parent aac98df8
Hide whitespace changes
Inline Side-by-side
Showing with 206 additions and 158 deletions
experiments/simbricks/runtime/common.py
View file @ 7549fa9e
......@@ -23,15 +23,16 @@
# Allow own class to be used as type for a method's argument
from __future__ import annotations
from abc import ABCMeta, abstractmethod
import shutil
import pathlib
import shutil
import typing as tp
from abc import ABCMeta, abstractmethod
import simbricks.exectools as exectools
from simbricks.experiment.experiment_environment import ExpEnv
from simbricks.experiment.experiment_output import ExpOutput
from simbricks.experiments import Experiment
import simbricks.exectools as exectools
class Run(object):
......@@ -82,4 +83,4 @@ class Runtime(metaclass=ABCMeta):
@abstractmethod
def start(self):
pass
\ No newline at end of file
pass
experiments/simbricks/runtime/distributed.py
View file @ 7549fa9e
......@@ -23,12 +23,15 @@
import asyncio
import pathlib
import simbricks.proxy as proxy
from simbricks.exectools import Executor
from simbricks.runtime.common import *
import simbricks.experiments as exp
import simbricks.proxy as proxy
class DistributedSimpleRuntime(Runtime):
def __init__(self, execs, verbose=False):
self.runnable = []
self.complete = []
......@@ -42,8 +45,9 @@ class DistributedSimpleRuntime(Runtime):
self.runnable.append(run)
async def do_run(self, run: Run):
runner = exp.ExperimentDistributedRunner(self.execs, run.experiment,
run.env, self.verbose)
runner = exp.ExperimentDistributedRunner(
self.execs, run.experiment, run.env, self.verbose
)
for exec in self.execs:
await run.prep_dirs(exec)
await runner.prepare()
......@@ -62,9 +66,11 @@ class DistributedSimpleRuntime(Runtime):
def auto_dist(
e: Experiment, execs: tp.List[Executor], proxy_type: str = 'sockets'
):
""" Converts an Experiment into a DistributedExperiment. Assigns network to
executor zero, and then round-robin assignment of hosts to executors,
while also assigning all nics for a host to the same executor.
"""
Converts an Experiment into a DistributedExperiment.
Assigns network to executor zero, and then round-robin assignment of hosts
to executors, while also assigning all nics for a host to the same executor.
"""
if len(execs) < 2:
......@@ -120,4 +126,4 @@ def auto_dist(
for nic in e.nics: # TODO: e.nics does not exist in class Experiment
de.add_nic(nic)
return de
\ No newline at end of file
return de
experiments/simbricks/runtime/local.py
View file @ 7549fa9e
......@@ -24,9 +24,10 @@ import asyncio
import pathlib
from typing import Optional
import simbricks.exectools as exectools
from simbricks.runtime.common import *
import simbricks.experiments as exp
import simbricks.exectools as exectools
class LocalSimpleRuntime(Runtime):
......
experiments/simbricks/runtime/slurm.py
View file @ 7549fa9e
......@@ -20,14 +20,16 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import pickle
import os
import pathlib
import pickle
import re
from simbricks.runtime.common import *
class SlurmRuntime(Runtime):
def __init__(self, slurmdir, args, verbose=False, cleanup=True):
self.runnable = []
self.slurmdir = slurmdir
......@@ -42,7 +44,7 @@ class SlurmRuntime(Runtime):
exp = run.experiment
e_idx = exp.name + f'-{run.index}' + '.exp'
exp_path = os.path.join(self.slurmdir, e_idx)
log_idx = exp.name + f'-{run.index}' + '.log'
exp_log = os.path.join(self.slurmdir, log_idx)
......@@ -51,10 +53,10 @@ class SlurmRuntime(Runtime):
print(exp_path)
print(exp_log)
print(exp_script)
# write out pickled run
with open(exp_path, 'wb') as f:
run.prereq = None # we don't want to pull in the prereq too
run.prereq = None # we don't want to pull in the prereq too
pickle.dump(run, f)
# create slurm batch script
......
experiments/simbricks/simulator_utils.py
View file @ 7549fa9e
......@@ -23,9 +23,10 @@
import typing as tp
from simbricks.nodeconfig import AppConfig, NodeConfig
from simbricks.simulators import HostSim, I40eMultiNIC, NetSim, NICSim
from simbricks.experiments import Experiment
from simbricks.nodeconfig import NodeConfig, AppConfig
from simbricks.simulators import NetSim, NICSim, HostSim, I40eMultiNIC
def create_basic_hosts(
......@@ -40,8 +41,10 @@ def create_basic_hosts(
ip_start: int = 1,
ip_prefix: int = 24
):
"""Creates and configures multiple hosts to be simulated using the given parameters.
"""
Creates and configures multiple hosts to be simulated using the given
parameters.
Args:
`num`: number of hosts to create
"""
......@@ -82,8 +85,10 @@ def create_multinic_hosts(
ip_start: int = 1,
ip_prefix: int = 24
):
"""Creates and configures multiple hosts to be simulated using the given parameters. These hosts use multiple NICs.
"""
Creates and configures multiple hosts to be simulated using the given
parameters. These hosts use multiple NICs.
Args:
num: number of hosts to create
"""
......@@ -130,8 +135,10 @@ def create_dctcp_hosts(
mtu: int,
ip_start: int = 1
):
"""Creates and configures multiple hosts to be simulated in a DCTCP experiment using the given parameters.
"""
Creates and configures multiple hosts to be simulated in a DCTCP experiment
using the given parameters.
Args:
num: number of hosts to create
cpu_freq: CPU frequency to simulate, e.g. '5GHz'
......@@ -158,4 +165,4 @@ def create_dctcp_hosts(
hosts.append(host)
return hosts
\ No newline at end of file
return hosts
experiments/simbricks/simulators.py
View file @ 7549fa9e
......@@ -26,8 +26,8 @@ from __future__ import annotations
import math
import typing as tp
from simbricks.nodeconfig import NodeConfig
from simbricks.experiment.experiment_environment import ExpEnv
from simbricks.nodeconfig import NodeConfig
class Simulator(object):
......@@ -134,6 +134,7 @@ class NICSim(PCIDevSim):
def sockets_wait(self, env):
return super().sockets_wait(env) + [env.nic_eth_path(self)]
class NetSim(Simulator):
"""Base class for network simulators."""
name = ''
......@@ -174,10 +175,10 @@ class NetSim(Simulator):
return self.nics + self.net_connect + self.hosts_direct
def sockets_cleanup(self, env: ExpEnv):
return [s for (_,s) in self.listen_sockets(env)]
return [s for (_, s) in self.listen_sockets(env)]
def sockets_wait(self, env: ExpEnv):
return [s for (_,s) in self.listen_sockets(env)]
return [s for (_, s) in self.listen_sockets(env)]
class HostSim(Simulator):
......@@ -248,21 +249,25 @@ class QemuHost(HostSim):
def prep_cmds(self, env):
to_path = env.hdcopy_path(self)
return [f'{env.qemu_img_path} create -f qcow2 -o '
return [
f'{env.qemu_img_path} create -f qcow2 -o '
f'backing_file="{env.hd_path(self.node_config.disk_image)}" '
f'{env.hdcopy_path(self)}']
f'{env.hdcopy_path(self)}'
]
def run_cmd(self, env):
accel = ',accel=kvm:tcg' if not self.sync else ''
cmd = (f'{env.qemu_path} -machine q35{accel} -serial mon:stdio '
cmd = (
f'{env.qemu_path} -machine q35{accel} -serial mon:stdio '
'-cpu Skylake-Server -display none -nic none '
f'-kernel {env.qemu_kernel_path} '
f'-drive file={env.hdcopy_path(self)},if=ide,index=0,media=disk '
f'-drive file={env.cfgtar_path(self)},if=ide,index=1,media=disk,'
'driver=raw '
'driver=raw '
'-append "earlyprintk=ttyS0 console=ttyS0 root=/dev/sda1 '
'init=/home/ubuntu/guestinit.sh rw" '
f'-m {self.node_config.memory} -smp {self.node_config.cores} ')
'init=/home/ubuntu/guestinit.sh rw" '
f'-m {self.node_config.memory} -smp {self.node_config.cores} '
)
if self.sync:
unit = self.cpu_freq[-3:]
......@@ -288,9 +293,10 @@ class QemuHost(HostSim):
cmd += ' '
# qemu does not currently support net direct ports
assert(len(self.net_directs) == 0)
assert (len(self.net_directs) == 0)
return cmd
class Gem5Host(HostSim):
cpu_type_cp = 'X86KvmCPU'
cpu_type = 'TimingSimpleCPU'
......@@ -322,7 +328,8 @@ class Gem5Host(HostSim):
cmd = f'{env.gem5_path(self.variant)} --outdir={env.gem5_outdir(self)} '
cmd += ' '.join(self.extra_main_args)
cmd += (f' {env.gem5_py_path} --caches --l2cache --l3cache '
cmd += (
f' {env.gem5_py_path} --caches --l2cache --l3cache '
'--l1d_size=32kB --l1i_size=32kB --l2_size=2MB --l3_size=32MB '
'--l1d_assoc=8 --l1i_assoc=8 --l2_assoc=4 --l3_assoc=16 '
f'--cacheline_size=64 --cpu-clock={self.cpu_freq} --sys-clock={self.sys_clock} '
......@@ -332,7 +339,8 @@ class Gem5Host(HostSim):
f'--disk-image={env.cfgtar_path(self)} '
f'--cpu-type={cpu_type} --mem-size={self.node_config.memory}MB '
f'--num-cpus={self.node_config.cores} '
'--ddio-enabled --ddio-way-part=8 --mem-type=DDR4_2400_16x4 ')
'--ddio-enabled --ddio-way-part=8 --mem-type=DDR4_2400_16x4 '
)
if env.create_cp:
cmd += '--max-checkpoints=1 '
......@@ -341,30 +349,33 @@ class Gem5Host(HostSim):
cmd += '-r 1 '
for dev in self.pcidevs:
cmd += (f'--simbricks-pci=connect:{env.dev_pci_path(dev)}'
f':latency={self.pci_latency}ns'
f':sync_interval={self.sync_period}ns')
cmd += (
f'--simbricks-pci=connect:{env.dev_pci_path(dev)}'
f':latency={self.pci_latency}ns'
f':sync_interval={self.sync_period}ns'
)
if cpu_type == 'TimingSimpleCPU':
cmd += ':sync'
cmd +=' '
cmd += ' '
for net in self.net_directs:
cmd += ('--simbricks-eth-e1000=listen'
f':{env.net2host_eth_path(net, self)}'
f':{env.net2host_shm_path(net, self)}'
f':latency={net.eth_latency}ns'
f':sync_interval={net.sync_period}ns')
cmd += (
'--simbricks-eth-e1000=listen'
f':{env.net2host_eth_path(net, self)}'
f':{env.net2host_shm_path(net, self)}'
f':latency={net.eth_latency}ns'
f':sync_interval={net.sync_period}ns'
)
if cpu_type == 'TimingSimpleCPU':
cmd += ':sync'
cmd +=' '
cmd += ' '
cmd += ' '.join(self.extra_config_args)
return cmd
class CorundumVerilatorNIC(NICSim):
clock_freq = 250 # MHz
clock_freq = 250 # MHz
def __init__(self):
super().__init__()
......@@ -374,25 +385,32 @@ class CorundumVerilatorNIC(NICSim):
return 512
def run_cmd(self, env):
return self.basic_run_cmd(env, '/corundum/corundum_verilator',
str(self.clock_freq))
return self.basic_run_cmd(
env, '/corundum/corundum_verilator', str(self.clock_freq)
)
class CorundumBMNIC(NICSim):
def __init__(self):
super().__init__()
def run_cmd(self, env):
return self.basic_run_cmd(env, '/corundum_bm/corundum_bm')
class I40eNIC(NICSim):
def __init__(self):
super().__init__()
def run_cmd(self, env):
return self.basic_run_cmd(env, '/i40e_bm/i40e_bm')
class E1000NIC(NICSim):
debug = False
def __init__(self):
super().__init__()
......@@ -402,6 +420,7 @@ class E1000NIC(NICSim):
cmd = 'env E1000_DEBUG=1 ' + cmd
return cmd
class MultiSubNIC(NICSim):
name = ''
multinic = None
......@@ -419,6 +438,7 @@ class MultiSubNIC(NICSim):
def start_delay(self):
return 0
class I40eMultiNIC(Simulator):
name = ''
......@@ -458,6 +478,7 @@ class I40eMultiNIC(Simulator):
class WireNet(NetSim):
def __init__(self):
super().__init__()
......@@ -472,6 +493,7 @@ class WireNet(NetSim):
cmd += ' ' + env.pcap_file
return cmd
class SwitchNet(NetSim):
sync = True
......@@ -487,9 +509,9 @@ class SwitchNet(NetSim):
if len(env.pcap_file) > 0:
cmd += ' -p ' + env.pcap_file
for (_,n) in self.connect_sockets(env):
for (_, n) in self.connect_sockets(env):
cmd += ' -s ' + n
for (_,n) in self.listen_sockets(env):
for (_, n) in self.listen_sockets(env):
cmd += ' -h ' + n
return cmd
......@@ -502,6 +524,7 @@ class SwitchNet(NetSim):
cleanup.append(s + '-shm')
return cleanup
class TofinoNet(NetSim):
tofino_log_path = '/tmp/model.ldjson'
sync = True
......@@ -514,17 +537,19 @@ class TofinoNet(NetSim):
cmd += f' -S {self.sync_period} -E {self.eth_latency} -t {self.tofino_log_path}'
if not self.sync:
cmd += ' -u'
for (_,n) in self.connect_sockets(env):
for (_, n) in self.connect_sockets(env):
cmd += ' -s ' + n
return cmd
class NS3DumbbellNet(NetSim):
def __init__(self):
super().__init__()
def run_cmd(self, env):
ports = ''
for (n,s) in self.connect_sockets(env):
for (n, s) in self.connect_sockets(env):
if 'server' in n.name:
ports += '--CosimPortLeft=' + s + ' '
else:
......@@ -535,13 +560,15 @@ class NS3DumbbellNet(NetSim):
return cmd
class NS3BridgeNet(NetSim):
def __init__(self):
super().__init__()
def run_cmd(self, env):
ports = ''
for (_,n) in self.connect_sockets(env):
for (_, n) in self.connect_sockets(env):
ports += '--CosimPort=' + n + ' '
cmd = env.repodir + '/sims/external/ns-3' + '/cosim-run.sh cosim cosim-bridge-example ' + ports + ' ' + self.opt
......@@ -549,13 +576,15 @@ class NS3BridgeNet(NetSim):
return cmd
class NS3SequencerNet(NetSim):
def __init__(self):
super().__init__()
def run_cmd(self, env):
ports = ''
for (n,s) in self.connect_sockets(env):
for (n, s) in self.connect_sockets(env):
if 'client' in n.name:
ports += '--ClientPort=' + s + ' '
elif 'replica' in n.name:
......@@ -569,6 +598,7 @@ class NS3SequencerNet(NetSim):
class FEMUDev(PCIDevSim):
def __init__(self):
super().__init__()
......
results/ScaleHost.py
View file @ 7549fa9e
......@@ -20,11 +20,11 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import sys
import json
import os
import pathlib
import shutil
import json
import sys
if len(sys.argv) != 2:
print('Usage: udp_scale.py OUTDIR')
......@@ -43,12 +43,10 @@ for cl in types_of_client:
diff_time = ''
else:
exp_log = json.load(log)
start_time = exp_log["start_time"]
end_time = exp_log["end_time"]
diff_time = (end_time - start_time)/60 #min
start_time = exp_log['start_time']
end_time = exp_log['end_time']
diff_time = (end_time - start_time) / 60 #min
diff_time = str(diff_time)
log.close()
print('%d\t%s' % (cl, diff_time))
results/ScaleLoad.py
View file @ 7549fa9e
......@@ -20,12 +20,12 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import sys
import json
import math
import os
import pathlib
import shutil
import json
import math
import sys
num_runs = 3
if len(sys.argv) != 2:
......@@ -42,14 +42,14 @@ for bw in types_of_bw:
avg_time = 0
std = 0
all_time = []
for i in range(1, num_runs+1):
for i in range(1, num_runs + 1):
log_path = '%sgt-ib-sw-Load-%dm-%d.json' % (basedir, bw, i)
if os.path.exists(log_path):
log = open(log_path, 'r')
exp_log = json.load(log)
start_time = exp_log["start_time"]
end_time = exp_log["end_time"]
diff_time = (end_time - start_time)/60 #min
start_time = exp_log['start_time']
end_time = exp_log['end_time']
diff_time = (end_time - start_time) / 60 #min
total_time += diff_time
all_time.append(diff_time)
diff_time = str(diff_time)
......@@ -60,16 +60,13 @@ for bw in types_of_bw:
#print('%d\t%s' % (bw, diff_time))
avg_time = total_time/num_runs
avg_time = total_time / num_runs
#print('avg_time: ' + str(avg_time))
for i in range(0, num_runs):
std += (all_time[i] - avg_time) * (all_time[i] - avg_time)
std = std/num_runs
std = std / num_runs
std = math.sqrt(std)
#print(str(std))
print('%d %s %f' % (bw, avg_time, std))
results/dctcp.py
View file @ 7549fa9e
......@@ -22,6 +22,7 @@
import itertools
import sys
import utils.iperf
if len(sys.argv) != 2:
......@@ -39,7 +40,6 @@ configs = list(itertools.product(types_of_host, mtus))
confignames = [h + '-' + str(mtu) for h, mtu in configs]
print('\t'.join(['threshold'] + confignames))
for k_val in range(0, max_k + 1, k_step):
line = [str(k_val)]
for h, mtu in configs:
......
results/dist_memcache.py
View file @ 7549fa9e
......@@ -19,12 +19,15 @@
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
""" Generates data file for dist_memcache scalability graph. First column is
the number of hosts, second column the qemu timing simulation time in hours,
and the third column is the gem5 simulation time."""
"""
Generates data file for dist_memcache scalability graph.
First column is the number of hosts, second column the qemu timing simulation
time in hours, and the third column is the gem5 simulation time.
"""
import sys
import json
import sys
if len(sys.argv) != 2:
print('Usage: dist_memcache.py OUTDIR')
......@@ -36,22 +39,21 @@ racks = [1, 5, 10, 15, 25]
host_types = ['qt', 'gem5']
for n_racks in racks:
l = str(n_racks * n_hosts_per_rack)
for host_type in host_types:
log_path = '%sdist_memcache-%s-%d-%d-1.json' % (basedir, host_type,
n_racks, n_hosts_per_rack)
try:
log = open(log_path, 'r')
except:
diff_time = ''
else:
exp_log = json.load(log)
start_time = exp_log["start_time"]
end_time = exp_log["end_time"]
diff_time = float(end_time - start_time)/60/60
l += '\t' + str(diff_time)
print(l)
l = str(n_racks * n_hosts_per_rack)
for host_type in host_types:
log_path = '%sdist_memcache-%s-%d-%d-1.json' % (
basedir, host_type, n_racks, n_hosts_per_rack
)
try:
log = open(log_path, 'r')
except:
diff_time = ''
else:
exp_log = json.load(log)
start_time = exp_log['start_time']
end_time = exp_log['end_time']
diff_time = float(end_time - start_time) / 60 / 60
l += '\t' + str(diff_time)
print(l)
results/modetcp.py
View file @ 7549fa9e
......@@ -20,23 +20,23 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import sys
import json
import os
import pathlib
import shutil
import json
import sys
# How to use
# $ python3 modetcp.py paper_data/modetcp
# $ python3 modetcp.py paper_data/modetcp
#
mode = ['0', '1']
nics = ['cb', 'cv', 'ib']
num_client = ['1', '4']
outdir = sys.argv[1]
def parse_sim_time(path):
ret = {}
if not os.path.exists(path):
......@@ -44,7 +44,7 @@ def parse_sim_time(path):
with open(path, 'r') as f:
data = json.load(f)
ret['simtime'] = (data['end_time'] - data['start_time'])/60
ret['simtime'] = (data['end_time'] - data['start_time']) / 60
f.close()
return ret
......@@ -62,5 +62,3 @@ for c in num_client:
t = ''
line = line + ' ' + f'{t}'
print(line)
results/multi-inst-avgtime.py
View file @ 7549fa9e
......@@ -20,21 +20,21 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import sys
import json
import os
import pathlib
import shutil
import json
import sys
# How to use
# $ python3 modetcp.py paper_data/modetcp
# $ python3 modetcp.py paper_data/modetcp
#
num_runs = 8
outdir = sys.argv[1]
def parse_sim_time(path):
ret = {}
if not os.path.exists(path):
......@@ -42,13 +42,14 @@ def parse_sim_time(path):
with open(path, 'r') as f:
data = json.load(f)
ret['simtime'] = (data['end_time'] - data['start_time'])/60
ret['simtime'] = (data['end_time'] - data['start_time']) / 60
f.close()
return ret
total_time = 0
for n in range (1, num_runs+1):
for n in range(1, num_runs + 1):
path = '%s/p8-gt-ib-sw-Host-1000m-1-%s.json' % (outdir, n)
data = parse_sim_time(path)
......@@ -58,10 +59,5 @@ for n in range (1, num_runs+1):
else:
t = ''
print(t)
avg = total_time/num_runs
avg = total_time / num_runs
print('avg: ' + str(avg))
results/netperf.py
View file @ 7549fa9e
......@@ -20,10 +20,10 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
from utils.netperf import *
import sys
from time import strftime
from time import gmtime
from time import gmtime, strftime
from utils.netperf import *
def fmt_lat(lat):
......@@ -60,7 +60,7 @@ for (h, h_l) in hosts:
path = '%s/netperf-%s-%s-%s-1.json' % (outdir, h, net, nic)
data = parse_netperf_run(path)
if 'simtime' in data:
t = strftime("%H:%M:%S", gmtime(data['simtime']))
t = strftime('%H:%M:%S', gmtime(data['simtime']))
else:
t = ''
......
results/nopaxos.py
View file @ 7549fa9e
......@@ -22,6 +22,7 @@
import itertools
import sys
import utils.parse_nopaxos
if len(sys.argv) != 2:
......@@ -33,14 +34,14 @@ basedir = sys.argv[1] + '/'
types_of_seq = ['ehseq', 'swseq']
num_clients = [1, 2, 3, 4, 5, 6, 7, 8, 10, 12]
print('num_client ehseq-tput(req/sec) ehseq-lat(us) swseq-tput(req/sec) swseq-lat(us)\n')
print(
'num_client ehseq-tput(req/sec) ehseq-lat(us) swseq-tput(req/sec) swseq-lat(us)\n'
)
for num_c in num_clients:
line = [str(num_c)]
for seq in types_of_seq:
path_pat = '%snopaxos-gt-ib-%s-%d-1.json' % (basedir, seq, num_c)
res = utils.parse_nopaxos.parse_nopaxos_run(num_c, seq, path_pat)
#print(path_pat)
......@@ -57,5 +58,4 @@ for num_c in num_clients:
line.append('%.2f' % (tput))
line.append(f'{lat}')
print(' '.join(line))
results/paper_data/utils/iperf.py
View file @ 7549fa9e
......@@ -20,16 +20,19 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import fnmatch
import glob
import itertools
import json
import os
import fnmatch
import re
import itertools
import sys
def parse_iperf_run(data, skip=1, use=8):
tp_pat = re.compile(r'\[ *\d*\] *([0-9\.]*)- *([0-9\.]*) sec.*Bytes *([0-9\.]*) ([GM])bits.*')
tp_pat = re.compile(
r'\[ *\d*\] *([0-9\.]*)- *([0-9\.]*) sec.*Bytes *([0-9\.]*) ([GM])bits.*'
)
tps_time = {}
for hn in fnmatch.filter(data['sims'].keys(), 'host.client.*'):
sim = data['sims'][hn]
......@@ -50,7 +53,7 @@ def parse_iperf_run(data, skip=1, use=8):
if m.group(4) == 'G':
tps_time[time].append(float(m.group(3)))
elif m.group(4) == 'M':
m_tps = float(m.group(3))/1000
m_tps = float(m.group(3)) / 1000
tps_time[time].append(m_tps)
tps = []
......@@ -58,11 +61,11 @@ def parse_iperf_run(data, skip=1, use=8):
x = sum(tps_time[t])
tps.append(x)
if len(tps) == 0:
return None
return sum(tps) / len(tps)
def parse_iperf(basename, skip=1, use=8):
runs = []
for path in glob.glob(basename + '-*.json'):
......@@ -79,6 +82,7 @@ def parse_iperf(basename, skip=1, use=8):
if not runs:
return {'avg': None, 'min': None, 'max': None}
else:
return {'avg': sum(runs) / len(runs), 'min': min(runs),
'max': max(runs)}
return {
'avg': sum(runs) / len(runs), 'min': min(runs), 'max': max(runs)
}
result = {}
results/pci_validation.py
View file @ 7549fa9e
......@@ -20,9 +20,10 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import sys
import re
import json
import re
import sys
def transform_internal(ts, component, msg):
if not component.startswith('system.pc.simbricks_0'):
......@@ -40,6 +41,7 @@ def transform_internal(ts, component, msg):
return (ts + ' ' + msg)
def transform_external(ts, component, msg):
if msg.startswith('igbe: requesting restart clock:') or \
msg == 'igbe: scheduled' or \
......@@ -49,9 +51,9 @@ def transform_external(ts, component, msg):
elif msg.startswith('[rxdesc]') or msg.startswith('[txdesc]'):
msg = msg[9:]
return (ts + ' ' + msg)
if len(sys.argv) != 3:
print('Usage: pci_validation.py JSON-DIR VARIANT')
print(' VARIANT can be internal or external')
......
results/pcilat.py
View file @ 7549fa9e
......@@ -22,6 +22,7 @@
import itertools
import sys
import utils.iperf
if len(sys.argv) != 2:
......@@ -41,7 +42,7 @@ configs = [
print('\t'.join(['config'] + list(map(str, lats))))
for (ht,nt,lab) in configs:
for (ht, nt, lab) in configs:
cols = [str(lab)]
for lat in lats:
path_pat = '%spcilat-%s-%s-switch-%d' % (basedir, ht, nt, lat)
......
results/sync-overhead.py
View file @ 7549fa9e
......@@ -20,22 +20,22 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import sys
import json
import os
import pathlib
import shutil
import json
import sys
# How to use
# $ python3 modetcp.py paper_data/modetcp
# $ python3 modetcp.py paper_data/modetcp
#
mode = ['no_simb-gt', 'noTraf-gt-ib-sw']
cmd = ['sleep', 'busy']
outdir = sys.argv[1]
def parse_sim_time(path):
ret = {}
if not os.path.exists(path):
......@@ -43,7 +43,7 @@ def parse_sim_time(path):
with open(path, 'r') as f:
data = json.load(f)
ret['simtime'] = (data['end_time'] - data['start_time'])/60
ret['simtime'] = (data['end_time'] - data['start_time']) / 60
f.close()
return ret
......@@ -60,5 +60,3 @@ for m in mode:
t = ''
line = line + ' ' + f'{t}'
print(line)
results/utils/iperf.py
View file @ 7549fa9e
......@@ -20,16 +20,19 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import fnmatch
import glob
import itertools
import json
import os
import fnmatch
import re
import itertools
import sys
def parse_iperf_run(data, skip=1, use=8):
tp_pat = re.compile(r'\[ *\d*\] *([0-9\.]*)- *([0-9\.]*) sec.*Bytes *([0-9\.]*) ([GM])bits.*')
tp_pat = re.compile(
r'\[ *\d*\] *([0-9\.]*)- *([0-9\.]*) sec.*Bytes *([0-9\.]*) ([GM])bits.*'
)
tps_time = {}
for hn in fnmatch.filter(data['sims'].keys(), 'host.client.*'):
sim = data['sims'][hn]
......@@ -50,7 +53,7 @@ def parse_iperf_run(data, skip=1, use=8):
if m.group(4) == 'G':
tps_time[time].append(float(m.group(3)))
elif m.group(4) == 'M':
m_tps = float(m.group(3))/1000
m_tps = float(m.group(3)) / 1000
tps_time[time].append(m_tps)
tps = []
......@@ -58,11 +61,11 @@ def parse_iperf_run(data, skip=1, use=8):
x = sum(tps_time[t])
tps.append(x)
if len(tps) == 0:
return None
return sum(tps) / len(tps)
def parse_iperf(basename, skip=1, use=8):
runs = []
for path in glob.glob(basename + '-*.json'):
......@@ -79,6 +82,7 @@ def parse_iperf(basename, skip=1, use=8):
if not runs:
return {'avg': None, 'min': None, 'max': None}
else:
return {'avg': sum(runs) / len(runs), 'min': min(runs),
'max': max(runs)}
return {
'avg': sum(runs) / len(runs), 'min': min(runs), 'max': max(runs)
}
result = {}
results/utils/netperf.py
View file @ 7549fa9e
......@@ -21,8 +21,9 @@
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import json
import re
import os
import re
def parse_netperf_run(path):
ret = {}
......@@ -50,8 +51,6 @@ def parse_netperf_run(path):
m = tp_pat.match(tp_line)
ret['throughput'] = float(m.group(1))
lath_pat = re.compile(r'\s*Mean Latency.*')
start = None
i = 0
......@@ -64,7 +63,9 @@ def parse_netperf_run(path):
if start is not None:
lat_line = lines[start + 1]
lat_pat = re.compile(r'\s*([-0-9\.]*),([-0-9\.]*),([-0-9\.]*),([-0-9\.]*).*')
lat_pat = re.compile(
r'\s*([-0-9\.]*),([-0-9\.]*),([-0-9\.]*),([-0-9\.]*).*'
)
m = lat_pat.match(lat_line)
ret['latenyMean'] = float(m.group(1))
ret['latenyTail'] = float(m.group(4))
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment