Skip to content

GitLab

"git@developer.sourcefind.cn:wangsen/mineru.git" did not exist on "f4316f02d32ffa2095101bc8287006aa8812d479"
Commit 05fe9ca5 authored by Jonas Kaufmann's avatar Jonas Kaufmann Committed by Antoine Kaufmann
Browse files

fix pylint issues in results/

parent fc3f0602
Hide whitespace changes
Inline Side-by-side
Showing with 109 additions and 128 deletions
results/dctcp.py
View file @ 05fe9ca5
......@@ -43,7 +43,7 @@ print('\t'.join(['threshold'] + confignames))
for k_val in range(0, max_k + 1, k_step):
line = [str(k_val)]
for h, mtu in configs:
path_pat = '%s%s-ib-dumbbell-DCTCPm%d-%d' % (basedir, h, k_val, mtu)
path_pat = f'{basedir}{h}-ib-dumbbell-DCTCPm{k_val}-{mtu}'
res = utils.iperf.parse_iperf(path_pat)
if res['avg'] is None:
......@@ -52,11 +52,12 @@ for k_val in range(0, max_k + 1, k_step):
tp = res['avg']
# TP * (MTU ) / (MTU - IP (20) - TCP w/option (24))
if (h == 'gt' or h == 'qt'):
if h in ('gt', 'qt'):
tp_calib = tp * (mtu) / (mtu - 20 - 24)
else:
# TP * (MTU + ETH(14) + PHY(24)) / (MTU - IP (20) - TCP w/option (24))
# TP * (MTU + ETH(14) + PHY(24)) / (MTU - IP (20)
# - TCP w/option (24))
tp_calib = tp * (mtu + 14 + 24) / (mtu - 20 - 24)
line.append('%.2f' % (tp_calib))
line.append(f'{tp_calib:.2f}')
print('\t'.join(line))
results/dist_memcache.py
View file @ 05fe9ca5
......@@ -28,6 +28,7 @@ time in hours, and the third column is the gem5 simulation time.
import json
import sys
from os.path import exists
if len(sys.argv) != 2:
print('Usage: dist_memcache.py OUTDIR')
......@@ -41,18 +42,18 @@ host_types = ['qt', 'gem5']
for n_racks in racks:
l = str(n_racks * n_hosts_per_rack)
for host_type in host_types:
log_path = '%sdist_memcache-%s-%d-%d-1.json' % (
basedir, host_type, n_racks, n_hosts_per_rack
log_path = (
f'{basedir}dist_memcache-{host_type}-{n_racks}-{n_hosts_per_rack}'
'-1.json'
)
try:
log = open(log_path, 'r')
except:
diff_time = ''
if exists(log):
with open(log_path, 'r', encoding='utf-8') as log:
exp_log = json.load(log)
start_time = exp_log['start_time']
end_time = exp_log['end_time']
diff_time = float(end_time - start_time) / 60 / 60
else:
exp_log = json.load(log)
start_time = exp_log['start_time']
end_time = exp_log['end_time']
diff_time = float(end_time - start_time) / 60 / 60
diff_time = ''
l += '\t' + str(diff_time)
......
results/modetcp.py
View file @ 05fe9ca5
......@@ -22,8 +22,6 @@
import json
import os
import pathlib
import shutil
import sys
# How to use
......@@ -37,11 +35,12 @@ num_client = ['1', '4']
outdir = sys.argv[1]
# pylint: disable=redefined-outer-name
def parse_sim_time(path):
ret = {}
if not os.path.exists(path):
return ret
with open(path, 'r') as f:
with open(path, 'r', encoding='utf-8') as f:
data = json.load(f)
ret['simtime'] = (data['end_time'] - data['start_time']) / 60
......@@ -50,15 +49,12 @@ def parse_sim_time(path):
for c in num_client:
print('%s-client ModES Epoch' % (c))
print(f'{c}-client ModES Epoch')
for n in nics:
line = f'{n}'
for m in mode:
path = '%s/mode-%s-gt-%s-switch-%s-1.json' % (outdir, m, n, c)
path = f'{outdir}/mode-{m}-gt-{n}-switch-{c}-1.json'
data = parse_sim_time(path)
if 'simtime' in data:
t = data['simtime']
else:
t = ''
line = line + ' ' + f'{t}'
t = data.get('simtime', '')
line = f'{line} {t}'
print(line)
results/multi-inst-avgtime.py → results/multi_inst_avgtime.py
View file @ 05fe9ca5
......@@ -22,8 +22,6 @@
import json
import os
import pathlib
import shutil
import sys
# How to use
......@@ -35,11 +33,12 @@ num_runs = 8
outdir = sys.argv[1]
# pylint: disable=redefined-outer-name
def parse_sim_time(path):
ret = {}
if not os.path.exists(path):
return ret
with open(path, 'r') as f:
with open(path, 'r', encoding='utf-8') as f:
data = json.load(f)
ret['simtime'] = (data['end_time'] - data['start_time']) / 60
......@@ -51,7 +50,7 @@ total_time = 0
for n in range(1, num_runs + 1):
path = '%s/p8-gt-ib-sw-Host-1000m-1-%s.json' % (outdir, n)
path = f'{outdir}/p8-gt-ib-sw-Host-1000m-1-{n}.json' % (outdir, n)
data = parse_sim_time(path)
if 'simtime' in data:
t = data['simtime']
......
results/netperf.py
View file @ 05fe9ca5
......@@ -23,7 +23,7 @@
import sys
from time import gmtime, strftime
from utils.netperf import *
from utils.netperf import parse_netperf_run
def fmt_lat(lat):
......@@ -32,20 +32,21 @@ def fmt_lat(lat):
x = float(lat)
if x >= 1000.:
return '%.1f\\,ms' % (x / 1000)
return f'{x / 1000:.1f}\\,ms'
else:
return '%d\\,$\\mu$s' % (int(x))
return f'{int(x)}\\,$\\mu$s'
# pylint: disable=redefined-outer-name
def fmt_tp(tp):
if not tp:
return ''
x = float(tp)
if x > 1000.:
return '%.2f\\,G' % (x / 1000)
return f'{x / 1000:.2f}\\,G'
else:
return '%d\\,M' % (int(x))
return f'{int(x)}\\,M'
hosts = [('qemu', 'QK'), ('qt', 'QT'), ('gem5', 'G5')]
......@@ -57,7 +58,7 @@ outdir = sys.argv[1]
for (h, h_l) in hosts:
for (nic, nic_l) in nics:
for (net, net_l) in nets:
path = '%s/netperf-%s-%s-%s-1.json' % (outdir, h, net, nic)
path = f'{outdir}/netperf-{h}-{net}-{nic}-1.json'
data = parse_netperf_run(path)
if 'simtime' in data:
t = strftime('%H:%M:%S', gmtime(data['simtime']))
......@@ -67,7 +68,4 @@ for (h, h_l) in hosts:
tp = fmt_tp(data.get('throughput', ''))
latMean = fmt_lat(data.get('latenyMean', ''))
latTail = fmt_lat(data.get('latenyTail', ''))
print(
' %s & %s & %s & %s & %s & %s \\\\' %
(h_l, nic_l, net_l, tp, latMean, t)
)
print(f' {h_l} & {nic_l} & {net_l} & {tp} & {latMean} & t \\\\')
results/nopaxos.py
View file @ 05fe9ca5
......@@ -34,15 +34,16 @@ types_of_seq = ['ehseq', 'swseq']
num_clients = [1, 2, 3, 4, 5, 6, 7, 8, 10, 12]
print(
'num_client ehseq-tput(req/sec) ehseq-lat(us) swseq-tput(req/sec) swseq-lat(us)\n'
'num_client ehseq-tput(req/sec) ehseq-lat(us) swseq-tput(req/sec)'
' swseq-lat(us)\n'
)
for num_c in num_clients:
line = [str(num_c)]
for seq in types_of_seq:
path_pat = '%snopaxos-gt-ib-%s-%d-1.json' % (basedir, seq, num_c)
res = utils.parse_nopaxos.parse_nopaxos_run(num_c, seq, path_pat)
path_pat = f'{basedir}nopaxos-gt-ib-{seq}-{num_c}-1.json'
res = utils.parse_nopaxos.parse_nopaxos_run(num_c, path_pat)
#print(path_pat)
if ((res['throughput'] is None) or (res['latency'] is None)):
......@@ -54,7 +55,7 @@ for num_c in num_clients:
tput = res['throughput']
lat = res['latency']
line.append('%.2f' % (tput))
line.append(f'{tput:.2f}')
line.append(f'{lat}')
print(' '.join(line))
results/paper_data/utils/iperf.py
View file @ 05fe9ca5
......@@ -22,16 +22,14 @@
import fnmatch
import glob
import itertools
import json
import os
import re
import sys
def parse_iperf_run(data, skip=1, use=8):
tp_pat = re.compile(
r'\[ *\d*\] *([0-9\.]*)- *([0-9\.]*) sec.*Bytes *([0-9\.]*) ([GM])bits.*'
r'\[ *\d*\] *([0-9\.]*)- *([0-9\.]*) sec.*Bytes *([0-9\.]*)'
r' ([GM])bits.*'
)
tps_time = {}
for hn in fnmatch.filter(data['sims'].keys(), 'host.client.*'):
......@@ -47,7 +45,7 @@ def parse_iperf_run(data, skip=1, use=8):
if time >= skip + use:
continue
if not time in tps_time:
if time not in tps_time:
tps_time[time] = []
if m.group(4) == 'G':
......@@ -73,7 +71,7 @@ def parse_iperf(basename, skip=1, use=8):
# skip checkpoints
continue
with open(path, 'r') as f:
with open(path, 'r', encoding='utf-8') as f:
data = json.load(f)
result = parse_iperf_run(data, skip, use)
if result is not None:
......
results/pci_validation.py
View file @ 05fe9ca5
......@@ -32,17 +32,18 @@ def transform_internal(ts, component, msg):
return None
elif msg.startswith('read device register ') and 'res=' in msg:
return None
elif msg.startswith('our dma') or \
msg.startswith('issuing dma') or \
msg.startswith('processing ip packet') or \
msg.startswith('transmitting non-ip packet') or \
msg.startswith('transmitting ip packet'):
elif (
msg.startswith('our dma') or msg.startswith('issuing dma') or
msg.startswith('processing ip packet') or
msg.startswith('transmitting non-ip packet') or
msg.startswith('transmitting ip packet')
):
return None
return (ts + ' ' + msg)
return f'{ts} {msg}'
def transform_external(ts, component, msg):
def transform_external(ts, msg):
if msg.startswith('igbe: requesting restart clock:') or \
msg == 'igbe: scheduled' or \
msg == 'igbe: rescheduling next cycle' or \
......@@ -51,7 +52,7 @@ def transform_external(ts, component, msg):
elif msg.startswith('[rxdesc]') or msg.startswith('[txdesc]'):
msg = msg[9:]
return (ts + ' ' + msg)
return f'{ts} {msg}'
if len(sys.argv) != 3:
......@@ -62,7 +63,9 @@ if len(sys.argv) != 3:
outdir = sys.argv[1]
variant = sys.argv[2]
with open(outdir + '/' + 'pci_validation-' + variant + '-1.json', 'r') as f:
with open(
f'{outdir}/pci_validation-{variant}-1.json', 'r', encoding='utf-8'
) as f:
data = json.load(f)
if variant == 'internal':
......@@ -80,6 +83,6 @@ for l in it:
if not m:
continue
l = transform(m.group(1), m.group(2), m.group(3).lower())
l = transform(m.group(1), m.group(3).lower())
if l:
print(l)
results/pcilat.py
View file @ 05fe9ca5
......@@ -20,7 +20,6 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import itertools
import sys
import utils.iperf
......@@ -45,7 +44,7 @@ print('\t'.join(['config'] + list(map(str, lats))))
for (ht, nt, lab) in configs:
cols = [str(lab)]
for lat in lats:
path_pat = '%spcilat-%s-%s-switch-%d' % (basedir, ht, nt, lat)
path_pat = f'{basedir}pcilat-{ht}-{nt}-switch-{lat}'
res = utils.iperf.parse_iperf(path_pat)
if res['avg'] is None:
......
results/scale_host.py
View file @ 05fe9ca5
......@@ -21,10 +21,8 @@
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import json
import os
import pathlib
import shutil
import sys
from os.path import exists
if len(sys.argv) != 2:
print('Usage: udp_scale.py OUTDIR')
......@@ -35,18 +33,15 @@ types_of_client = [1, 4, 9, 14, 20]
bw = 1000
for cl in types_of_client:
log_path = '%sgt-ib-sw-Host-%dm-%d-1.json' % (basedir, bw, cl)
log_path = f'{basedir}gt-ib-sw-Host-{bw}m-{cl}-1.json'
try:
log = open(log_path, 'r')
except:
diff_time = ''
else:
exp_log = json.load(log)
start_time = exp_log['start_time']
end_time = exp_log['end_time']
diff_time = (end_time - start_time) / 60 #min
diff_time = str(diff_time)
log.close()
diff_time = ''
if exists(log_path):
with open(log_path, 'r', encoding='utf-8') as log:
exp_log = json.load(log)
start_time = exp_log['start_time']
end_time = exp_log['end_time']
diff_time = (end_time - start_time) / 60 #min
diff_time = str(diff_time)
print('%d\t%s' % (cl, diff_time))
print(f'{cl}\t{diff_time}')
results/scale_load.py
View file @ 05fe9ca5
......@@ -23,8 +23,6 @@
import json
import math
import os
import pathlib
import shutil
import sys
num_runs = 3
......@@ -43,20 +41,18 @@ for bw in types_of_bw:
std = 0
all_time = []
for i in range(1, num_runs + 1):
log_path = '%sgt-ib-sw-Load-%dm-%d.json' % (basedir, bw, i)
if os.path.exists(log_path):
log = open(log_path, 'r')
exp_log = json.load(log)
start_time = exp_log['start_time']
end_time = exp_log['end_time']
diff_time = (end_time - start_time) / 60 #min
total_time += diff_time
all_time.append(diff_time)
diff_time = str(diff_time)
log_path = f'{basedir}gt-ib-sw-Load-{bw}m-{i}.json'
log.close()
else:
diff_time = ''
diff_time = ''
if os.path.exists(log_path):
with open(log_path, 'r', encoding='utf-8') as log:
exp_log = json.load(log)
start_time = exp_log['start_time']
end_time = exp_log['end_time']
diff_time = (end_time - start_time) / 60 #min
total_time += diff_time
all_time.append(diff_time)
diff_time = str(diff_time)
#print('%d\t%s' % (bw, diff_time))
......@@ -69,4 +65,4 @@ for bw in types_of_bw:
std = std / num_runs
std = math.sqrt(std)
#print(str(std))
print('%d %s %f' % (bw, avg_time, std))
print(f'{bw} {avg_time} {std}')
results/sync-overhead.py → results/sync_overhead.py
View file @ 05fe9ca5
......@@ -22,8 +22,6 @@
import json
import os
import pathlib
import shutil
import sys
# How to use
......@@ -36,11 +34,12 @@ cmd = ['sleep', 'busy']
outdir = sys.argv[1]
# pylint: disable=redefined-outer-name
def parse_sim_time(path):
ret = {}
if not os.path.exists(path):
return ret
with open(path, 'r') as f:
with open(path, 'r', encoding='utf-8') as f:
data = json.load(f)
ret['simtime'] = (data['end_time'] - data['start_time']) / 60
......@@ -52,11 +51,8 @@ print('mode sleep busy')
for m in mode:
line = m
for c in cmd:
path = '%s/%s-%s-1.json' % (outdir, m, c)
path = f'{outdir}/{m}-{c}-1.json'
data = parse_sim_time(path)
if 'simtime' in data:
t = data['simtime']
else:
t = ''
t = data.get('simtime', '')
line = line + ' ' + f'{t}'
print(line)
results/utils/iperf.py
View file @ 05fe9ca5
......@@ -22,16 +22,14 @@
import fnmatch
import glob
import itertools
import json
import os
import re
import sys
def parse_iperf_run(data, skip=1, use=8):
tp_pat = re.compile(
r'\[ *\d*\] *([0-9\.]*)- *([0-9\.]*) sec.*Bytes *([0-9\.]*) ([GM])bits.*'
r'\[ *\d*\] *([0-9\.]*)- *([0-9\.]*) sec.*Bytes *([0-9\.]*)'
r' ([GM])bits.*'
)
tps_time = {}
for hn in fnmatch.filter(data['sims'].keys(), 'host.client.*'):
......@@ -47,7 +45,7 @@ def parse_iperf_run(data, skip=1, use=8):
if time >= skip + use:
continue
if not time in tps_time:
if time not in tps_time:
tps_time[time] = []
if m.group(4) == 'G':
......@@ -73,7 +71,7 @@ def parse_iperf(basename, skip=1, use=8):
# skip checkpoints
continue
with open(path, 'r') as f:
with open(path, 'r', encoding='utf-8') as f:
data = json.load(f)
result = parse_iperf_run(data, skip, use)
if result is not None:
......
results/utils/netperf.py
View file @ 05fe9ca5
......@@ -30,7 +30,7 @@ def parse_netperf_run(path):
if not os.path.exists(path):
return ret
with open(path, 'r') as f:
with open(path, 'r', encoding='utf-8') as f:
data = json.load(f)
ret['simtime'] = data['end_time'] - data['start_time']
......
results/utils/parse_nopaxos.py
View file @ 05fe9ca5
......@@ -25,7 +25,7 @@ import os
import re
def parse_nopaxos_run(num_c, seq, path):
def parse_nopaxos_run(num_c, path):
ret = {}
ret['throughput'] = None
......@@ -38,31 +38,31 @@ def parse_nopaxos_run(num_c, seq, path):
if not os.path.exists(path):
return ret
f_log = open(path, 'r')
log = json.load(f_log)
with open(path, 'r', encoding='utf-8') as f_log:
log = json.load(f_log)
total_tput = 0
total_avglat = 0
for i in range(num_c):
sim_name = f'host.client.{i}'
#print(sim_name)
total_tput = 0
total_avglat = 0
for i in range(num_c):
sim_name = f'host.client.{i}'
#print(sim_name)
# in this host log stdout
for j in log['sims'][sim_name]['stdout']:
#print(j)
m_t = tp_pat.match(j)
m_l = lat_pat.match(j)
if m_l:
# in this host log stdout
for j in log['sims'][sim_name]['stdout']:
#print(j)
lat = float(m_l.group(2)) / 1000 # us latency
#print(lat)
total_avglat += lat
m_t = tp_pat.match(j)
m_l = lat_pat.match(j)
if m_l:
#print(j)
lat = float(m_l.group(2)) / 1000 # us latency
#print(lat)
total_avglat += lat
if m_t:
if m_t:
n_req = float(m_t.group(2))
n_time = float(m_t.group(3))
total_tput += n_req / n_time
n_req = float(m_t.group(2))
n_time = float(m_t.group(3))
total_tput += n_req / n_time
avglat = total_avglat / num_c
#print(avglat)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment