simulators.py 15.9 KB
Newer Older
Antoine Kaufmann's avatar
Antoine Kaufmann committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
# Copyright 2021 Max Planck Institute for Software Systems, and
# National University of Singapore
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

23
import math
24
25
26
import typing as tp

from simbricks.nodeconfig import NodeConfig
27
from simbricks.experiment.experiment_environment import ExpEnv
28

29

30
class Simulator(object):
31
32
    """Base class for all simulators."""

33
34
35
    def __init__(self):
        self.extra_deps = []

36
    def resreq_cores(self):
37
        """Number of cores required for this simulator."""
38
39
40
        return 1

    def resreq_mem(self):
41
        """Memory required for this simulator (in MB)."""
42
43
        return 64

44
45
    def prep_cmds(self, env: ExpEnv) -> tp.List[str]:
        """Commands to run to prepare simulator."""
46
47
        return []

48
49
    def run_cmd(self, env: ExpEnv) -> tp.Optional[str]:
        """Command to run to execute simulator."""
50
        return None
51

52
    def dependencies(self):
53
        """Other simulators this one depends on."""
54
55
56
        return []

    # Sockets to be cleaned up
57
    def sockets_cleanup(self, env: ExpEnv):
58
59
60
        return []

    # sockets to wait for indicating the simulator is ready
61
    def sockets_wait(self, env: ExpEnv):
62
63
64
65
66
67
68
69
        return []

    def start_delay(self):
        return 5

    def wait_terminate(self):
        return False

70

71
72
class PCIDevSim(Simulator):
    name = ''
Jialin Li's avatar
Jialin Li committed
73
    sync_mode = 0
74
    start_tick = 0
75
76
    sync_period = 500
    pci_latency = 500
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92

    def __init__(self):
        super().__init__()

    def full_name(self):
        return 'dev.' + self.name

    def sockets_cleanup(self, env):
        return [env.dev_pci_path(self), env.dev_shm_path(self)]

    def sockets_wait(self, env):
        return [env.dev_pci_path(self)]


class NICSim(PCIDevSim):
    network = None
93
94
    eth_latency = 500

95
96
97
    def __init__(self):
        super().__init__()

98
99
100
101
    def set_network(self, net):
        self.network = net
        net.nics.append(self)

102
103
    def basic_args(self, env, extra=None):
        cmd = '%s %s %s %d %d %d %d %d' % \
104
105
            (env.dev_pci_path(self), env.nic_eth_path(self),
                    env.dev_shm_path(self), self.sync_mode, self.start_tick,
106
                    self.sync_period, self.pci_latency, self.eth_latency)
107
108
109
110

        if extra is not None:
            cmd += ' ' + extra
        return cmd
111

112
113
114
115
116
117
    def basic_run_cmd(self, env, name, extra=None):
        cmd = '%s/%s %s' % \
            (env.repodir + '/sims/nic', name,
             self.basic_args(env, extra))
        return cmd

118
    def full_name(self):
119
        return 'nic.' + self.name
120

121
    def sockets_cleanup(self, env):
122
        return super().sockets_cleanup(env) + [env.nic_eth_path(self)]
123
124

    def sockets_wait(self, env):
125
        return super().sockets_wait(env) + [env.nic_eth_path(self)]
126

127
class NetSim(Simulator):
128
    """Base class for network simulators."""
129
    name = ''
130
    opt = ''
Jialin Li's avatar
Jialin Li committed
131
    sync_mode = 0
132
133
    sync_period = 500
    eth_latency = 500
134
135
136

    def __init__(self):
        self.nics = []
137
138
139
        self.net_listen = []
        self.net_connect = []
        super().__init__()
140
141

    def full_name(self):
142
        return 'net.' + self.name
143

144
    def connect_network(self, net):
145
        """Connect this network to the listening peer `net`"""
146
147
148
        net.net_listen.append(self)
        self.net_connect.append(net)

149
150
151
152
    def connect_sockets(self, env):
        sockets = []
        for n in self.nics:
            sockets.append((n, env.nic_eth_path(n)))
153
154
        for n in self.net_connect:
            sockets.append((n, env.n2n_eth_path(n, self)))
155
156
        return sockets

157
158
159
160
161
162
    def listen_sockets(self, env):
        listens = []
        for net in self.net_listen:
            listens.append((net, env.n2n_eth_path(self, net)))
        return listens

163
    def dependencies(self):
164
165
166
167
168
169
170
        return self.nics + self.net_connect

    def sockets_cleanup(self, env):
        return [s for (_,s) in self.listen_sockets(env)]

    def sockets_wait(self, env):
        return [s for (_,s) in self.listen_sockets(env)]
171

172

173
174
class HostSim(Simulator):
    node_config: NodeConfig
175
    """Config for the simulated host. """
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
    name = ''
    wait = False
    """
    `True` - Wait for process of simulator to exit.
    `False` - Don't wait and instead stop the process.
    """
    sleep = 0
    cpu_freq = '8GHz'

    sync_mode = 0
    sync_period = 500
    pci_latency = 500

    def __init__(self):
        self.pcidevs: tp.List[PCIDevSim] = []
        super().__init__()

    def full_name(self):
        return 'host.' + self.name

    def add_nic(self, dev: NICSim):
        self.add_pcidev(dev)

199
    def add_pcidev(self, dev: PCIDevSim):
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
        dev.name = self.name + '.' + dev.name
        self.pcidevs.append(dev)

    def set_config(self, nc: NodeConfig):
        self.node_config = nc

    def dependencies(self):
        deps = []
        for dev in self.pcidevs:
            deps.append(dev)
            if isinstance(dev, NICSim):
                deps.append(dev.network)
        return deps

    def wait_terminate(self):
        return self.wait


218
class QemuHost(HostSim):
219
    sync = False
220
    cpu_freq = '4GHz'
221
222
223
224

    def __init__(self):
        super().__init__()

225
    def resreq_cores(self):
226
227
228
229
        if self.sync:
            return 1
        else:
            return self.node_config.cores + 1
230
231

    def resreq_mem(self):
Hejing Li's avatar
Hejing Li committed
232
        return 8192
233
234
235
236

    def prep_cmds(self, env):
        to_path = env.hdcopy_path(self)
        return [f'{env.qemu_img_path} create -f qcow2 -o '
237
            f'backing_file="{env.hd_path(self.node_config.disk_image)}" '
238
239
240
            f'{env.hdcopy_path(self)}']

    def run_cmd(self, env):
241
242
243
        accel = ',accel=kvm:tcg' if not self.sync else ''
        cmd = (f'{env.qemu_path} -machine q35{accel} -serial mon:stdio '
            '-cpu Skylake-Server -display none -nic none '
244
245
246
247
248
249
            f'-kernel {env.qemu_kernel_path} '
            f'-drive file={env.hdcopy_path(self)},if=ide,index=0,media=disk '
            f'-drive file={env.cfgtar_path(self)},if=ide,index=1,media=disk,'
                'driver=raw '
            '-append "earlyprintk=ttyS0 console=ttyS0 root=/dev/sda1 '
                'init=/home/ubuntu/guestinit.sh rw" '
250
            f'-m {self.node_config.memory} -smp {self.node_config.cores} ')
251
252

        if self.sync:
253
254
255
256
257
258
259
260
261
262
            unit = self.cpu_freq[-3:]
            if unit.lower() == 'ghz':
                base = 0
            elif unit.lower() == 'mhz':
                base = 3
            else:
                raise Exception('cpu frequency specified in unsupported unit')
            num = float(self.cpu_freq[:-3])
            shift = base - int(math.ceil(math.log(num, 2)))

263
            cmd += f' -icount shift={shift},sleep=off '
264

265
        for dev in self.pcidevs:
266
            cmd += f'-device simbricks-pci,socket={env.dev_pci_path(dev)}'
267
268
269
270
271
272
273
274
            if self.sync:
                cmd += ',sync=on'
                cmd += f',pci-latency={self.pci_latency}'
                cmd += f',sync-period={self.sync_period}'
            else:
                cmd += ',sync=off'
            cmd += ' '

275
276
277
278
279
        return cmd

class Gem5Host(HostSim):
    cpu_type_cp = 'X86KvmCPU'
    cpu_type = 'TimingSimpleCPU'
280
    sys_clock = '1GHz'
281

282
283
    def __init__(self):
        super().__init__()
284

285
286
287
288
    def set_config(self, nc):
        nc.sim = 'gem5'
        super().set_config(nc)

289
290
291
292
293
294
    def resreq_cores(self):
        return 1

    def resreq_mem(self):
        return 4096

295
296
    def prep_cmds(self, env):
        return [f'mkdir -p {env.gem5_cpdir(self)}']
297
298
299
300
301
302
303
304
305

    def run_cmd(self, env):
        cpu_type = self.cpu_type
        if env.create_cp:
            cpu_type = self.cpu_type_cp

        cmd = (f'{env.gem5_path} --outdir={env.gem5_outdir(self)} '
            f'{env.gem5_py_path} --caches --l2cache --l3cache '
            '--l1d_size=32kB --l1i_size=32kB --l2_size=2MB --l3_size=32MB '
306
            '--l1d_assoc=8 --l1i_assoc=8 --l2_assoc=4 --l3_assoc=16 '
Hejing Li's avatar
Hejing Li committed
307
            f'--cacheline_size=64 --cpu-clock={self.cpu_freq} --sys-clock={self.sys_clock} '
308
309
            f'--checkpoint-dir={env.gem5_cpdir(self)} '
            f'--kernel={env.gem5_kernel_path} '
310
            f'--disk-image={env.hd_raw_path(self.node_config.disk_image)} '
311
            f'--disk-image={env.cfgtar_path(self)} '
312
            f'--cpu-type={cpu_type} --mem-size={self.node_config.memory}MB '
313
            f'--num-cpus={self.node_config.cores} '
314
315
            '--ddio-enabled --ddio-way-part=8 --mem-type=DDR4_2400_16x4 ')

316
317
318
        if (env.no_simbricks):
            cmd += '--no-simbricks '

319
320
        if env.create_cp:
            cmd += '--max-checkpoints=1 '
321

322
        if env.restore_cp:
323
            cmd += '-r 1 '
324

325
326
327
328
329
        if len(self.pcidevs) > 0:
            assert len(self.pcidevs) == 1 # our gem5 python script supports only 1
            dev = self.pcidevs[0]
            cmd += f'--simbricks-pci={env.dev_pci_path(dev)} '
            cmd += f'--simbricks-shm={env.dev_shm_path(dev)} '
330
            if cpu_type == 'TimingSimpleCPU':
331
                cmd += '--simbricks-sync '
332
333
334
                cmd += f'--simbricks-sync_mode={self.sync_mode} '
                cmd += f'--simbricks-pci-lat={self.pci_latency} '
                cmd += f'--simbricks-sync-int={self.sync_period} '
335
336
337
            if isinstance(dev, I40eNIC) or \
                    (isinstance(dev, MultiSubNIC) and \
                     isinstance(dev.multinic, I40eMultiNIC)):
338
                cmd += '--simbricks-type=i40e '
339
340
            elif isinstance(dev, FEMUDev):
                cmd += '--simbricks-type=femu '
341
342
343
344
345
        return cmd



class CorundumVerilatorNIC(NICSim):
346
347
    clock_freq = 250 # MHz

348
349
350
    def __init__(self):
        super().__init__()

351
352
353
354
355
    def resreq_mem(self):
        # this is a guess
        return 512

    def run_cmd(self, env):
Hejing Li's avatar
Hejing Li committed
356
        return self.basic_run_cmd(env, '/corundum/corundum_verilator',
357
            str(self.clock_freq))
358
359

class CorundumBMNIC(NICSim):
360
361
362
    def __init__(self):
        super().__init__()

363
    def run_cmd(self, env):
Hejing Li's avatar
Hejing Li committed
364
        return self.basic_run_cmd(env, '/corundum_bm/corundum_bm')
365
366

class I40eNIC(NICSim):
367
368
369
    def __init__(self):
        super().__init__()

370
    def run_cmd(self, env):
Hejing Li's avatar
Hejing Li committed
371
        return self.basic_run_cmd(env, '/i40e_bm/i40e_bm')
372

373
374
375
376
377
378
379
380
class E1000NIC(NICSim):
    def __init__(self):
        super().__init__()

    def run_cmd(self, env):
        #return 'valgrind -v -v -v ' + self.basic_run_cmd(env, '/e1000_gem5/e1000_gem5')
        return self.basic_run_cmd(env, '/e1000_gem5/e1000_gem5')

381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
class MultiSubNIC(NICSim):
    name = ''
    multinic = None

    def __init__(self, mn):
        self.multinic = mn
        super().__init__()

    def full_name(self):
        return self.multinic.full_name() + '.' + self.name

    def dependencies(self):
        return super().dependencies() + [self.multinic]

    def start_delay(self):
        return 0

class I40eMultiNIC(Simulator):
    def __init__(self):
        self.subnics = []
        super().__init__()

    def create_subnic(self):
        sn = MultiSubNIC(self)
        self.subnics.append(sn)
        return sn

    def full_name(self):
        return 'multinic.' + self.name

    def run_cmd(self, env):
        args = ''
        first = True
        for sn in self.subnics:
            if not first:
                args += ' -- '
            first = False
            args += sn.basic_args(env)
        return '%s/sims/nic/i40e_bm/i40e_bm %s' % (env.repodir, args)

    def sockets_cleanup(self, env):
        ss = []
        for sn in self.subnics:
            ss += sn.sockets_cleanup(env)
        return ss

    def sockets_wait(self, env):
        ss = []
        for sn in self.subnics:
            ss += sn.sockets_wait(env)
        return ss
432
433
434


class WireNet(NetSim):
435
436
437
    def __init__(self):
        super().__init__()

438
    def run_cmd(self, env):
439
440
        connects = self.connect_sockets()
        assert len(connects) == 2
441
        cmd = '%s/sims/net/wire/net_wire %s %s %d %d %d' % \
442
443
                (env.repodir, connects[0][1],
                        connects[1][1],
Jialin Li's avatar
Jialin Li committed
444
                        self.sync_mode, self.sync_period, self.eth_latency)
445
446
447
        if len(env.pcap_file) > 0:
            cmd += ' ' + env.pcap_file
        return cmd
448
449

class SwitchNet(NetSim):
450
451
    sync = True

452
453
454
    def __init__(self):
        super().__init__()

455
    def run_cmd(self, env):
456
        cmd = env.repodir + '/sims/net/switch/net_switch'
457
        cmd += f' -S {self.sync_period} -E {self.eth_latency}'
458
459
460
461

        if not self.sync:
            cmd += ' -u'

462
463
        if len(env.pcap_file) > 0:
            cmd += ' -p ' + env.pcap_file
464
465
466
467
        for (_,n) in self.connect_sockets(env):
            cmd += ' -s ' + n
        for (_,n) in self.listen_sockets(env):
            cmd += ' -h ' + n
468
        return cmd
469

470
471
472
473
474
475
476
477
478
    def sockets_cleanup(self, env):
        # cleanup here will just have listening eth sockets, switch also creates
        # shm regions for each with a "-shm" suffix
        cleanup = []
        for s in super().sockets_cleanup(env):
            cleanup.append(s)
            cleanup.append(s + '-shm')
        return cleanup

479
class TofinoNet(NetSim):
480
481
    tofino_log_path = '/tmp/model.ldjson'
    sync = True
Jialin Li's avatar
Jialin Li committed
482

483
484
485
    def __init__(self):
        super().__init__()

486
487
    def run_cmd(self, env):
        cmd = env.repodir + '/sims/tofino/tofino'
488
        cmd += f' -S {self.sync_period} -E {self.eth_latency} -t {self.tofino_log_path}'
489
490
        if not self.sync:
            cmd += ' -u'
491
        for (_,n) in self.connect_sockets(env):
492
            cmd += ' -s ' + n
493
        return cmd
494

495
class NS3DumbbellNet(NetSim):
496
497
498
    def __init__(self):
        super().__init__()

499
500
    def run_cmd(self, env):
        ports = ''
501
        for (n,s) in self.connect_sockets(env):
502
            if 'server' in n.name:
503
                ports += '--CosimPortLeft=' + s + ' '
504
            else:
505
                ports += '--CosimPortRight=' + s + ' '
506

507
        cmd = env.repodir + '/sims/external/ns-3' + '/cosim-run.sh cosim cosim-dumbbell-example ' + ports + ' ' + self.opt
508
509
510
511
        print(cmd)

        return cmd

512
class NS3BridgeNet(NetSim):
513
514
515
    def __init__(self):
        super().__init__()

516
517
    def run_cmd(self, env):
        ports = ''
518
        for (_,n) in self.connect_sockets(env):
519
            ports += '--CosimPort=' + n + ' '
520

521
        cmd = env.repodir + '/sims/external/ns-3' + '/cosim-run.sh cosim cosim-bridge-example ' + ports + ' ' + self.opt
522
523
524
        print(cmd)

        return cmd
525

526
class NS3SequencerNet(NetSim):
527
528
529
    def __init__(self):
        super().__init__()

530
531
    def run_cmd(self, env):
        ports = ''
532
        for (n,s) in self.connect_sockets(env):
533
            if 'client' in n.name:
534
                ports += '--ClientPort=' + s + ' '
535
            elif 'replica' in n.name:
536
                ports += '--ServerPort=' + s + ' '
537
            elif 'sequencer' in n.name:
538
                ports += '--EndhostSequencerPort=' + s + ' '
539
540
            else:
                raise Exception('Wrong NIC type')
541
        cmd = env.repodir + '/sims/external/ns-3' + '/cosim-run.sh sequencer sequencer-single-switch-example ' + ports + ' ' + self.opt
542
543
544
        return cmd


545
546
547
548
549
550
551
552
553
class FEMUDev(PCIDevSim):
    def __init__(self):
        super().__init__()

    def run_cmd(self, env):
        cmd = '%s%s %s %s' % \
            (env.repodir, '/sims/external/femu/femu-simbricks',
             env.dev_pci_path(self), env.dev_shm_path(self))
        return cmd