simulators.py 36.9 KB
Newer Older
Antoine Kaufmann's avatar
Antoine Kaufmann committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
# Copyright 2021 Max Planck Institute for Software Systems, and
# National University of Singapore
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

23
24
25
# Allow own class to be used as type for a method's argument
from __future__ import annotations

26
import math
27
import sys
28
29
import typing as tp

30
31
from simbricks.orchestration.experiment.experiment_environment import ExpEnv
from simbricks.orchestration.nodeconfig import NodeConfig
32
33
from simbricks.orchestration.e2e_topologies import E2ETopology
from simbricks.orchestration import e2e_components as e2e
34

35

36
class Simulator(object):
37
38
    """Base class for all simulators."""

39
40
    def __init__(self) -> None:
        self.extra_deps: tp.List[Simulator] = []
Jonas Kaufmann's avatar
Jonas Kaufmann committed
41
        self.name = ''
42

43
    def resreq_cores(self) -> int:
44
45
46
47
48
        """
        Number of cores this simulator requires during execution.

        This is used for scheduling multiple runs and experiments.
        """
49
50
        return 1

51
    def resreq_mem(self) -> int:
52
53
54
55
56
        """
        Number of memory in MB this simulator requires during execution.

        This is used for scheduling multiple runs and experiments.
        """
57
58
        return 64

59
    def full_name(self) -> str:
Jonas Kaufmann's avatar
Jonas Kaufmann committed
60
61
62
        """Full name of the simulator."""
        return ''

63
    # pylint: disable=unused-argument
64
    def prep_cmds(self, env: ExpEnv) -> tp.List[str]:
65
        """Commands to prepare execution of this simulator."""
66
67
        return []

68
    # pylint: disable=unused-argument
69
    def run_cmd(self, env: ExpEnv) -> tp.Optional[str]:
70
        """Command to execute this simulator."""
71
        return None
72

Jonas Kaufmann's avatar
Jonas Kaufmann committed
73
    def dependencies(self) -> tp.List[Simulator]:
74
        """Other simulators to execute before this one."""
75
76
77
        return []

    # Sockets to be cleaned up
78
    # pylint: disable=unused-argument
79
    def sockets_cleanup(self, env: ExpEnv) -> tp.List[str]:
80
81
82
        return []

    # sockets to wait for indicating the simulator is ready
83
    # pylint: disable=unused-argument
84
    def sockets_wait(self, env: ExpEnv) -> tp.List[str]:
85
86
        return []

87
    def start_delay(self) -> int:
88
89
        return 5

90
    def wait_terminate(self) -> bool:
91
92
        return False

93

94
class PCIDevSim(Simulator):
95
96
    """Base class for PCIe device simulators."""

97
    def __init__(self) -> None:
98
99
100
        super().__init__()

        self.sync_mode = 0
101
102
        """Synchronization mode. 0 is running unsynchronized, 1 synchronized.
        Depending on the concrete simulator, there may be additional modes."""
103
        self.start_tick = 0
104
105
106
107
108
        """The timestamp at which to start the simulation. This is useful when
        the simulator is only attached at a later point in time and needs to
        synchronize with connected simulators. For example, this could be used
        when taking checkpoints to only attach certain simulators after the
        checkpoint has been taken."""
109
        self.sync_period = 500
110
111
        """Period in nanoseconds of sending synchronization messages from this
        device to connected components."""
112
        self.pci_latency = 500
113
114
        """Latency in nanoseconds for sending messages to components connected
        via PCI."""
115

116
    def full_name(self) -> str:
117
118
        return 'dev.' + self.name

119
    def is_nic(self) -> bool:
120
121
        return False

122
    def sockets_cleanup(self, env: ExpEnv) -> tp.List[str]:
123
124
        return [env.dev_pci_path(self), env.dev_shm_path(self)]

125
    def sockets_wait(self, env: ExpEnv) -> tp.List[str]:
126
127
128
129
        return [env.dev_pci_path(self)]


class NICSim(PCIDevSim):
130
131
    """Base class for NIC simulators."""

132
    def __init__(self) -> None:
133
134
135
136
137
        super().__init__()

        self.network: tp.Optional[NetSim] = None
        self.mac: tp.Optional[str] = None
        self.eth_latency = 500
138
139
        """Ethernet latency in nanoseconds from this NIC to the network
        component."""
140

141
    def set_network(self, net: NetSim) -> None:
142
        """Connect this NIC to a network simulator."""
143
        self.network = net
144
        net.connect_nic(self)
145

146
    def basic_args(self, env: ExpEnv, extra: tp.Optional[str] = None) -> str:
147
148
        cmd = (
            f'{env.dev_pci_path(self)} {env.nic_eth_path(self)}'
149
150
            f' {env.dev_shm_path(self)} {self.sync_mode} {self.start_tick}'
            f' {self.sync_period} {self.pci_latency} {self.eth_latency}'
151
        )
152
153
        if self.mac is not None:
            cmd += ' ' + (''.join(reversed(self.mac.split(':'))))
154
155
156
157

        if extra is not None:
            cmd += ' ' + extra
        return cmd
158

159
160
161
    def basic_run_cmd(
        self, env: ExpEnv, name: str, extra: tp.Optional[str] = None
    ) -> str:
162
        cmd = f'{env.repodir}/sims/nic/{name} {self.basic_args(env, extra)}'
163
164
        return cmd

165
    def full_name(self) -> str:
166
        return 'nic.' + self.name
167

168
    def is_nic(self) -> bool:
169
170
        return True

171
    def sockets_cleanup(self, env: ExpEnv) -> tp.List[str]:
172
        return super().sockets_cleanup(env) + [env.nic_eth_path(self)]
173

174
    def sockets_wait(self, env: ExpEnv) -> tp.List[str]:
175
        return super().sockets_wait(env) + [env.nic_eth_path(self)]
176

Jonas Kaufmann's avatar
Jonas Kaufmann committed
177

178
class NetSim(Simulator):
179
    """Base class for network simulators."""
180

181
    def __init__(self) -> None:
182
183
184
185
        super().__init__()

        self.opt = ''
        self.sync_mode = 0
186
187
        """Synchronization mode. 0 is running unsynchronized, 1 synchronized.
        Depending on the concrete simulator, there may be additional modes."""
188
        self.sync_period = 500
189
190
        """Synchronization period in nanoseconds from this network to connected
        components."""
191
        self.eth_latency = 500
192
193
        """Ethernet latency in nanoseconds from this network to connected
        components."""
194
195
196
197
        self.nics: list[NICSim] = []
        self.hosts_direct: list[HostSim] = []
        self.net_listen: list[NetSim] = []
        self.net_connect: list[NetSim] = []
198
        self.wait = False
199

200
    def full_name(self) -> str:
201
        return 'net.' + self.name
202

203
204
205
    def connect_nic(self, nic: NICSim) -> None:
        self.nics.append(nic)

206
    def connect_network(self, net: NetSim) -> None:
207
        """Connect this network to the listening peer `net`"""
208
209
210
        net.net_listen.append(self)
        self.net_connect.append(net)

Jonas Kaufmann's avatar
Jonas Kaufmann committed
211
    def connect_sockets(self, env: ExpEnv) -> tp.List[tp.Tuple[Simulator, str]]:
212
213
214
        sockets = []
        for n in self.nics:
            sockets.append((n, env.nic_eth_path(n)))
215
216
        for n in self.net_connect:
            sockets.append((n, env.n2n_eth_path(n, self)))
217
218
        for h in self.hosts_direct:
            sockets.append((h, env.net2host_eth_path(self, h)))
219
220
        return sockets

221
    def listen_sockets(self, env: ExpEnv) -> tp.List[tp.Tuple[NetSim, str]]:
222
223
224
225
226
        listens = []
        for net in self.net_listen:
            listens.append((net, env.n2n_eth_path(self, net)))
        return listens

227
    def dependencies(self) -> tp.List[Simulator]:
228
        return self.nics + self.net_connect + self.hosts_direct
229

230
    def sockets_cleanup(self, env: ExpEnv) -> tp.List[str]:
Jonas Kaufmann's avatar
Jonas Kaufmann committed
231
        return [s for (_, s) in self.listen_sockets(env)]
232

233
    def sockets_wait(self, env: ExpEnv) -> tp.List[str]:
Jonas Kaufmann's avatar
Jonas Kaufmann committed
234
        return [s for (_, s) in self.listen_sockets(env)]
235

236
237
238
    def wait_terminate(self) -> Bool:
        return self.wait

239

240
241
242
243
# FIXME: Class hierarchy is broken here as an ugly hack
class MemDevSim(NICSim):
    """Base class for memory device simulators."""

244
    def __init__(self) -> None:
245
246
247
248
249
250
251
        super().__init__()

        self.mem_latency = 500
        self.addr = 0xe000000000000000
        self.size = 1024 * 1024 * 1024  # 1GB
        self.as_id = 0

252
    def full_name(self) -> str:
253
254
        return 'mem.' + self.name

255
    def sockets_cleanup(self, env: ExpEnv) -> tp.List[str]:
256
257
        return [env.dev_mem_path(self), env.dev_shm_path(self)]

258
    def sockets_wait(self, env: ExpEnv) -> tp.List[str]:
259
260
261
262
263
264
        return [env.dev_mem_path(self)]


class NetMemSim(NICSim):
    """Base class for netork memory simulators."""

265
    def __init__(self) -> None:
266
267
268
269
270
271
        super().__init__()

        self.addr = 0xe000000000000000
        self.size = 1024 * 1024 * 1024  # 1GB
        self.as_id = 0

272
    def full_name(self) -> str:
273
274
        return 'netmem.' + self.name

275
    def sockets_cleanup(self, env: ExpEnv) -> tp.List[str]:
276
277
        return [env.nic_eth_path(self), env.dev_shm_path(self)]

278
    def sockets_wait(self, env: ExpEnv) -> tp.List[str]:
279
280
281
        return [env.nic_eth_path(self)]


282
class HostSim(Simulator):
283
    """Base class for host simulators."""
284

285
    def __init__(self, node_config: NodeConfig) -> None:
286
        super().__init__()
287
        self.node_config = node_config
288
        """System configuration for this simulated host. """
289
290
        self.wait = False
        """
291
292
293
        `True` - Wait for this simulator to finish execution. `False` - Don't
        wait and instead shutdown the simulator as soon as all other awaited
        simulators have completed execution.
294
295
        """
        self.sleep = 0
296
        self.cpu_freq = '4GHz'
297
298

        self.sync_mode = 0
299
300
        """Synchronization mode. 0 is running unsynchronized, 1 synchronized.
        Depending on the concrete simulator, there may be additional modes."""
301
        self.sync_period = 500
302
303
        """Period in nanoseconds of sending synchronization messages from this
        device to connected components."""
304
        self.pci_latency = 500
305
306
        """Latency in nanoseconds for sending messages to components connected
        via PCIe."""
307
        self.mem_latency = 500
308
309
        """Latency in nanoseconds for sending messages to components connected
        via Ethernet."""
310

311
312
313
314
315
316
317
318
        self.sync_drift: tp.Optional[int] = None
        """Conversion factor from SimBricks' picosecond timestamps to the host's
        nanosecond clock. None or 1000 is 1:1, while other values result in
        simulated clock drift for a host."""
        self.sync_offset: tp.Optional[int] = None
        """Clock offset for this host in picoseconds. None or 0 does not add an
        offset."""

319
        self.pcidevs: tp.List[PCIDevSim] = []
320
        self.net_directs: tp.List[NetSim] = []
321
        self.memdevs: tp.List[MemDevSim] = []
322

323
    @property
324
325
326
327
328
329
    def nics(self) -> tp.List[NICSim]:
        return [
            tp.cast(NICSim, pcidev)
            for pcidev in self.pcidevs
            if pcidev.is_nic()
        ]
330

331
    def full_name(self) -> str:
332
333
        return 'host.' + self.name

334
    def add_nic(self, dev: NICSim) -> None:
335
        """Add a NIC to this host."""
336
337
        self.add_pcidev(dev)

338
    def add_pcidev(self, dev: PCIDevSim) -> None:
339
        """Add a PCIe device to this host."""
340
341
342
        dev.name = self.name + '.' + dev.name
        self.pcidevs.append(dev)

343
    def add_memdev(self, dev: MemDevSim) -> None:
344
345
346
        dev.name = self.name + '.' + dev.name
        self.memdevs.append(dev)

347
    def add_netdirect(self, net: NetSim) -> None:
348
        """Add a direct connection to a network to this host."""
349
350
351
        net.hosts_direct.append(self)
        self.net_directs.append(net)

352
    def dependencies(self) -> tp.List[PCIDevSim]:
353
354
355
356
357
        deps = []
        for dev in self.pcidevs:
            deps.append(dev)
            if isinstance(dev, NICSim):
                deps.append(dev.network)
358
359
        for dev in self.memdevs:
            deps.append(dev)
360
361
        return deps

362
    def wait_terminate(self) -> bool:
363
364
365
        return self.wait


366
class QemuHost(HostSim):
367
368
    """Qemu host simulator."""

369
    def __init__(self, node_config: NodeConfig) -> None:
370
        super().__init__(node_config)
371
372

        self.sync = False
373
        """"Whether to synchronize with attached simulators."""
374

375
    def resreq_cores(self) -> int:
376
377
378
379
        if self.sync:
            return 1
        else:
            return self.node_config.cores + 1
380

381
    def resreq_mem(self) -> int:
Hejing Li's avatar
Hejing Li committed
382
        return 8192
383

384
    def prep_cmds(self, env: ExpEnv) -> tp.List[str]:
Jonas Kaufmann's avatar
Jonas Kaufmann committed
385
386
        return [
            f'{env.qemu_img_path} create -f qcow2 -o '
387
            f'backing_file="{env.hd_path(self.node_config.disk_image)}" '
Jonas Kaufmann's avatar
Jonas Kaufmann committed
388
389
            f'{env.hdcopy_path(self)}'
        ]
390

391
    def run_cmd(self, env: ExpEnv) -> str:
392
        accel = ',accel=kvm:tcg' if not self.sync else ''
393
394
395
396
397
        if self.node_config.kcmd_append:
            kcmd_append = ' ' + self.node_config.kcmd_append
        else:
            kcmd_append = ''

Jonas Kaufmann's avatar
Jonas Kaufmann committed
398
399
        cmd = (
            f'{env.qemu_path} -machine q35{accel} -serial mon:stdio '
400
            '-cpu Skylake-Server -display none -nic none '
401
402
403
            f'-kernel {env.qemu_kernel_path} '
            f'-drive file={env.hdcopy_path(self)},if=ide,index=0,media=disk '
            f'-drive file={env.cfgtar_path(self)},if=ide,index=1,media=disk,'
Jonas Kaufmann's avatar
Jonas Kaufmann committed
404
            'driver=raw '
405
            '-append "earlyprintk=ttyS0 console=ttyS0 root=/dev/sda1 '
406
            f'init=/home/ubuntu/guestinit.sh rw{kcmd_append}" '
Jonas Kaufmann's avatar
Jonas Kaufmann committed
407
408
            f'-m {self.node_config.memory} -smp {self.node_config.cores} '
        )
409
410

        if self.sync:
411
412
413
414
415
416
            unit = self.cpu_freq[-3:]
            if unit.lower() == 'ghz':
                base = 0
            elif unit.lower() == 'mhz':
                base = 3
            else:
417
                raise ValueError('cpu frequency specified in unsupported unit')
418
419
420
            num = float(self.cpu_freq[:-3])
            shift = base - int(math.ceil(math.log(num, 2)))

421
            cmd += f' -icount shift={shift},sleep=off '
422

423
        for dev in self.pcidevs:
424
            cmd += f'-device simbricks-pci,socket={env.dev_pci_path(dev)}'
425
426
427
428
            if self.sync:
                cmd += ',sync=on'
                cmd += f',pci-latency={self.pci_latency}'
                cmd += f',sync-period={self.sync_period}'
429
430
431
432
                if self.sync_drift is not None:
                    cmd += f',sync-drift={self.sync_drift}'
                if self.sync_offset is not None:
                    cmd += f',sync-offset={self.sync_offset}'
433
434
435
436
            else:
                cmd += ',sync=off'
            cmd += ' '

437
        # qemu does not currently support net direct ports
438
        assert len(self.net_directs) == 0
439
440
        # qemu does not currently support mem device ports
        assert len(self.memdevs) == 0
441
442
        return cmd

Jonas Kaufmann's avatar
Jonas Kaufmann committed
443

444
class Gem5Host(HostSim):
445
    """Gem5 host simulator."""
446

447
    def __init__(self, node_config: NodeConfig) -> None:
448
449
        node_config.sim = 'gem5'
        super().__init__(node_config)
450
451
452
        self.cpu_type_cp = 'X86KvmCPU'
        self.cpu_type = 'TimingSimpleCPU'
        self.sys_clock = '1GHz'
453
454
        self.extra_main_args = []
        self.extra_config_args = []
455
        self.variant = 'fast'
456
457
458
459
460
461
462
463
        self.modify_checkpoint_tick = True
        """Whether to modify the event queue tick before restoring a checkpoint.
        When this is enabled, the restored checkpoint will start at event queue
        tick 0. This is a performance optimization since now, connected
        simulators don't have to simulate and synchronize until the restored
        tick before the actual workload can be executed. Disable this if you
        need to retain the differences in virtual time between multiple gem5
        instances."""
464

465
    def resreq_cores(self) -> int:
466
467
        return 1

468
    def resreq_mem(self) -> int:
469
470
        return 4096

471
    def prep_cmds(self, env: ExpEnv) -> tp.List[str]:
472
473
474
475
476
477
478
        cmds = [f'mkdir -p {env.gem5_cpdir(self)}']
        if env.restore_cp and self.modify_checkpoint_tick:
            cmds.append(
                f'python3 {env.utilsdir}/modify_gem5_cp_tick.py --tick 0 '
                f'--cpdir {env.gem5_cpdir(self)}'
            )
        return cmds
479

480
    def run_cmd(self, env: ExpEnv) -> str:
481
482
483
484
        cpu_type = self.cpu_type
        if env.create_cp:
            cpu_type = self.cpu_type_cp

485
        cmd = f'{env.gem5_path(self.variant)} --outdir={env.gem5_outdir(self)} '
486
        cmd += ' '.join(self.extra_main_args)
Jonas Kaufmann's avatar
Jonas Kaufmann committed
487
        cmd += (
488
489
490
            f' {env.gem5_py_path} --caches --l2cache '
            '--l1d_size=32kB --l1i_size=32kB --l2_size=32MB '
            '--l1d_assoc=8 --l1i_assoc=8 --l2_assoc=16 '
491
492
            f'--cacheline_size=64 --cpu-clock={self.cpu_freq}'
            f' --sys-clock={self.sys_clock} '
493
494
            f'--checkpoint-dir={env.gem5_cpdir(self)} '
            f'--kernel={env.gem5_kernel_path} '
495
            f'--disk-image={env.hd_raw_path(self.node_config.disk_image)} '
496
            f'--disk-image={env.cfgtar_path(self)} '
497
            f'--cpu-type={cpu_type} --mem-size={self.node_config.memory}MB '
498
            f'--num-cpus={self.node_config.cores} '
499
            '--mem-type=DDR4_2400_16x4 '
Jonas Kaufmann's avatar
Jonas Kaufmann committed
500
        )
501

502
503
504
        if self.node_config.kcmd_append:
            cmd += f'--command-line-append="{self.node_config.kcmd_append}" '

505
506
        if env.create_cp:
            cmd += '--max-checkpoints=1 '
507

508
        if env.restore_cp:
509
            cmd += '-r 1 '
510

511
        for dev in self.pcidevs:
Jonas Kaufmann's avatar
Jonas Kaufmann committed
512
513
514
515
516
            cmd += (
                f'--simbricks-pci=connect:{env.dev_pci_path(dev)}'
                f':latency={self.pci_latency}ns'
                f':sync_interval={self.sync_period}ns'
            )
517
            if cpu_type == 'TimingSimpleCPU':
518
                cmd += ':sync'
Jonas Kaufmann's avatar
Jonas Kaufmann committed
519
            cmd += ' '
520

521
522
523
524
525
526
527
528
529
530
531
        for dev in self.memdevs:
            cmd += (
                f'--simbricks-mem={dev.size}@{dev.addr}@{dev.as_id}@'
                f'connect:{env.dev_mem_path(dev)}'
                f':latency={self.mem_latency}ns'
                f':sync_interval={self.sync_period}ns'
            )
            if cpu_type == 'TimingSimpleCPU':
                cmd += ':sync'
            cmd += ' '

532
        for net in self.net_directs:
Jonas Kaufmann's avatar
Jonas Kaufmann committed
533
534
535
536
537
538
539
            cmd += (
                '--simbricks-eth-e1000=listen'
                f':{env.net2host_eth_path(net, self)}'
                f':{env.net2host_shm_path(net, self)}'
                f':latency={net.eth_latency}ns'
                f':sync_interval={net.sync_period}ns'
            )
540
541
            if cpu_type == 'TimingSimpleCPU':
                cmd += ':sync'
Jonas Kaufmann's avatar
Jonas Kaufmann committed
542
            cmd += ' '
543
544

        cmd += ' '.join(self.extra_config_args)
545
546
547
        return cmd


548
549
550
class SimicsHost(HostSim):
    """Simics host simulator."""

551
    def __init__(self, node_config: NodeConfig) -> None:
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
        super().__init__(node_config)
        node_config.sim = 'simics'

        self.cpu_class = 'x86-cooper-lake'
        """Simics CPU class. Can be obtained by running `list-classes substr =
        processor_` inside Simics."""
        self.cpu_freq = 4000  # TODO Don't hide attribute in super class
        """CPU frequency in MHz"""
        self.timing = False
        """Whether to run Simics in a more precise timing mode. This adds a
        cache model."""
        self.append_cmdline: tp.List[str] = []
        """Additional parameters to append on the command-line when invoking
        Simics."""
        self.interactive = False
        """Whether to launch Simics in interactive GUI mode. This is helpful for
        debugging, e.g. enabling log messages in the mid of the simulation."""
        self.debug_messages = False
        """Whether to enable debug messages of SimBricks adapter devices."""

572
    def resreq_cores(self) -> int:
573
574
        return 2

575
    def resreq_mem(self) -> int:
576
577
        return self.node_config.memory

578
    def run_cmd(self, env: ExpEnv) -> str:
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
        if self.node_config.kcmd_append:
            raise RuntimeError(
                'Appending kernel command-line not yet implemented.'
            )

        if self.interactive and not env.create_cp:
            cmd = f'{env.simics_gui_path} -q '
        else:
            cmd = f'{env.simics_path} -q -batch-mode -werror '

        if env.restore_cp:
            # restore checkpoint
            cmd += f'-e \'read-configuration {env.simics_cpfile(self)}\' '
        else:
            # initialize simulated machine
            cmd += (
                '-e \'run-command-file '
                f'{env.simics_qsp_modern_core_path} '
                f'disk0_image = {env.hd_raw_path(self.node_config.disk_image)} '
                f'disk1_image = {env.cfgtar_path(self)} '
                f'cpu_comp_class = {self.cpu_class} '
                f'freq_mhz = {self.cpu_freq} '
                f'num_cores = {self.node_config.cores} '
                f'num_threads = {self.node_config.threads} '
603
604
                f'memory_megs = {self.node_config.memory} '
                'create_network = FALSE\' '
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
            )

        if env.create_cp:
            # stop simulation when encountering special checkpoint string on
            # serial console
            cmd += (
                '-e \'bp.console_string.break board.serconsole.con '
                '"ready to checkpoint"\' '
            )
            # run simulation
            cmd += '-e run '
            # create checkpoint
            cmd += f'-e \'write-configuration {env.simics_cpfile(self)}\' '
            return cmd

        if self.timing:
            # Add the cache model. Note that the caches aren't warmed up during
            # the boot process. The reason is that when later adding the memory
            # devices, we change the mapped memory. The cycle staller doesn't
            # like this and will SEGFAULT.
            #
            # The cache model doesn't store any memory contents and therefore
            # doesn't answer any memory transactions. It only inserts CPU stall
            # cycles on each cache level and can be queried for statistics as
            # well as which addresses are cached.
            #
            # Read penalties are based on https://www.7-cpu.com/cpu/Skylake.html
            cmd += (
                '-e \'new-cycle-staller name = cs0 '
                'stall-interval = 10000\' '
            )
            cmd += (
                '-e \'new-simple-cache-tool name = cachetool '
                'cycle-staller = cs0 -connect-all\' '
            )
            cmd += (
                '-e \'cachetool.add-l1i-cache name = l1i line-size = 64 '
                'sets = 64 ways = 8\' '
            )
            cmd += (
                '-e \'cachetool.add-l1d-cache name = l1d line-size = 64 '
                'sets = 64 ways = 8 -ip-read-prefetcher '
                'prefetch-additional = 1 read-penalty = 4\' '
            )
            cmd += (
                '-e \'cachetool.add-l2-cache name = l2 line-size = 64 '
                'sets = 8192 ways = 4 -prefetch-adjacent '
                'prefetch-additional = 4 read-penalty = 12\' '
            )
            cmd += (
                '-e \'cachetool.add-l3-cache name = l3 line-size = 64 '
                'sets = 32768 ways = 16 read-penalty = 42\' '
            )

        # Only simulate one cycle per CPU and then switch to the next. This is
        # necessary for the synchronization of the SimBricks adapter with all
        # the CPUs to work properly.
        cmd += '-e \'set-time-quantum 1\' '

        if self.memdevs:
            cmd += '-e \'load-module simbricks_mem_comp\' '

        for memdev in self.memdevs:
            cmd += (
                f'-e \'$mem = (new-simbricks-mem-comp '
                f'socket = "{env.dev_mem_path(memdev)}" '
                f'mem_latency = {self.mem_latency} '
                f'sync_period = {self.sync_period})\' '
            )
            cmd += (
                f'-e \'board.mb.dram_space.add-map $mem.simbricks_mem_dev '
                f'{memdev.addr:#x} {memdev.size:#x}\' '
            )
            if self.debug_messages:
                cmd += '-e \'$mem.log-level 3\' '

681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
        if self.pcidevs:
            cmd += '-e \'load-module simbricks_pcie_comp\' '

        i = 0
        for pcidev in self.pcidevs:
            cmd += (
                f'-e \'$pci = (create-simbricks-pcie-comp '
                f'socket = "{env.dev_pci_path(pcidev)}" '
                f'pci_latency = {self.pci_latency} '
                f'sync_period = {self.sync_period})\' '
            )
            cmd += f'-e \'connect board.mb.nb.pci_slot[{i}] $pci.pci_bus\' '
            cmd += '-e instantiate-components '
            if self.debug_messages:
                cmd += '-e \'$pci.log-level 3\' '
            i += 1

698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
        for param in self.append_cmdline:
            cmd += f'{param} '

        # The simulation keeps running when the host powers off. A log message
        # indicates the event when the machine is powering off. We place a
        # breakpoint on that log message, which will terminate Simics due to the
        # use of `-batch-mode`.
        cmd += (
            '-e \'bp.log.break object=board.mb.sb.lpc.bank.acpi_io_regs '
            'substr="Sleep state is unimplemented" type=unimpl\' '
        )

        return cmd + '-e run'


713
class CorundumVerilatorNIC(NICSim):
714

715
    def __init__(self) -> None:
716
717
        super().__init__()
        self.clock_freq = 250  # MHz
718

719
    def resreq_mem(self) -> int:
720
721
722
        # this is a guess
        return 512

723
    def run_cmd(self, env: ExpEnv) -> str:
Jonas Kaufmann's avatar
Jonas Kaufmann committed
724
725
726
727
        return self.basic_run_cmd(
            env, '/corundum/corundum_verilator', str(self.clock_freq)
        )

728
729

class CorundumBMNIC(NICSim):
Jonas Kaufmann's avatar
Jonas Kaufmann committed
730

731
    def run_cmd(self, env: ExpEnv) -> str:
Hejing Li's avatar
Hejing Li committed
732
        return self.basic_run_cmd(env, '/corundum_bm/corundum_bm')
733

Jonas Kaufmann's avatar
Jonas Kaufmann committed
734

735
class I40eNIC(NICSim):
Jonas Kaufmann's avatar
Jonas Kaufmann committed
736

737
    def run_cmd(self, env: ExpEnv) -> str:
Hejing Li's avatar
Hejing Li committed
738
        return self.basic_run_cmd(env, '/i40e_bm/i40e_bm')
739

Jonas Kaufmann's avatar
Jonas Kaufmann committed
740

741
class E1000NIC(NICSim):
742

743
    def __init__(self) -> None:
744
745
        super().__init__()
        self.debug = False
Jonas Kaufmann's avatar
Jonas Kaufmann committed
746

747
    def run_cmd(self, env: ExpEnv) -> str:
748
749
750
751
        cmd = self.basic_run_cmd(env, '/e1000_gem5/e1000_gem5')
        if self.debug:
            cmd = 'env E1000_DEBUG=1 ' + cmd
        return cmd
752

Jonas Kaufmann's avatar
Jonas Kaufmann committed
753

754
755
class MultiSubNIC(NICSim):

756
    def __init__(self, mn: Simulator) -> None:
757
        super().__init__()
758
        self.multinic = mn
759

760
    def full_name(self) -> str:
761
762
        return self.multinic.full_name() + '.' + self.name

763
    def dependencies(self) -> tp.List[Simulator]:
764
765
        return super().dependencies() + [self.multinic]

766
    def start_delay(self) -> int:
767
768
        return 0

Jonas Kaufmann's avatar
Jonas Kaufmann committed
769

Jonas Kaufmann's avatar
Jonas Kaufmann committed
770
class I40eMultiNIC(Simulator):
Jonas Kaufmann's avatar
Jonas Kaufmann committed
771

772
    def __init__(self) -> None:
773
        super().__init__()
774
        self.subnics: tp.List[NICSim] = []
775

776
    def create_subnic(self) -> MultiSubNIC:
777
778
779
780
        sn = MultiSubNIC(self)
        self.subnics.append(sn)
        return sn

781
    def full_name(self) -> str:
782
783
        return 'multinic.' + self.name

784
    def run_cmd(self, env: ExpEnv) -> str:
785
786
787
788
789
790
791
        args = ''
        first = True
        for sn in self.subnics:
            if not first:
                args += ' -- '
            first = False
            args += sn.basic_args(env)
792
        return f'{env.repodir}/sims/nic/i40e_bm/i40e_bm {args}'
793

794
    def sockets_cleanup(self, env: ExpEnv) -> tp.List[str]:
795
796
797
798
799
        ss = []
        for sn in self.subnics:
            ss += sn.sockets_cleanup(env)
        return ss

800
    def sockets_wait(self, env: ExpEnv) -> tp.List[str]:
801
802
803
804
        ss = []
        for sn in self.subnics:
            ss += sn.sockets_wait(env)
        return ss
805
806
807


class WireNet(NetSim):
Jonas Kaufmann's avatar
Jonas Kaufmann committed
808

809
    def run_cmd(self, env: ExpEnv) -> str:
Jialin Li's avatar
Jialin Li committed
810
        connects = self.connect_sockets(env)
811
        assert len(connects) == 2
812
813
        cmd = (
            f'{env.repodir}/sims/net/wire/net_wire {connects[0][1]}'
814
            f' {connects[1][1]} {self.sync_mode} {self.sync_period}'
815
816
            f' {self.eth_latency}'
        )
817
818
819
        if len(env.pcap_file) > 0:
            cmd += ' ' + env.pcap_file
        return cmd
820

Jonas Kaufmann's avatar
Jonas Kaufmann committed
821

822
class SwitchNet(NetSim):
823

824
    def __init__(self) -> None:
825
826
        super().__init__()
        self.sync = True
827
        """Whether to synchronize with attached simulators."""
828

829
    def run_cmd(self, env: ExpEnv) -> str:
830
        cmd = env.repodir + '/sims/net/switch/net_switch'
831
        cmd += f' -S {self.sync_period} -E {self.eth_latency}'
832
833
834
835

        if not self.sync:
            cmd += ' -u'

836
837
        if len(env.pcap_file) > 0:
            cmd += ' -p ' + env.pcap_file
Jonas Kaufmann's avatar
Jonas Kaufmann committed
838
        for (_, n) in self.connect_sockets(env):
839
            cmd += ' -s ' + n
Jonas Kaufmann's avatar
Jonas Kaufmann committed
840
        for (_, n) in self.listen_sockets(env):
841
            cmd += ' -h ' + n
842
        return cmd
843

844
    def sockets_cleanup(self, env: ExpEnv) -> tp.List[str]:
845
        # cleanup here will just have listening eth sockets, switch also creates
Hejing Li's avatar
Hejing Li committed
846
847
848
849
850
851
852
853
854
855
        # shm regions for each with a "-shm" suffix
        cleanup = []
        for s in super().sockets_cleanup(env):
            cleanup.append(s)
            cleanup.append(s + '-shm')
        return cleanup


class MemSwitchNet(NetSim):

856
    def __init__(self) -> None:
Hejing Li's avatar
Hejing Li committed
857
858
859
860
861
        super().__init__()
        self.sync = True
        """ AS_ID,VADDR_START,VADDR_END,MEMNODE_MAC,PHYS_START """
        self.mem_map = []

862
    def run_cmd(self, env: ExpEnv) -> str:
Hejing Li's avatar
Hejing Li committed
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
        cmd = env.repodir + '/sims/mem/memswitch/memswitch'
        cmd += f' -S {self.sync_period} -E {self.eth_latency}'

        if not self.sync:
            cmd += ' -u'

        if len(env.pcap_file) > 0:
            cmd += ' -p ' + env.pcap_file
        for (_, n) in self.connect_sockets(env):
            cmd += ' -s ' + n
        for (_, n) in self.listen_sockets(env):
            cmd += ' -h ' + n
        for m in self.mem_map:
            cmd += ' -m ' + f' {m[0]},{m[1]},{m[2]},'
            cmd += (''.join(reversed(m[3].split(':'))))
            cmd += f',{m[4]}'
        return cmd

881
    def sockets_cleanup(self, env: ExpEnv) -> tp.List[str]:
Hejing Li's avatar
Hejing Li committed
882
        # cleanup here will just have listening eth sockets, switch also creates
883
884
885
886
887
888
889
        # shm regions for each with a "-shm" suffix
        cleanup = []
        for s in super().sockets_cleanup(env):
            cleanup.append(s)
            cleanup.append(s + '-shm')
        return cleanup

Jonas Kaufmann's avatar
Jonas Kaufmann committed
890

891
class TofinoNet(NetSim):
892

893
    def __init__(self) -> None:
894
895
896
        super().__init__()
        self.tofino_log_path = '/tmp/model.ldjson'
        self.sync = True
Jialin Li's avatar
Jialin Li committed
897

898
    def run_cmd(self, env: ExpEnv) -> str:
899
900
901
902
903
        cmd = f'{env.repodir}/sims/net/tofino/tofino'
        cmd += (
            f' -S {self.sync_period} -E {self.eth_latency}'
            f' -t {self.tofino_log_path}'
        )
904
905
        if not self.sync:
            cmd += ' -u'
Jonas Kaufmann's avatar
Jonas Kaufmann committed
906
        for (_, n) in self.connect_sockets(env):
907
            cmd += ' -s ' + n
908
        return cmd
909

Jonas Kaufmann's avatar
Jonas Kaufmann committed
910

911
912
913
914
class NS3E2ENet(NetSim):

    def __init__(self) -> None:
        super().__init__()
915
916
917
918
        self.first_run = True
        self.e2e_components: tp.List[tp.Union[e2e.E2ETopologyNode,
                                              e2e.E2ETopologyChannel]] = []
        self.e2e_topologies: tp.List[E2ETopology] = []
919
        self.use_file = True
920
921
922
923
924
925
926
927
928
929
930

    def add_component(
        self,
        component: tp.Union[e2e.E2ETopologyNode,
                            e2e.E2ETopologyChannel,
                            E2ETopology]
    ):
        if isinstance(component, E2ETopology):
            self.e2e_topologies.append(component)
        else:
            self.e2e_components.append(component)
931
932

    def resolve_socket_paths(
933
934
935
936
        self,
        env: ExpEnv,
        e2e_sim: tp.Union[e2e.E2ESimbricksNetwork, e2e.E2ESimbricksHost],
        listen: bool = False
937
    ) -> None:
938
939
        if e2e_sim.simbricks_component is None:
            print('E2E Simbricks adapter does not contain a simulator')
940
            sys.exit(1)
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
        if e2e_sim.adapter_type == e2e.SimbricksAdapterType.NIC:
            e2e_sim.unix_socket = env.nic_eth_path(e2e_sim.simbricks_component)
        elif e2e_sim.adapter_type == e2e.SimbricksAdapterType.NETWORK:
            if listen:
                e2e_sim.unix_socket = env.n2n_eth_path(
                    self, e2e_sim.simbricks_component
                )
            else:
                e2e_sim.unix_socket = env.n2n_eth_path(
                    e2e_sim.simbricks_component, self
                )
        elif e2e_sim.adapter_type == e2e.SimbricksAdapterType.HOST:
            e2e_sim.unix_socket = env.net2host_eth_path(
                self, e2e_sim.simbricks_component
            )
956
957

    def run_cmd(self, env):
958
959
960
961
962
963
964
        if self.first_run:
            for topo in self.e2e_topologies:
                topo.add_to_network(self)

        for component in self.e2e_components:
            if self.first_run:
                component.resolve_paths()
965

966
967
            for c in component.components:
                if isinstance(c, e2e.E2ESimbricksHost):
968
                    self.resolve_socket_paths(env, c)
969
970
971
972
973
974
                elif isinstance(c, e2e.E2ESimbricksNetworkNetIf):
                    self.resolve_socket_paths(env, c)
                    if self.first_run:
                        self.connect_network(c.simbricks_component)
                elif isinstance(c, e2e.E2ESimbricksNetworkNicIf):
                    self.resolve_socket_paths(env, c, True)
975

976
977
        self.first_run = False

978
979
980
981
        params: tp.List[str] = []
        for component in self.e2e_components:
            params.append(component.ns3_config())

982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
        params_str = f'{" ".join(params)} {self.opt}'

        if self.use_file:
            file_path = env.ns3_e2e_params_file(self)
            with open(file_path, 'w', encoding='utf-8') as f:
                f.write(params_str)
            cmd = (
                f'{env.repodir}/sims/external/ns-3'
                f'/simbricks-run.sh e2e-cc-example --ConfigFile={file_path}'
            )
        else:
            cmd = (
                f'{env.repodir}/sims/external/ns-3'
                f'/simbricks-run.sh e2e-cc-example {params_str}'
            )
997
998
999
1000
1001
        print(cmd)

        return cmd


1002
class NS3DumbbellNet(NetSim):
Jonas Kaufmann's avatar
Jonas Kaufmann committed
1003

1004
    def run_cmd(self, env: ExpEnv) -> str:
1005
        ports = ''
Jonas Kaufmann's avatar
Jonas Kaufmann committed
1006
        for (n, s) in self.connect_sockets(env):
1007
            if 'server' in n.name:
Marvin Meiers's avatar
Marvin Meiers committed
1008
                ports += f'--SimbricksPortLeft={s} '
1009
            else:
Marvin Meiers's avatar
Marvin Meiers committed
1010
                ports += f'--SimbricksPortRight={s} '
1011

1012
1013
        cmd = (
            f'{env.repodir}/sims/external/ns-3'
Marvin Meiers's avatar
Marvin Meiers committed
1014
            f'/simbricks-run.sh simbricks-dumbbell-example {ports} {self.opt}'
1015
        )
1016
1017
1018
1019
        print(cmd)

        return cmd

Jonas Kaufmann's avatar
Jonas Kaufmann committed
1020

1021
class NS3BridgeNet(NetSim):
Jonas Kaufmann's avatar
Jonas Kaufmann committed
1022

1023
    def run_cmd(self, env: ExpEnv) -> str:
1024
        ports = ''
Jonas Kaufmann's avatar
Jonas Kaufmann committed
1025
        for (_, n) in self.connect_sockets(env):
Marvin Meiers's avatar
Marvin Meiers committed
1026
            ports += '--SimbricksPort=' + n + ' '
1027

1028
1029
        cmd = (
            f'{env.repodir}/sims/external/ns-3'
Marvin Meiers's avatar
Marvin Meiers committed
1030
            f'/simbricks-run.sh simbricks-bridge-example {ports} {self.opt}'
1031
        )
1032
1033
1034
        print(cmd)

        return cmd
1035

Jonas Kaufmann's avatar
Jonas Kaufmann committed
1036

1037
class NS3SequencerNet(NetSim):
Jonas Kaufmann's avatar
Jonas Kaufmann committed
1038

1039
    def run_cmd(self, env: ExpEnv) -> str:
1040
        ports = ''
Jonas Kaufmann's avatar
Jonas Kaufmann committed
1041
        for (n, s) in self.connect_sockets(env):
1042
            if 'client' in n.name:
1043
                ports += '--ClientPort=' + s + ' '
1044
            elif 'replica' in n.name:
1045
                ports += '--ServerPort=' + s + ' '
1046
            elif 'sequencer' in n.name:
1047
                ports += '--ServerPort=' + s + ' '
1048
            else:
1049
                raise KeyError('Wrong NIC type')
1050
1051
        cmd = (
            f'{env.repodir}/sims/external/ns-3'
Marvin Meiers's avatar
Marvin Meiers committed
1052
            f'/simbricks-run.sh sequencer-single-switch-example'
1053
1054
            f' {ports} {self.opt}'
        )
1055
1056
1057
        return cmd


1058
class FEMUDev(PCIDevSim):
Jonas Kaufmann's avatar
Jonas Kaufmann committed
1059

1060
    def run_cmd(self, env: ExpEnv) -> str:
1061
1062
1063
1064
        cmd = (
            f'{env.repodir}/sims/external/femu/femu-simbricks'
            f' {env.dev_pci_path(self)} {env.dev_shm_path(self)}'
        )
1065
        return cmd
1066
1067
1068
1069


class BasicMemDev(MemDevSim):

1070
    def run_cmd(self, env: ExpEnv) -> str:
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
        cmd = (
            f'{env.repodir}/sims/mem/basicmem/basicmem'
            f' {self.size} {self.addr} {self.as_id}'
            f' {env.dev_mem_path(self)} {env.dev_shm_path(self)}'
            f' {self.sync_mode} {self.start_tick} {self.sync_period}'
            f' {self.mem_latency}'
        )
        return cmd


class MemNIC(MemDevSim):

1083
    def run_cmd(self, env: ExpEnv) -> str:
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
        cmd = (
            f'{env.repodir}/sims/mem/memnic/memnic'
            f' {env.dev_mem_path(self)} {env.nic_eth_path(self)}'
            f' {env.dev_shm_path(self)}'
        )

        if self.mac is not None:
            cmd += ' ' + (''.join(reversed(self.mac.split(':'))))

        cmd += f' {self.sync_mode} {self.start_tick} {self.sync_period}'
        cmd += f' {self.mem_latency} {self.eth_latency}'

        return cmd

1098
    def sockets_cleanup(self, env: ExpEnv) -> tp.List[str]:
1099
1100
        return super().sockets_cleanup(env) + [env.nic_eth_path(self)]

1101
    def sockets_wait(self, env: ExpEnv) -> tp.List[str]:
1102
1103
1104
1105
1106
        return super().sockets_wait(env) + [env.nic_eth_path(self)]


class NetMem(NetMemSim):

1107
    def run_cmd(self, env: ExpEnv) -> str:
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
        cmd = (
            f'{env.repodir}/sims/mem/netmem/netmem'
            f' {self.size} {self.addr} {self.as_id}'
            f' {env.nic_eth_path(self)}'
            f' {env.dev_shm_path(self)}'
        )
        if self.mac is not None:
            cmd += ' ' + (''.join(reversed(self.mac.split(':'))))

        cmd += f' {self.sync_mode} {self.start_tick} {self.sync_period}'
        cmd += f' {self.eth_latency}'

        return cmd