simulators.py 34.5 KB
Newer Older
Antoine Kaufmann's avatar
Antoine Kaufmann committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
# Copyright 2021 Max Planck Institute for Software Systems, and
# National University of Singapore
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

23
24
25
# Allow own class to be used as type for a method's argument
from __future__ import annotations

26
import math
27
import sys
28
29
import typing as tp

30
31
from simbricks.orchestration.experiment.experiment_environment import ExpEnv
from simbricks.orchestration.nodeconfig import NodeConfig
32
from simbricks.orchestration.e2e_components import E2ETopology, E2ESimbricksHost
33

34

35
class Simulator(object):
36
37
    """Base class for all simulators."""

38
39
    def __init__(self) -> None:
        self.extra_deps: tp.List[Simulator] = []
Jonas Kaufmann's avatar
Jonas Kaufmann committed
40
        self.name = ''
41

42
    def resreq_cores(self) -> int:
43
44
45
46
47
        """
        Number of cores this simulator requires during execution.

        This is used for scheduling multiple runs and experiments.
        """
48
49
        return 1

50
    def resreq_mem(self) -> int:
51
52
53
54
55
        """
        Number of memory in MB this simulator requires during execution.

        This is used for scheduling multiple runs and experiments.
        """
56
57
        return 64

58
    def full_name(self) -> str:
Jonas Kaufmann's avatar
Jonas Kaufmann committed
59
60
61
        """Full name of the simulator."""
        return ''

62
    # pylint: disable=unused-argument
63
    def prep_cmds(self, env: ExpEnv) -> tp.List[str]:
64
        """Commands to prepare execution of this simulator."""
65
66
        return []

67
    # pylint: disable=unused-argument
68
    def run_cmd(self, env: ExpEnv) -> tp.Optional[str]:
69
        """Command to execute this simulator."""
70
        return None
71

Jonas Kaufmann's avatar
Jonas Kaufmann committed
72
    def dependencies(self) -> tp.List[Simulator]:
73
        """Other simulators to execute before this one."""
74
75
76
        return []

    # Sockets to be cleaned up
77
    # pylint: disable=unused-argument
78
    def sockets_cleanup(self, env: ExpEnv) -> tp.List[str]:
79
80
81
        return []

    # sockets to wait for indicating the simulator is ready
82
    # pylint: disable=unused-argument
83
    def sockets_wait(self, env: ExpEnv) -> tp.List[str]:
84
85
        return []

86
    def start_delay(self) -> int:
87
88
        return 5

89
    def wait_terminate(self) -> bool:
90
91
        return False

92

93
class PCIDevSim(Simulator):
94
95
    """Base class for PCIe device simulators."""

96
    def __init__(self) -> None:
97
98
99
        super().__init__()

        self.sync_mode = 0
100
101
        """Synchronization mode. 0 is running unsynchronized, 1 synchronized.
        Depending on the concrete simulator, there may be additional modes."""
102
        self.start_tick = 0
103
104
105
106
107
        """The timestamp at which to start the simulation. This is useful when
        the simulator is only attached at a later point in time and needs to
        synchronize with connected simulators. For example, this could be used
        when taking checkpoints to only attach certain simulators after the
        checkpoint has been taken."""
108
        self.sync_period = 500
109
110
        """Period in nanoseconds of sending synchronization messages from this
        device to connected components."""
111
        self.pci_latency = 500
112
113
        """Latency in nanoseconds for sending messages to components connected
        via PCI."""
114

115
    def full_name(self) -> str:
116
117
        return 'dev.' + self.name

118
    def is_nic(self) -> bool:
119
120
        return False

121
    def sockets_cleanup(self, env: ExpEnv) -> tp.List[str]:
122
123
        return [env.dev_pci_path(self), env.dev_shm_path(self)]

124
    def sockets_wait(self, env: ExpEnv) -> tp.List[str]:
125
126
127
128
        return [env.dev_pci_path(self)]


class NICSim(PCIDevSim):
129
130
    """Base class for NIC simulators."""

131
    def __init__(self) -> None:
132
133
134
135
136
        super().__init__()

        self.network: tp.Optional[NetSim] = None
        self.mac: tp.Optional[str] = None
        self.eth_latency = 500
137
138
        """Ethernet latency in nanoseconds from this NIC to the network
        component."""
139

140
    def set_network(self, net: NetSim) -> None:
141
        """Connect this NIC to a network simulator."""
142
143
144
        self.network = net
        net.nics.append(self)

145
    def basic_args(self, env: ExpEnv, extra: tp.Optional[str] = None) -> str:
146
147
        cmd = (
            f'{env.dev_pci_path(self)} {env.nic_eth_path(self)}'
148
149
            f' {env.dev_shm_path(self)} {self.sync_mode} {self.start_tick}'
            f' {self.sync_period} {self.pci_latency} {self.eth_latency}'
150
        )
151
152
        if self.mac is not None:
            cmd += ' ' + (''.join(reversed(self.mac.split(':'))))
153
154
155
156

        if extra is not None:
            cmd += ' ' + extra
        return cmd
157

158
159
160
    def basic_run_cmd(
        self, env: ExpEnv, name: str, extra: tp.Optional[str] = None
    ) -> str:
161
        cmd = f'{env.repodir}/sims/nic/{name} {self.basic_args(env, extra)}'
162
163
        return cmd

164
    def full_name(self) -> str:
165
        return 'nic.' + self.name
166

167
    def is_nic(self) -> bool:
168
169
        return True

170
    def sockets_cleanup(self, env: ExpEnv) -> tp.List[str]:
171
        return super().sockets_cleanup(env) + [env.nic_eth_path(self)]
172

173
    def sockets_wait(self, env: ExpEnv) -> tp.List[str]:
174
        return super().sockets_wait(env) + [env.nic_eth_path(self)]
175

Jonas Kaufmann's avatar
Jonas Kaufmann committed
176

177
class NetSim(Simulator):
178
    """Base class for network simulators."""
179

180
    def __init__(self) -> None:
181
182
183
184
        super().__init__()

        self.opt = ''
        self.sync_mode = 0
185
186
        """Synchronization mode. 0 is running unsynchronized, 1 synchronized.
        Depending on the concrete simulator, there may be additional modes."""
187
        self.sync_period = 500
188
189
        """Synchronization period in nanoseconds from this network to connected
        components."""
190
        self.eth_latency = 500
191
192
        """Ethernet latency in nanoseconds from this network to connected
        components."""
193
194
195
196
        self.nics: list[NICSim] = []
        self.hosts_direct: list[HostSim] = []
        self.net_listen: list[NetSim] = []
        self.net_connect: list[NetSim] = []
197

198
    def full_name(self) -> str:
199
        return 'net.' + self.name
200

201
    def connect_network(self, net: NetSim) -> None:
202
        """Connect this network to the listening peer `net`"""
203
204
205
        net.net_listen.append(self)
        self.net_connect.append(net)

Jonas Kaufmann's avatar
Jonas Kaufmann committed
206
    def connect_sockets(self, env: ExpEnv) -> tp.List[tp.Tuple[Simulator, str]]:
207
208
209
        sockets = []
        for n in self.nics:
            sockets.append((n, env.nic_eth_path(n)))
210
211
        for n in self.net_connect:
            sockets.append((n, env.n2n_eth_path(n, self)))
212
213
        for h in self.hosts_direct:
            sockets.append((h, env.net2host_eth_path(self, h)))
214
215
        return sockets

216
    def listen_sockets(self, env: ExpEnv) -> tp.List[tp.Tuple[NetSim, str]]:
217
218
219
220
221
        listens = []
        for net in self.net_listen:
            listens.append((net, env.n2n_eth_path(self, net)))
        return listens

222
    def dependencies(self) -> tp.List[Simulator]:
223
        return self.nics + self.net_connect + self.hosts_direct
224

225
    def sockets_cleanup(self, env: ExpEnv) -> tp.List[str]:
Jonas Kaufmann's avatar
Jonas Kaufmann committed
226
        return [s for (_, s) in self.listen_sockets(env)]
227

228
    def sockets_wait(self, env: ExpEnv) -> tp.List[str]:
Jonas Kaufmann's avatar
Jonas Kaufmann committed
229
        return [s for (_, s) in self.listen_sockets(env)]
230

231

232
233
234
235
# FIXME: Class hierarchy is broken here as an ugly hack
class MemDevSim(NICSim):
    """Base class for memory device simulators."""

236
    def __init__(self) -> None:
237
238
239
240
241
242
243
        super().__init__()

        self.mem_latency = 500
        self.addr = 0xe000000000000000
        self.size = 1024 * 1024 * 1024  # 1GB
        self.as_id = 0

244
    def full_name(self) -> str:
245
246
        return 'mem.' + self.name

247
    def sockets_cleanup(self, env: ExpEnv) -> tp.List[str]:
248
249
        return [env.dev_mem_path(self), env.dev_shm_path(self)]

250
    def sockets_wait(self, env: ExpEnv) -> tp.List[str]:
251
252
253
254
255
256
        return [env.dev_mem_path(self)]


class NetMemSim(NICSim):
    """Base class for netork memory simulators."""

257
    def __init__(self) -> None:
258
259
260
261
262
263
        super().__init__()

        self.addr = 0xe000000000000000
        self.size = 1024 * 1024 * 1024  # 1GB
        self.as_id = 0

264
    def full_name(self) -> str:
265
266
        return 'netmem.' + self.name

267
    def sockets_cleanup(self, env: ExpEnv) -> tp.List[str]:
268
269
        return [env.nic_eth_path(self), env.dev_shm_path(self)]

270
    def sockets_wait(self, env: ExpEnv) -> tp.List[str]:
271
272
273
        return [env.nic_eth_path(self)]


274
class HostSim(Simulator):
275
    """Base class for host simulators."""
276

277
    def __init__(self, node_config: NodeConfig) -> None:
278
        super().__init__()
279
        self.node_config = node_config
280
        """System configuration for this simulated host. """
281
282
        self.wait = False
        """
283
284
285
        `True` - Wait for this simulator to finish execution. `False` - Don't
        wait and instead shutdown the simulator as soon as all other awaited
        simulators have completed execution.
286
287
        """
        self.sleep = 0
288
        self.cpu_freq = '4GHz'
289
290

        self.sync_mode = 0
291
292
        """Synchronization mode. 0 is running unsynchronized, 1 synchronized.
        Depending on the concrete simulator, there may be additional modes."""
293
        self.sync_period = 500
294
295
        """Period in nanoseconds of sending synchronization messages from this
        device to connected components."""
296
        self.pci_latency = 500
297
298
        """Latency in nanoseconds for sending messages to components connected
        via PCIe."""
299
        self.mem_latency = 500
300
301
        """Latency in nanoseconds for sending messages to components connected
        via Ethernet."""
302

303
304
305
306
307
308
309
310
        self.sync_drift: tp.Optional[int] = None
        """Conversion factor from SimBricks' picosecond timestamps to the host's
        nanosecond clock. None or 1000 is 1:1, while other values result in
        simulated clock drift for a host."""
        self.sync_offset: tp.Optional[int] = None
        """Clock offset for this host in picoseconds. None or 0 does not add an
        offset."""

311
        self.pcidevs: tp.List[PCIDevSim] = []
312
        self.net_directs: tp.List[NetSim] = []
313
        self.memdevs: tp.List[MemDevSim] = []
314

315
    @property
316
317
318
319
320
321
    def nics(self) -> tp.List[NICSim]:
        return [
            tp.cast(NICSim, pcidev)
            for pcidev in self.pcidevs
            if pcidev.is_nic()
        ]
322

323
    def full_name(self) -> str:
324
325
        return 'host.' + self.name

326
    def add_nic(self, dev: NICSim) -> None:
327
        """Add a NIC to this host."""
328
329
        self.add_pcidev(dev)

330
    def add_pcidev(self, dev: PCIDevSim) -> None:
331
        """Add a PCIe device to this host."""
332
333
334
        dev.name = self.name + '.' + dev.name
        self.pcidevs.append(dev)

335
    def add_memdev(self, dev: MemDevSim) -> None:
336
337
338
        dev.name = self.name + '.' + dev.name
        self.memdevs.append(dev)

339
    def add_netdirect(self, net: NetSim) -> None:
340
        """Add a direct connection to a network to this host."""
341
342
343
        net.hosts_direct.append(self)
        self.net_directs.append(net)

344
    def dependencies(self) -> tp.List[PCIDevSim]:
345
346
347
348
349
        deps = []
        for dev in self.pcidevs:
            deps.append(dev)
            if isinstance(dev, NICSim):
                deps.append(dev.network)
350
351
        for dev in self.memdevs:
            deps.append(dev)
352
353
        return deps

354
    def wait_terminate(self) -> bool:
355
356
357
        return self.wait


358
class QemuHost(HostSim):
359
360
    """Qemu host simulator."""

361
    def __init__(self, node_config: NodeConfig) -> None:
362
        super().__init__(node_config)
363
364

        self.sync = False
365
        """"Whether to synchronize with attached simulators."""
366

367
    def resreq_cores(self) -> int:
368
369
370
371
        if self.sync:
            return 1
        else:
            return self.node_config.cores + 1
372

373
    def resreq_mem(self) -> int:
Hejing Li's avatar
Hejing Li committed
374
        return 8192
375

376
    def prep_cmds(self, env: ExpEnv) -> tp.List[str]:
Jonas Kaufmann's avatar
Jonas Kaufmann committed
377
378
        return [
            f'{env.qemu_img_path} create -f qcow2 -o '
379
            f'backing_file="{env.hd_path(self.node_config.disk_image)}" '
Jonas Kaufmann's avatar
Jonas Kaufmann committed
380
381
            f'{env.hdcopy_path(self)}'
        ]
382

383
    def run_cmd(self, env: ExpEnv) -> str:
384
        accel = ',accel=kvm:tcg' if not self.sync else ''
385
386
387
388
389
        if self.node_config.kcmd_append:
            kcmd_append = ' ' + self.node_config.kcmd_append
        else:
            kcmd_append = ''

Jonas Kaufmann's avatar
Jonas Kaufmann committed
390
391
        cmd = (
            f'{env.qemu_path} -machine q35{accel} -serial mon:stdio '
392
            '-cpu Skylake-Server -display none -nic none '
393
394
395
            f'-kernel {env.qemu_kernel_path} '
            f'-drive file={env.hdcopy_path(self)},if=ide,index=0,media=disk '
            f'-drive file={env.cfgtar_path(self)},if=ide,index=1,media=disk,'
Jonas Kaufmann's avatar
Jonas Kaufmann committed
396
            'driver=raw '
397
            '-append "earlyprintk=ttyS0 console=ttyS0 root=/dev/sda1 '
398
            f'init=/home/ubuntu/guestinit.sh rw{kcmd_append}" '
Jonas Kaufmann's avatar
Jonas Kaufmann committed
399
400
            f'-m {self.node_config.memory} -smp {self.node_config.cores} '
        )
401
402

        if self.sync:
403
404
405
406
407
408
            unit = self.cpu_freq[-3:]
            if unit.lower() == 'ghz':
                base = 0
            elif unit.lower() == 'mhz':
                base = 3
            else:
409
                raise ValueError('cpu frequency specified in unsupported unit')
410
411
412
            num = float(self.cpu_freq[:-3])
            shift = base - int(math.ceil(math.log(num, 2)))

413
            cmd += f' -icount shift={shift},sleep=off '
414

415
        for dev in self.pcidevs:
416
            cmd += f'-device simbricks-pci,socket={env.dev_pci_path(dev)}'
417
418
419
420
            if self.sync:
                cmd += ',sync=on'
                cmd += f',pci-latency={self.pci_latency}'
                cmd += f',sync-period={self.sync_period}'
421
422
423
424
                if self.sync_drift is not None:
                    cmd += f',sync-drift={self.sync_drift}'
                if self.sync_offset is not None:
                    cmd += f',sync-offset={self.sync_offset}'
425
426
427
428
            else:
                cmd += ',sync=off'
            cmd += ' '

429
        # qemu does not currently support net direct ports
430
        assert len(self.net_directs) == 0
431
432
        # qemu does not currently support mem device ports
        assert len(self.memdevs) == 0
433
434
        return cmd

Jonas Kaufmann's avatar
Jonas Kaufmann committed
435

436
class Gem5Host(HostSim):
437
    """Gem5 host simulator."""
438

439
    def __init__(self, node_config: NodeConfig) -> None:
440
441
        node_config.sim = 'gem5'
        super().__init__(node_config)
442
443
444
        self.cpu_type_cp = 'X86KvmCPU'
        self.cpu_type = 'TimingSimpleCPU'
        self.sys_clock = '1GHz'
445
446
        self.extra_main_args = []
        self.extra_config_args = []
447
        self.variant = 'fast'
448
449
450
451
452
453
454
455
        self.modify_checkpoint_tick = True
        """Whether to modify the event queue tick before restoring a checkpoint.
        When this is enabled, the restored checkpoint will start at event queue
        tick 0. This is a performance optimization since now, connected
        simulators don't have to simulate and synchronize until the restored
        tick before the actual workload can be executed. Disable this if you
        need to retain the differences in virtual time between multiple gem5
        instances."""
456

457
    def resreq_cores(self) -> int:
458
459
        return 1

460
    def resreq_mem(self) -> int:
461
462
        return 4096

463
    def prep_cmds(self, env: ExpEnv) -> tp.List[str]:
464
465
466
467
468
469
470
        cmds = [f'mkdir -p {env.gem5_cpdir(self)}']
        if env.restore_cp and self.modify_checkpoint_tick:
            cmds.append(
                f'python3 {env.utilsdir}/modify_gem5_cp_tick.py --tick 0 '
                f'--cpdir {env.gem5_cpdir(self)}'
            )
        return cmds
471

472
    def run_cmd(self, env: ExpEnv) -> str:
473
474
475
476
        cpu_type = self.cpu_type
        if env.create_cp:
            cpu_type = self.cpu_type_cp

477
        cmd = f'{env.gem5_path(self.variant)} --outdir={env.gem5_outdir(self)} '
478
        cmd += ' '.join(self.extra_main_args)
Jonas Kaufmann's avatar
Jonas Kaufmann committed
479
        cmd += (
480
481
482
            f' {env.gem5_py_path} --caches --l2cache '
            '--l1d_size=32kB --l1i_size=32kB --l2_size=32MB '
            '--l1d_assoc=8 --l1i_assoc=8 --l2_assoc=16 '
483
484
            f'--cacheline_size=64 --cpu-clock={self.cpu_freq}'
            f' --sys-clock={self.sys_clock} '
485
486
            f'--checkpoint-dir={env.gem5_cpdir(self)} '
            f'--kernel={env.gem5_kernel_path} '
487
            f'--disk-image={env.hd_raw_path(self.node_config.disk_image)} '
488
            f'--disk-image={env.cfgtar_path(self)} '
489
            f'--cpu-type={cpu_type} --mem-size={self.node_config.memory}MB '
490
            f'--num-cpus={self.node_config.cores} '
491
            '--mem-type=DDR4_2400_16x4 '
Jonas Kaufmann's avatar
Jonas Kaufmann committed
492
        )
493

494
495
496
        if self.node_config.kcmd_append:
            cmd += f'--command-line-append="{self.node_config.kcmd_append}" '

497
498
        if env.create_cp:
            cmd += '--max-checkpoints=1 '
499

500
        if env.restore_cp:
501
            cmd += '-r 1 '
502

503
        for dev in self.pcidevs:
Jonas Kaufmann's avatar
Jonas Kaufmann committed
504
505
506
507
508
            cmd += (
                f'--simbricks-pci=connect:{env.dev_pci_path(dev)}'
                f':latency={self.pci_latency}ns'
                f':sync_interval={self.sync_period}ns'
            )
509
            if cpu_type == 'TimingSimpleCPU':
510
                cmd += ':sync'
Jonas Kaufmann's avatar
Jonas Kaufmann committed
511
            cmd += ' '
512

513
514
515
516
517
518
519
520
521
522
523
        for dev in self.memdevs:
            cmd += (
                f'--simbricks-mem={dev.size}@{dev.addr}@{dev.as_id}@'
                f'connect:{env.dev_mem_path(dev)}'
                f':latency={self.mem_latency}ns'
                f':sync_interval={self.sync_period}ns'
            )
            if cpu_type == 'TimingSimpleCPU':
                cmd += ':sync'
            cmd += ' '

524
        for net in self.net_directs:
Jonas Kaufmann's avatar
Jonas Kaufmann committed
525
526
527
528
529
530
531
            cmd += (
                '--simbricks-eth-e1000=listen'
                f':{env.net2host_eth_path(net, self)}'
                f':{env.net2host_shm_path(net, self)}'
                f':latency={net.eth_latency}ns'
                f':sync_interval={net.sync_period}ns'
            )
532
533
            if cpu_type == 'TimingSimpleCPU':
                cmd += ':sync'
Jonas Kaufmann's avatar
Jonas Kaufmann committed
534
            cmd += ' '
535
536

        cmd += ' '.join(self.extra_config_args)
537
538
539
        return cmd


540
541
542
class SimicsHost(HostSim):
    """Simics host simulator."""

543
    def __init__(self, node_config: NodeConfig) -> None:
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
        super().__init__(node_config)
        node_config.sim = 'simics'

        self.cpu_class = 'x86-cooper-lake'
        """Simics CPU class. Can be obtained by running `list-classes substr =
        processor_` inside Simics."""
        self.cpu_freq = 4000  # TODO Don't hide attribute in super class
        """CPU frequency in MHz"""
        self.timing = False
        """Whether to run Simics in a more precise timing mode. This adds a
        cache model."""
        self.append_cmdline: tp.List[str] = []
        """Additional parameters to append on the command-line when invoking
        Simics."""
        self.interactive = False
        """Whether to launch Simics in interactive GUI mode. This is helpful for
        debugging, e.g. enabling log messages in the mid of the simulation."""
        self.debug_messages = False
        """Whether to enable debug messages of SimBricks adapter devices."""

564
    def resreq_cores(self) -> int:
565
566
        return 2

567
    def resreq_mem(self) -> int:
568
569
        return self.node_config.memory

570
    def run_cmd(self, env: ExpEnv) -> str:
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
        if self.node_config.kcmd_append:
            raise RuntimeError(
                'Appending kernel command-line not yet implemented.'
            )

        if self.interactive and not env.create_cp:
            cmd = f'{env.simics_gui_path} -q '
        else:
            cmd = f'{env.simics_path} -q -batch-mode -werror '

        if env.restore_cp:
            # restore checkpoint
            cmd += f'-e \'read-configuration {env.simics_cpfile(self)}\' '
        else:
            # initialize simulated machine
            cmd += (
                '-e \'run-command-file '
                f'{env.simics_qsp_modern_core_path} '
                f'disk0_image = {env.hd_raw_path(self.node_config.disk_image)} '
                f'disk1_image = {env.cfgtar_path(self)} '
                f'cpu_comp_class = {self.cpu_class} '
                f'freq_mhz = {self.cpu_freq} '
                f'num_cores = {self.node_config.cores} '
                f'num_threads = {self.node_config.threads} '
595
596
                f'memory_megs = {self.node_config.memory} '
                'create_network = FALSE\' '
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
            )

        if env.create_cp:
            # stop simulation when encountering special checkpoint string on
            # serial console
            cmd += (
                '-e \'bp.console_string.break board.serconsole.con '
                '"ready to checkpoint"\' '
            )
            # run simulation
            cmd += '-e run '
            # create checkpoint
            cmd += f'-e \'write-configuration {env.simics_cpfile(self)}\' '
            return cmd

        if self.timing:
            # Add the cache model. Note that the caches aren't warmed up during
            # the boot process. The reason is that when later adding the memory
            # devices, we change the mapped memory. The cycle staller doesn't
            # like this and will SEGFAULT.
            #
            # The cache model doesn't store any memory contents and therefore
            # doesn't answer any memory transactions. It only inserts CPU stall
            # cycles on each cache level and can be queried for statistics as
            # well as which addresses are cached.
            #
            # Read penalties are based on https://www.7-cpu.com/cpu/Skylake.html
            cmd += (
                '-e \'new-cycle-staller name = cs0 '
                'stall-interval = 10000\' '
            )
            cmd += (
                '-e \'new-simple-cache-tool name = cachetool '
                'cycle-staller = cs0 -connect-all\' '
            )
            cmd += (
                '-e \'cachetool.add-l1i-cache name = l1i line-size = 64 '
                'sets = 64 ways = 8\' '
            )
            cmd += (
                '-e \'cachetool.add-l1d-cache name = l1d line-size = 64 '
                'sets = 64 ways = 8 -ip-read-prefetcher '
                'prefetch-additional = 1 read-penalty = 4\' '
            )
            cmd += (
                '-e \'cachetool.add-l2-cache name = l2 line-size = 64 '
                'sets = 8192 ways = 4 -prefetch-adjacent '
                'prefetch-additional = 4 read-penalty = 12\' '
            )
            cmd += (
                '-e \'cachetool.add-l3-cache name = l3 line-size = 64 '
                'sets = 32768 ways = 16 read-penalty = 42\' '
            )

        # Only simulate one cycle per CPU and then switch to the next. This is
        # necessary for the synchronization of the SimBricks adapter with all
        # the CPUs to work properly.
        cmd += '-e \'set-time-quantum 1\' '

        if self.memdevs:
            cmd += '-e \'load-module simbricks_mem_comp\' '

        for memdev in self.memdevs:
            cmd += (
                f'-e \'$mem = (new-simbricks-mem-comp '
                f'socket = "{env.dev_mem_path(memdev)}" '
                f'mem_latency = {self.mem_latency} '
                f'sync_period = {self.sync_period})\' '
            )
            cmd += (
                f'-e \'board.mb.dram_space.add-map $mem.simbricks_mem_dev '
                f'{memdev.addr:#x} {memdev.size:#x}\' '
            )
            if self.debug_messages:
                cmd += '-e \'$mem.log-level 3\' '

673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
        if self.pcidevs:
            cmd += '-e \'load-module simbricks_pcie_comp\' '

        i = 0
        for pcidev in self.pcidevs:
            cmd += (
                f'-e \'$pci = (create-simbricks-pcie-comp '
                f'socket = "{env.dev_pci_path(pcidev)}" '
                f'pci_latency = {self.pci_latency} '
                f'sync_period = {self.sync_period})\' '
            )
            cmd += f'-e \'connect board.mb.nb.pci_slot[{i}] $pci.pci_bus\' '
            cmd += '-e instantiate-components '
            if self.debug_messages:
                cmd += '-e \'$pci.log-level 3\' '
            i += 1

690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
        for param in self.append_cmdline:
            cmd += f'{param} '

        # The simulation keeps running when the host powers off. A log message
        # indicates the event when the machine is powering off. We place a
        # breakpoint on that log message, which will terminate Simics due to the
        # use of `-batch-mode`.
        cmd += (
            '-e \'bp.log.break object=board.mb.sb.lpc.bank.acpi_io_regs '
            'substr="Sleep state is unimplemented" type=unimpl\' '
        )

        return cmd + '-e run'


705
class CorundumVerilatorNIC(NICSim):
706

707
    def __init__(self) -> None:
708
709
        super().__init__()
        self.clock_freq = 250  # MHz
710

711
    def resreq_mem(self) -> int:
712
713
714
        # this is a guess
        return 512

715
    def run_cmd(self, env: ExpEnv) -> str:
Jonas Kaufmann's avatar
Jonas Kaufmann committed
716
717
718
719
        return self.basic_run_cmd(
            env, '/corundum/corundum_verilator', str(self.clock_freq)
        )

720
721

class CorundumBMNIC(NICSim):
Jonas Kaufmann's avatar
Jonas Kaufmann committed
722

723
    def run_cmd(self, env: ExpEnv) -> str:
Hejing Li's avatar
Hejing Li committed
724
        return self.basic_run_cmd(env, '/corundum_bm/corundum_bm')
725

Jonas Kaufmann's avatar
Jonas Kaufmann committed
726

727
class I40eNIC(NICSim):
Jonas Kaufmann's avatar
Jonas Kaufmann committed
728

729
    def run_cmd(self, env: ExpEnv) -> str:
Hejing Li's avatar
Hejing Li committed
730
        return self.basic_run_cmd(env, '/i40e_bm/i40e_bm')
731

Jonas Kaufmann's avatar
Jonas Kaufmann committed
732

733
class E1000NIC(NICSim):
734

735
    def __init__(self) -> None:
736
737
        super().__init__()
        self.debug = False
Jonas Kaufmann's avatar
Jonas Kaufmann committed
738

739
    def run_cmd(self, env: ExpEnv) -> str:
740
741
742
743
        cmd = self.basic_run_cmd(env, '/e1000_gem5/e1000_gem5')
        if self.debug:
            cmd = 'env E1000_DEBUG=1 ' + cmd
        return cmd
744

Jonas Kaufmann's avatar
Jonas Kaufmann committed
745

746
747
class MultiSubNIC(NICSim):

748
    def __init__(self, mn: Simulator) -> None:
749
        super().__init__()
750
        self.multinic = mn
751

752
    def full_name(self) -> str:
753
754
        return self.multinic.full_name() + '.' + self.name

755
    def dependencies(self) -> tp.List[Simulator]:
756
757
        return super().dependencies() + [self.multinic]

758
    def start_delay(self) -> int:
759
760
        return 0

Jonas Kaufmann's avatar
Jonas Kaufmann committed
761

Jonas Kaufmann's avatar
Jonas Kaufmann committed
762
class I40eMultiNIC(Simulator):
Jonas Kaufmann's avatar
Jonas Kaufmann committed
763

764
    def __init__(self) -> None:
765
        super().__init__()
766
        self.subnics: tp.List[NICSim] = []
767

768
    def create_subnic(self) -> MultiSubNIC:
769
770
771
772
        sn = MultiSubNIC(self)
        self.subnics.append(sn)
        return sn

773
    def full_name(self) -> str:
774
775
        return 'multinic.' + self.name

776
    def run_cmd(self, env: ExpEnv) -> str:
777
778
779
780
781
782
783
        args = ''
        first = True
        for sn in self.subnics:
            if not first:
                args += ' -- '
            first = False
            args += sn.basic_args(env)
784
        return f'{env.repodir}/sims/nic/i40e_bm/i40e_bm {args}'
785

786
    def sockets_cleanup(self, env: ExpEnv) -> tp.List[str]:
787
788
789
790
791
        ss = []
        for sn in self.subnics:
            ss += sn.sockets_cleanup(env)
        return ss

792
    def sockets_wait(self, env: ExpEnv) -> tp.List[str]:
793
794
795
796
        ss = []
        for sn in self.subnics:
            ss += sn.sockets_wait(env)
        return ss
797
798
799


class WireNet(NetSim):
Jonas Kaufmann's avatar
Jonas Kaufmann committed
800

801
    def run_cmd(self, env: ExpEnv) -> str:
Jialin Li's avatar
Jialin Li committed
802
        connects = self.connect_sockets(env)
803
        assert len(connects) == 2
804
805
        cmd = (
            f'{env.repodir}/sims/net/wire/net_wire {connects[0][1]}'
806
            f' {connects[1][1]} {self.sync_mode} {self.sync_period}'
807
808
            f' {self.eth_latency}'
        )
809
810
811
        if len(env.pcap_file) > 0:
            cmd += ' ' + env.pcap_file
        return cmd
812

Jonas Kaufmann's avatar
Jonas Kaufmann committed
813

814
class SwitchNet(NetSim):
815

816
    def __init__(self) -> None:
817
818
        super().__init__()
        self.sync = True
819
        """Whether to synchronize with attached simulators."""
820

821
    def run_cmd(self, env: ExpEnv) -> str:
822
        cmd = env.repodir + '/sims/net/switch/net_switch'
823
        cmd += f' -S {self.sync_period} -E {self.eth_latency}'
824
825
826
827

        if not self.sync:
            cmd += ' -u'

828
829
        if len(env.pcap_file) > 0:
            cmd += ' -p ' + env.pcap_file
Jonas Kaufmann's avatar
Jonas Kaufmann committed
830
        for (_, n) in self.connect_sockets(env):
831
            cmd += ' -s ' + n
Jonas Kaufmann's avatar
Jonas Kaufmann committed
832
        for (_, n) in self.listen_sockets(env):
833
            cmd += ' -h ' + n
834
        return cmd
835

836
    def sockets_cleanup(self, env: ExpEnv) -> tp.List[str]:
837
        # cleanup here will just have listening eth sockets, switch also creates
Hejing Li's avatar
Hejing Li committed
838
839
840
841
842
843
844
845
846
847
        # shm regions for each with a "-shm" suffix
        cleanup = []
        for s in super().sockets_cleanup(env):
            cleanup.append(s)
            cleanup.append(s + '-shm')
        return cleanup


class MemSwitchNet(NetSim):

848
    def __init__(self) -> None:
Hejing Li's avatar
Hejing Li committed
849
850
851
852
853
        super().__init__()
        self.sync = True
        """ AS_ID,VADDR_START,VADDR_END,MEMNODE_MAC,PHYS_START """
        self.mem_map = []

854
    def run_cmd(self, env: ExpEnv) -> str:
Hejing Li's avatar
Hejing Li committed
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
        cmd = env.repodir + '/sims/mem/memswitch/memswitch'
        cmd += f' -S {self.sync_period} -E {self.eth_latency}'

        if not self.sync:
            cmd += ' -u'

        if len(env.pcap_file) > 0:
            cmd += ' -p ' + env.pcap_file
        for (_, n) in self.connect_sockets(env):
            cmd += ' -s ' + n
        for (_, n) in self.listen_sockets(env):
            cmd += ' -h ' + n
        for m in self.mem_map:
            cmd += ' -m ' + f' {m[0]},{m[1]},{m[2]},'
            cmd += (''.join(reversed(m[3].split(':'))))
            cmd += f',{m[4]}'
        return cmd

873
    def sockets_cleanup(self, env: ExpEnv) -> tp.List[str]:
Hejing Li's avatar
Hejing Li committed
874
        # cleanup here will just have listening eth sockets, switch also creates
875
876
877
878
879
880
881
        # shm regions for each with a "-shm" suffix
        cleanup = []
        for s in super().sockets_cleanup(env):
            cleanup.append(s)
            cleanup.append(s + '-shm')
        return cleanup

Jonas Kaufmann's avatar
Jonas Kaufmann committed
882

883
class TofinoNet(NetSim):
884

885
    def __init__(self) -> None:
886
887
888
        super().__init__()
        self.tofino_log_path = '/tmp/model.ldjson'
        self.sync = True
Jialin Li's avatar
Jialin Li committed
889

890
    def run_cmd(self, env: ExpEnv) -> str:
891
892
893
894
895
        cmd = f'{env.repodir}/sims/net/tofino/tofino'
        cmd += (
            f' -S {self.sync_period} -E {self.eth_latency}'
            f' -t {self.tofino_log_path}'
        )
896
897
        if not self.sync:
            cmd += ' -u'
Jonas Kaufmann's avatar
Jonas Kaufmann committed
898
        for (_, n) in self.connect_sockets(env):
899
            cmd += ' -s ' + n
900
        return cmd
901

Jonas Kaufmann's avatar
Jonas Kaufmann committed
902

903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
class NS3E2ENet(NetSim):

    def __init__(self) -> None:
        super().__init__()
        self.e2e_components: tp.List[E2ETopology] = []

    def resolve_socket_paths(
        self, env: ExpEnv, e2e_sim: E2ESimbricksHost
    ) -> None:
        if e2e_sim.simbricks_host is None:
            print('E2E Simbricks host does not contain a simulator')
            sys.exit(1)
        e2e_sim.unix_socket = env.nic_eth_path(e2e_sim.simbricks_host)

    def run_cmd(self, env):
        for topo in self.e2e_components:
919
920
            if not topo.has_path:
                topo.resolve_paths()
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
            for c in topo.components:
                if isinstance(c, E2ESimbricksHost):
                    self.resolve_socket_paths(env, c)

        params: tp.List[str] = []
        for component in self.e2e_components:
            params.append(component.ns3_config())

        cmd = (
            f'{env.repodir}/sims/external/ns-3'
            f'/simbricks-run.sh e2e-cc-example {" ".join(params)} {self.opt}'
        )
        print(cmd)

        return cmd


938
class NS3DumbbellNet(NetSim):
Jonas Kaufmann's avatar
Jonas Kaufmann committed
939

940
    def run_cmd(self, env: ExpEnv) -> str:
941
        ports = ''
Jonas Kaufmann's avatar
Jonas Kaufmann committed
942
        for (n, s) in self.connect_sockets(env):
943
            if 'server' in n.name:
Marvin Meiers's avatar
Marvin Meiers committed
944
                ports += f'--SimbricksPortLeft={s} '
945
            else:
Marvin Meiers's avatar
Marvin Meiers committed
946
                ports += f'--SimbricksPortRight={s} '
947

948
949
        cmd = (
            f'{env.repodir}/sims/external/ns-3'
Marvin Meiers's avatar
Marvin Meiers committed
950
            f'/simbricks-run.sh simbricks-dumbbell-example {ports} {self.opt}'
951
        )
952
953
954
955
        print(cmd)

        return cmd

Jonas Kaufmann's avatar
Jonas Kaufmann committed
956

957
class NS3BridgeNet(NetSim):
Jonas Kaufmann's avatar
Jonas Kaufmann committed
958

959
    def run_cmd(self, env: ExpEnv) -> str:
960
        ports = ''
Jonas Kaufmann's avatar
Jonas Kaufmann committed
961
        for (_, n) in self.connect_sockets(env):
Marvin Meiers's avatar
Marvin Meiers committed
962
            ports += '--SimbricksPort=' + n + ' '
963

964
965
        cmd = (
            f'{env.repodir}/sims/external/ns-3'
Marvin Meiers's avatar
Marvin Meiers committed
966
            f'/simbricks-run.sh simbricks-bridge-example {ports} {self.opt}'
967
        )
968
969
970
        print(cmd)

        return cmd
971

Jonas Kaufmann's avatar
Jonas Kaufmann committed
972

973
class NS3SequencerNet(NetSim):
Jonas Kaufmann's avatar
Jonas Kaufmann committed
974

975
    def run_cmd(self, env: ExpEnv) -> str:
976
        ports = ''
Jonas Kaufmann's avatar
Jonas Kaufmann committed
977
        for (n, s) in self.connect_sockets(env):
978
            if 'client' in n.name:
979
                ports += '--ClientPort=' + s + ' '
980
            elif 'replica' in n.name:
981
                ports += '--ServerPort=' + s + ' '
982
            elif 'sequencer' in n.name:
983
                ports += '--ServerPort=' + s + ' '
984
            else:
985
                raise KeyError('Wrong NIC type')
986
987
        cmd = (
            f'{env.repodir}/sims/external/ns-3'
Marvin Meiers's avatar
Marvin Meiers committed
988
            f'/simbricks-run.sh sequencer-single-switch-example'
989
990
            f' {ports} {self.opt}'
        )
991
992
993
        return cmd


994
class FEMUDev(PCIDevSim):
Jonas Kaufmann's avatar
Jonas Kaufmann committed
995

996
    def run_cmd(self, env: ExpEnv) -> str:
997
998
999
1000
        cmd = (
            f'{env.repodir}/sims/external/femu/femu-simbricks'
            f' {env.dev_pci_path(self)} {env.dev_shm_path(self)}'
        )
1001
        return cmd
1002
1003
1004
1005


class BasicMemDev(MemDevSim):

1006
    def run_cmd(self, env: ExpEnv) -> str:
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
        cmd = (
            f'{env.repodir}/sims/mem/basicmem/basicmem'
            f' {self.size} {self.addr} {self.as_id}'
            f' {env.dev_mem_path(self)} {env.dev_shm_path(self)}'
            f' {self.sync_mode} {self.start_tick} {self.sync_period}'
            f' {self.mem_latency}'
        )
        return cmd


class MemNIC(MemDevSim):

1019
    def run_cmd(self, env: ExpEnv) -> str:
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
        cmd = (
            f'{env.repodir}/sims/mem/memnic/memnic'
            f' {env.dev_mem_path(self)} {env.nic_eth_path(self)}'
            f' {env.dev_shm_path(self)}'
        )

        if self.mac is not None:
            cmd += ' ' + (''.join(reversed(self.mac.split(':'))))

        cmd += f' {self.sync_mode} {self.start_tick} {self.sync_period}'
        cmd += f' {self.mem_latency} {self.eth_latency}'

        return cmd

1034
    def sockets_cleanup(self, env: ExpEnv) -> tp.List[str]:
1035
1036
        return super().sockets_cleanup(env) + [env.nic_eth_path(self)]

1037
    def sockets_wait(self, env: ExpEnv) -> tp.List[str]:
1038
1039
1040
1041
1042
        return super().sockets_wait(env) + [env.nic_eth_path(self)]


class NetMem(NetMemSim):

1043
    def run_cmd(self, env: ExpEnv) -> str:
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
        cmd = (
            f'{env.repodir}/sims/mem/netmem/netmem'
            f' {self.size} {self.addr} {self.as_id}'
            f' {env.nic_eth_path(self)}'
            f' {env.dev_shm_path(self)}'
        )
        if self.mac is not None:
            cmd += ' ' + (''.join(reversed(self.mac.split(':'))))

        cmd += f' {self.sync_mode} {self.start_tick} {self.sync_period}'
        cmd += f' {self.eth_latency}'

        return cmd