basic support for qemu + curundum-bm + corundum verilator

experiments/pyexps/netperf_sysconf.py

@@ -19,7 +19,8 @@ experiments = []
 sys = system.System()
 
 # create a host instance and a NIC instance then install the NIC on the host
-host0 = system.I40ELinuxHost(sys)
+host0 = system.CorundumLinuxHost(sys)
+# host0 = system.I40ELinuxHost(sys)
 pcie0 = system.PCIeHostInterface(host0)
 cfg_disk0 = system.DistroDiskImage(h=host0, name="base")
 host0.add_disk(cfg_disk0)
@@ -27,7 +28,8 @@ tar_disk0 = system.LinuxConfigDiskImage(h=host0)
 host0.add_disk(tar_disk0)
 
 host0.add_if(pcie0)
-nic0 = system.IntelI40eNIC(sys)
+nic0 = system.CorundumNIC(sys)
+# nic0 = system.IntelI40eNIC(sys)
 nic0.add_ipv4("10.0.0.1")
 pcichannel0 = system.PCIeChannel(pcie0, nic0._pci_if)
 
@@ -59,7 +61,9 @@ ethchannel1 = system.EthChannel(switch.eth_ifs[1], nic1._eth_if)
 # configure the software to run on the host
 # host0.add_app(system.NetperfClient(host0, nic1._ip))
 # host1.add_app(system.NetperfServer(host1))
-host0.add_app(system.PingClient(host0, nic1._ip))
+ping_client_app = system.PingClient(host0, nic1._ip)
+ping_client_app.wait = True
+host0.add_app(ping_client_app)
 host1.add_app(system.Sleep(host1, infinite=True))
 
 """
@@ -102,27 +106,30 @@ for host_type in host_types:
             else:
                 raise NameError(net_type)
 
-            host_inst0 = host_sim(simulation)
+            host_inst0 = sim.QemuSim(simulation)
             host_inst0.add(host0)
-            host_inst0.wait_terminate = True
-            host_inst0.cpu_type = 'X86KvmCPU'
+            # host_inst0.wait_terminate = True
+            # host_inst0.cpu_type = 'X86KvmCPU'
 
-            host_inst1 = host_sim(simulation)
+            # host_inst1 = sim.Gem5Sim(simulation)
+            host_inst1 = sim.QemuSim(simulation)
             host_inst1.add(host1)
-            host_inst1.cpu_type = 'X86KvmCPU'
+            # host_inst1.cpu_type = 'X86KvmCPU'
 
-            nic_inst0 = nic_sim(simulation)
+            # nic_inst0 = sim.I40eNicSim(simulation=simulation)
+            # nic_inst0 = sim.CorundumBMNICSim(simulation)
+            nic_inst0 = sim.CorundumVerilatorNICSim(simulation)
             nic_inst0.add(nic0)
 
-            nic_inst1 = nic_sim(simulation)
+            nic_inst1 = sim.I40eNicSim(simulation=simulation)
             nic_inst1.add(nic1)
 
-            net_inst = net_sim(simulation)
+            net_inst = sim.SwitchNet(simulation)
             net_inst.add(switch)
 
-            # sim_helpers.enable_sync_simulation(
-            #     simulation=simulation, amount=500, ratio=sim.Time.Nanoseconds
-            # )
+            sim_helpers.enable_sync_simulation(
+                simulation=simulation, amount=500, ratio=sim.Time.Nanoseconds
+            )
 
             print(simulation.name + "   all simulators:")
             sims = simulation.all_simulators()

experiments/simbricks/orchestration/instantiation/base.py

@@ -347,7 +347,8 @@ class Instantiation(util_base.IdObj):
         await self.executor.await_files(wait_socks, verbose=True)
 
     # TODO: add more methods constructing paths as required by methods in simulators or image handling classes
-
+    
+    # TODO: fix paths to support mutliple exeriment runs etc.
     def wrkdir(self) -> str:
         return pathlib.Path(self._env._workdir).resolve()
 
@@ -414,7 +415,6 @@ class Instantiation(util_base.IdObj):
             )
 
         joined = pathlib.Path(base).joinpath(relative_path).resolve()
-        print(f"joined={joined} from base={base}, relative_path={relative_path}")
         if enforce_existence and not joined.exists():
             raise Exception(f"couldn't join {base} and {relative_path}")
         return joined.as_posix()
@@ -442,7 +442,7 @@ class Instantiation(util_base.IdObj):
         return path
 
     def cfgtar_path(self, sim: sim_base.Simulator) -> str:
-        return f"{self._env._workdir}/cfg.{sim.name}.tar"
+        return f"{self.wrkdir()}/cfg.{sim.name}.tar"
 
     def join_tmp_base(self, relative_path: str) -> str:
         return self._join_paths(

experiments/simbricks/orchestration/simulation/host.py

@@ -24,11 +24,9 @@ from __future__ import annotations
 
 import math
 import asyncio
-import typing as tp
 import simbricks.orchestration.simulation.base as sim_base
 import simbricks.orchestration.system as system
 from simbricks.orchestration.instantiation import base as inst_base
-from simbricks.orchestration.experiment.experiment_environment_new import ExpEnv
 from simbricks.orchestration.system import host as sys_host
 from simbricks.orchestration.system import pcie as sys_pcie
 from simbricks.orchestration.system import mem as sys_mem
@@ -90,7 +88,7 @@ class Gem5Sim(HostSim):
 
     def checkpoint_commands(self) -> list[str]:
         return ["m5 checkpoint"]
-    
+
     def cleanup_commands(self) -> list[str]:
         return ["m5 exit"]
 
@@ -192,102 +190,112 @@ class Gem5Sim(HostSim):
 
         cmd += " ".join(self.extra_config_args)
 
-        print(f"GEM5 COMMAND!!! ===== {cmd}")
         return cmd
 
 
 class QemuSim(HostSim):
 
-    def __init__(self, e: sim_base.Simulation):
-        super().__init__(e)
+    def __init__(self, simulation: sim_base.Simulation) -> None:
+        super().__init__(
+            simulation=simulation,
+            executable="sims/external/qemu/build/x86_64-softmmu/qemu-system-x86_64",
+        )
+        self.name = f"QemuSim-{self._id}"
+        self._disks: list[tuple[str, str]] = [] # [(path, format)]
 
     def resreq_cores(self) -> int:
-        if self.sync:
-            return 1
-        else:
-            # change it to sum of all hosts
-            return self.hosts[0].cores + 1
+        return 1
 
     def resreq_mem(self) -> int:
         return 8192
 
     def supported_image_formats(self) -> list[str]:
-        return ["raw", "qcow2"]
+        return ["raw", "qcow"]
 
     async def prepare(self, inst: inst_base.Instantiation) -> None:
         await super().prepare(inst=inst)
 
-        prep_cmds = []
-        full_sys_hosts = tp.cast(
-            list[system.FullSystemHost],
-            self.filter_components_by_type(ty=system.FullSystemHost),
-        )
+        full_sys_hosts = self.filter_components_by_type(ty=sys_host.FullSystemHost)
+        if len(full_sys_hosts) != 1:
+            raise Exception("QEMU only supports simulating 1 FullSystemHost")
 
-        prep_cmds = []
-        for fsh in full_sys_hosts:
-            disks = tp.cast(list[system.DiskImage], fsh.disks)
-            for disk in disks:
-                prep_cmds.append(
-                    f"{inst.qemu_img_path()} create -f qcow2 -o "
-                    f'backing_file="{disk.path(inst=inst, format="qcow2")}" '
-                    f'{inst.hdcopy_path(img=disk, format="qcow2")}'
-                )
+        d = []
+        for disk in full_sys_hosts[0].disks:
+            format = (
+                "qcow2" if not isinstance(disk, sys_host.LinuxConfigDiskImage) else "raw"
+            )
+            copy_path = await disk.make_qcow_copy(
+                inst=inst,
+                format=format,
+            )
+            assert copy_path is not None
+            d.append((copy_path, format))
+        self._disks = d
 
-        task = asyncio.create_task(
-            inst.executor.run_cmdlist(label="prepare", cmds=prep_cmds, verbose=True)
+    def checkpoint_commands(self) -> list[str]:
+        return []
+
+    def cleanup_commands(self) -> list[str]:
+        return []
+
+    def run_cmd(self, inst: inst_base.Instantiation) -> str:
+
+        latency, period, sync = sim_base.Simulator.get_unique_latency_period_sync(
+            channels=self.get_channels()
         )
-        await task
 
-    def run_cmd(self, env: ExpEnv) -> str:
-        accel = ",accel=kvm:tcg" if not self.sync else ""
-        if self.hosts[0].disks[0].kcmd_append:
-            kcmd_append = " " + self.hosts[0].kcmd_append
-        else:
-            kcmd_append = ""
+        accel = ",accel=kvm:tcg" if not sync else ""
+        # if self.node_config.kcmd_append: # TODO: FIXME
+        #     kcmd_append = " " + self.node_config.kcmd_append
+        # else:
+        #     kcmd_append = ""
 
         cmd = (
-            f"{env.qemu_path} -machine q35{accel} -serial mon:stdio "
+            f"{inst.join_repo_base(relative_path=self._executable)} -machine q35{accel} -serial mon:stdio "
             "-cpu Skylake-Server -display none -nic none "
-            f"-kernel {env.qemu_kernel_path} "
-            f"-drive file={env.hdcopy_path(self)},if=ide,index=0,media=disk "
-            f"-drive file={env.cfgtar_path(self)},if=ide,index=1,media=disk,"
-            "driver=raw "
+            f"-kernel {inst.join_repo_base('images/bzImage')} "
+        )
+
+        full_sys_hosts = self.filter_components_by_type(ty=sys_host.FullSystemHost)
+        if len(full_sys_hosts) != 1:
+            raise Exception("QEMU only supports simulating 1 FullSystemHost")
+
+        for index, disk in enumerate(self._disks):
+            cmd += f"-drive file={disk[0]},if=ide,index={index},media=disk,driver={disk[1]} "
+        cmd += (
             '-append "earlyprintk=ttyS0 console=ttyS0 root=/dev/sda1 '
-            f'init=/home/ubuntu/guestinit.sh rw{kcmd_append}" '
-            f"-m {self.hosts[0].memory} -smp {self.hosts[0].cores} "
+            # f'init=/home/ubuntu/guestinit.sh rw{kcmd_append}" ' # TODO: FIXME
+            f'init=/home/ubuntu/guestinit.sh rw" '
+            f"-m {full_sys_hosts[0].memory} -smp {full_sys_hosts[0].cores} "
         )
 
-        if self.sync:
-            unit = self.hosts[0].cpu_freq[-3:]
+        if sync:
+            unit = full_sys_hosts[0].cpu_freq[-3:]
             if unit.lower() == "ghz":
                 base = 0
             elif unit.lower() == "mhz":
                 base = 3
             else:
                 raise ValueError("cpu frequency specified in unsupported unit")
-            num = float(self.hosts[0].cpu_freq[:-3])
+            num = float(full_sys_hosts[0].cpu_freq[:-3])
             shift = base - int(math.ceil(math.log(num, 2)))
 
             cmd += f" -icount shift={shift},sleep=off "
 
-        for dev in self.hosts[0].ifs:
-            if dev == dev.channel.a:
-                peer_if = dev.channel.b
-            else:
-                peer_if = dev.channel.a
-
-            peer_sim = self.experiment.find_sim(peer_if)
-            chn_sim = self.experiment.find_sim(dev.channel)
-
-            cmd += f"-device simbricks-pci,socket={env.dev_pci_path(peer_sim)}"
-            if self.sync:
+        fsh_interfaces = full_sys_hosts[0].interfaces()
+        pci_interfaces = system.Interface.filter_by_type(
+            interfaces=fsh_interfaces, ty=sys_pcie.PCIeHostInterface
+        )
+        for inf in pci_interfaces:
+            socket = self._get_socket(inst=inst, interface=inf)
+            if socket is None:
+                continue
+            assert socket._type is inst_base.SockType.CONNECT
+            cmd += f"-device simbricks-pci,socket={socket._path}"
+            if sync:
                 cmd += ",sync=on"
-                cmd += f",pci-latency={dev.channel.latency}"
-                cmd += f",sync-period={chn_sim.sync_period}"
-                # if self.sync_drift is not None:
-                #     cmd += f',sync-drift={self.sync_drift}'
-                # if self.sync_offset is not None:
-                #     cmd += f',sync-offset={self.sync_offset}'
+                cmd += f",pci-latency={latency}"
+                cmd += f",sync-period={period}"
             else:
                 cmd += ",sync=off"
             cmd += " "

experiments/simbricks/orchestration/simulation/pcidev.py

@@ -57,6 +57,7 @@ class NICSim(PCIDevSim):
         super().__init__(simulation=simulation, executable=executable, name=name)
 
     def add(self, nic: sys_nic.SimplePCIeNIC):
+        assert len(self._components) < 1
         super().add(nic)
 
     def run_cmd(self, inst: inst_base.Instantiation) -> str:
@@ -90,7 +91,6 @@ class NICSim(PCIDevSim):
         if self.extra_args is not None:
             cmd += " " + self.extra_args
 
-        print(f"NIC RUN COMMAND!!! ===== {cmd}")
         return cmd
 
 
@@ -116,7 +116,8 @@ class CorundumBMNICSim(NICSim):
         self.name=f"CorundumBMNICSim-{self._id}"
 
     def run_cmd(self, inst: inst_base.Instantiation) -> str:
-        return super().run_cmd(inst=inst)
+        cmd = super().run_cmd(inst=inst)
+        return cmd
 
 
 class CorundumVerilatorNICSim(NICSim):
@@ -135,5 +136,5 @@ class CorundumVerilatorNICSim(NICSim):
 
     def run_cmd(self, inst: inst_base.Instantiation) -> str:
         cmd = super().run_cmd(inst=inst)
-        cmd += str(self.clock_freq)
+        cmd += f" {self.clock_freq}"
         return cmd

experiments/simbricks/orchestration/system/host/base.py

@@ -25,7 +25,6 @@ from __future__ import annotations
 import typing as tp
 import io
 import asyncio
-from os import path
 import simbricks.orchestration.instantiation.base as instantiation
 from simbricks.orchestration.system import base as base
 from simbricks.orchestration.system import eth as eth
@@ -107,7 +106,7 @@ class BaseLinuxHost(FullSystemHost):
         """Commands to run to cleanup node."""
         cmds = self._concat_app_cmds(
             inst, app.BaseLinuxApplication.cleanup_cmds.__name__
-        ) 
+        )
         sim = inst.find_sim_by_spec(spec=self)
         cleanup = sim.cleanup_commands()
         cmds += cleanup
@@ -244,5 +243,9 @@ class CorundumLinuxHost(LinuxHost):
         self.drivers.append("/tmp/guest/mqnic.ko")
 
     def config_files(self, inst: instantiation.Instantiation) -> tp.Dict[str, tp.IO]:
-        m = {"mqnic.ko": open("../images/mqnic/mqnic.ko", "rb")}
-        return {**m, **super().config_files()}
+        m = {
+            "mqnic.ko": open(
+                f"{inst.join_repo_base(relative_path='images/mqnic/mqnic.ko')}", "rb"
+            )
+        }
+        return {**m, **super().config_files(inst=inst)}

experiments/simbricks/orchestration/system/host/disk_images.py

@@ -38,6 +38,7 @@ class DiskImage(utils_base.IdObj):
     def __init__(self, h: sys_host.Host) -> None:
         super().__init__()
         self.host: sys_host.Host = h
+        self._qemu_img_exec: str = "sims/external/qemu/build/qemu-img"
 
     @abc.abstractmethod
     def available_formats(self) -> list[str]:
@@ -46,7 +47,20 @@ class DiskImage(utils_base.IdObj):
     @abc.abstractmethod
     def path(self, inst: inst_base.Instantiation, format: str) -> str:
         raise Exception("must be overwritten")
-    
+
+    async def make_qcow_copy(self, inst: inst_base.Instantiation, format: str) -> str:
+        disk_path = pathlib.Path(self.path(inst=inst, format=format))
+        copy_path = inst.join_tmp_base(relative_path=f"hdcopy.{self._id}")
+        prep_cmds = [
+            (
+                f"{inst.join_repo_base(relative_path=self._qemu_img_exec)} create -f qcow2 -o "
+                f'backing_file="{disk_path}" '
+                f"{copy_path}"
+            )
+        ]
+        await inst.executor.run_cmdlist(label="prepare", cmds=prep_cmds, verbose=True)
+        return copy_path
+
     @staticmethod
     def assert_is_file(path: str) -> str:
         if not pathlib.Path(path).is_file():
@@ -66,7 +80,7 @@ class DiskImage(utils_base.IdObj):
                 break
 
         if format is None:
-            raise Exception('No supported image format found')
+            raise Exception("No supported image format found")
 
         await self._prepare_format(inst, format)
 
@@ -81,7 +95,7 @@ class ExternalDiskImage(DiskImage):
     def available_formats(self) -> list[str]:
         return self.formats
 
-    def path(self, inst: inst_base.Instantiation, format: str)  -> str:
+    def path(self, inst: inst_base.Instantiation, format: str) -> str:
         DiskImage.assert_is_file(self._path)
         return self._path
 
@@ -100,13 +114,14 @@ class DistroDiskImage(DiskImage):
         path = inst.hd_path(self.name)
         if format == "raw":
             path += ".raw"
-        elif format == "qcow":
+        elif format == "qcow2":
             pass
         else:
             raise RuntimeError("Unsupported disk format")
         DiskImage.assert_is_file(path)
         return path
 
+
 # Abstract base class for dynamically generated images
 class DynamicDiskImage(DiskImage):
     def __init__(self, h: sys_host.FullSystemHost) -> None:
@@ -119,6 +134,7 @@ class DynamicDiskImage(DiskImage):
     async def _prepare_format(self, inst: inst_base.Instantiation, format: str) -> None:
         pass
 
+
 # Builds the Tar with the commands to run etc.
 class LinuxConfigDiskImage(DynamicDiskImage):
     def __init__(self, h: sys_host.LinuxHost) -> None:
@@ -128,12 +144,15 @@ class LinuxConfigDiskImage(DynamicDiskImage):
     def available_formats(self) -> list[str]:
         return ["raw"]
 
+    async def make_qcow_copy(self, inst: inst_base.Instantiation, format: str) -> str:
+        return self.path(inst=inst, format=format)
+
     async def _prepare_format(self, inst: inst_base.Instantiation, format: str) -> None:
         path = self.path(inst, format)
         print(path)
-        with tarfile.open(path, 'w:') as tar:
+        with tarfile.open(path, "w:") as tar:
             # add main run script
-            cfg_i = tarfile.TarInfo('guest/run.sh')
+            cfg_i = tarfile.TarInfo("guest/run.sh")
             cfg_i.mode = 0o777
             cfg_f = self.host.strfile(self.host.config_str(inst))
             cfg_f.seek(0, io.SEEK_END)
@@ -143,8 +162,8 @@ class LinuxConfigDiskImage(DynamicDiskImage):
             cfg_f.close()
 
             # add additional config files
-            for (n, f) in self.host.config_files(inst).items():
-                f_i = tarfile.TarInfo('guest/' + n)
+            for n, f in self.host.config_files(inst).items():
+                f_i = tarfile.TarInfo("guest/" + n)
                 f_i.mode = 0o777
                 f.seek(0, io.SEEK_END)
                 f_i.size = f.tell()
@@ -153,7 +172,6 @@ class LinuxConfigDiskImage(DynamicDiskImage):
                 f.close()
 
 
-
 # This is an additional example: building disk images directly from python
 # Could of course also have a version that generates the packer config from
 # python