experiments: add memory devices support (basicmem, memnic, netmem)

Includes a new simulator type, and integration for host simulators to add
memory devices. Currently only implemented for gem5.
Co-authored-by: Hejing Li <hajeongee@gmail.com>

experiments/simbricks/orchestration/experiment/experiment_environment.py

@@ -62,6 +62,9 @@ class ExpEnv(object):
     def dev_pci_path(self, sim):
         return f'{self.workdir}/dev.pci.{sim.name}'
 
+    def dev_mem_path(self, sim):
+        return f'{self.workdir}/dev.mem.{sim.name}'
+
     def nic_eth_path(self, sim):
         return f'{self.workdir}/nic.eth.{sim.name}'
 

experiments/simbricks/orchestration/experiments.py

@@ -23,6 +23,7 @@
 import itertools
 import typing as tp
 
+from simbricks.orchestration import simulators
 from simbricks.orchestration.proxy import NetProxyConnecter, NetProxyListener
 from simbricks.orchestration.simulators import (
     HostSim, I40eMultiNIC, NetSim, NICSim, PCIDevSim, Simulator
@@ -57,6 +58,10 @@ class Experiment(object):
         """The host simulators to run."""
         self.pcidevs: tp.List[PCIDevSim] = []
         """The PCIe device simulators to run."""
+        self.memdevs: tp.List[simulators.MemDevSim] = []
+        """The memory device simulators to run."""
+        self.netmems: tp.List[simulators.NetMemSim] = []
+        """The network memory simulators to run."""
         self.networks: tp.List[NetSim] = []
         """The network simulators to run."""
         self.metadata = {}
@@ -83,6 +88,18 @@ class Experiment(object):
                 raise ValueError('Duplicate pcidev name')
         self.pcidevs.append(sim)
 
+    def add_memdev(self, sim: simulators.MemDevSim):
+        for d in self.memdevs:
+            if d.name == sim.name:
+                raise ValueError('Duplicate memdev name')
+        self.memdevs.append(sim)
+
+    def add_netmem(self, sim: simulators.NetMemSim):
+        for d in self.netmems:
+            if d.name == sim.name:
+                raise ValueError('Duplicate netmems name')
+        self.netmems.append(sim)
+
     def add_network(self, sim: NetSim):
         """Add a network simulator to the experiment."""
         for n in self.networks:
@@ -92,7 +109,9 @@ class Experiment(object):
 
     def all_simulators(self):
         """Returns all simulators defined to run in this experiment."""
-        return itertools.chain(self.hosts, self.pcidevs, self.networks)
+        return itertools.chain(
+            self.hosts, self.pcidevs, self.memdevs, self.netmems, self.networks
+        )
 
     def resreq_mem(self):
         """Memory required to run all simulators in this experiment."""

experiments/simbricks/orchestration/simulators.py

@@ -204,6 +204,57 @@ class NetSim(Simulator):
         return [s for (_, s) in self.listen_sockets(env)]
 
 
+# FIXME: Class hierarchy is broken here as an ugly hack
+class MemDevSim(NICSim):
+    """Base class for memory device simulators."""
+
+    def __init__(self):
+        super().__init__()
+
+        self.name = ''
+        self.sync_mode = 0
+        self.start_tick = 0
+        self.sync_period = 500
+        self.mem_latency = 500
+        self.addr = 0xe000000000000000
+        self.size = 1024 * 1024 * 1024  # 1GB
+        self.as_id = 0
+
+    def full_name(self):
+        return 'mem.' + self.name
+
+    def sockets_cleanup(self, env):
+        return [env.dev_mem_path(self), env.dev_shm_path(self)]
+
+    def sockets_wait(self, env):
+        return [env.dev_mem_path(self)]
+
+
+class NetMemSim(NICSim):
+    """Base class for netork memory simulators."""
+
+    def __init__(self):
+        super().__init__()
+
+        self.name = ''
+        self.sync_mode = 0
+        self.start_tick = 0
+        self.sync_period = 500
+        self.eth_latency = 500
+        self.addr = 0xe000000000000000
+        self.size = 1024 * 1024 * 1024  # 1GB
+        self.as_id = 0
+
+    def full_name(self):
+        return 'netmem.' + self.name
+
+    def sockets_cleanup(self, env):
+        return [env.nic_eth_path(self), env.dev_shm_path(self)]
+
+    def sockets_wait(self, env):
+        return [env.nic_eth_path(self)]
+
+
 class HostSim(Simulator):
     """Base class for host simulators."""
 
@@ -223,9 +274,11 @@ class HostSim(Simulator):
         self.sync_mode = 0
         self.sync_period = 500
         self.pci_latency = 500
+        self.mem_latency = 500
 
         self.pcidevs: tp.List[PCIDevSim] = []
         self.net_directs: tp.List[NetSim] = []
+        self.memdevs: tp.List[MemDevSim] = []
 
     @property
     def nics(self):
@@ -243,6 +296,10 @@ class HostSim(Simulator):
         dev.name = self.name + '.' + dev.name
         self.pcidevs.append(dev)
 
+    def add_memdev(self, dev: MemDevSim):
+        dev.name = self.name + '.' + dev.name
+        self.memdevs.append(dev)
+
     def add_netdirect(self, net: NetSim):
         """Add a direct connection to a network to this host."""
         net.hosts_direct.append(self)
@@ -254,6 +311,8 @@ class HostSim(Simulator):
             deps.append(dev)
             if isinstance(dev, NICSim):
                 deps.append(dev.network)
+        for dev in self.memdevs:
+            deps.append(dev)
         return deps
 
     def wait_terminate(self):
@@ -329,6 +388,8 @@ class QemuHost(HostSim):
 
         # qemu does not currently support net direct ports
         assert len(self.net_directs) == 0
+        # qemu does not currently support mem device ports
+        assert len(self.memdevs) == 0
         return cmd
 
 
@@ -395,6 +456,17 @@ class Gem5Host(HostSim):
                 cmd += ':sync'
             cmd += ' '
 
+        for dev in self.memdevs:
+            cmd += (
+                f'--simbricks-mem={dev.size}@{dev.addr}@{dev.as_id}@'
+                f'connect:{env.dev_mem_path(dev)}'
+                f':latency={self.mem_latency}ns'
+                f':sync_interval={self.sync_period}ns'
+            )
+            if cpu_type == 'TimingSimpleCPU':
+                cmd += ':sync'
+            cmd += ' '
+
         for net in self.net_directs:
             cmd += (
                 '--simbricks-eth-e1000=listen'
@@ -635,3 +707,58 @@ class FEMUDev(PCIDevSim):
             f' {env.dev_pci_path(self)} {env.dev_shm_path(self)}'
         )
         return cmd
+
+
+class BasicMemDev(MemDevSim):
+
+    def run_cmd(self, env):
+        cmd = (
+            f'{env.repodir}/sims/mem/basicmem/basicmem'
+            f' {self.size} {self.addr} {self.as_id}'
+            f' {env.dev_mem_path(self)} {env.dev_shm_path(self)}'
+            f' {self.sync_mode} {self.start_tick} {self.sync_period}'
+            f' {self.mem_latency}'
+        )
+        return cmd
+
+
+class MemNIC(MemDevSim):
+
+    def run_cmd(self, env):
+        cmd = (
+            f'{env.repodir}/sims/mem/memnic/memnic'
+            f' {env.dev_mem_path(self)} {env.nic_eth_path(self)}'
+            f' {env.dev_shm_path(self)}'
+        )
+
+        if self.mac is not None:
+            cmd += ' ' + (''.join(reversed(self.mac.split(':'))))
+
+        cmd += f' {self.sync_mode} {self.start_tick} {self.sync_period}'
+        cmd += f' {self.mem_latency} {self.eth_latency}'
+
+        return cmd
+
+    def sockets_cleanup(self, env):
+        return super().sockets_cleanup(env) + [env.nic_eth_path(self)]
+
+    def sockets_wait(self, env):
+        return super().sockets_wait(env) + [env.nic_eth_path(self)]
+
+
+class NetMem(NetMemSim):
+
+    def run_cmd(self, env):
+        cmd = (
+            f'{env.repodir}/sims/mem/netmem/netmem'
+            f' {self.size} {self.addr} {self.as_id}'
+            f' {env.nic_eth_path(self)}'
+            f' {env.dev_shm_path(self)}'
+        )
+        if self.mac is not None:
+            cmd += ' ' + (''.join(reversed(self.mac.split(':'))))
+
+        cmd += f' {self.sync_mode} {self.start_tick} {self.sync_period}'
+        cmd += f' {self.eth_latency}'
+
+        return cmd