Merge branch 'master' into nopaxos

.gitignore

@@ -2,12 +2,15 @@
 *.a
 *.ko
 .*.cmd
+*.pyc
+__pycache__/
 _vimrc_local.vim
 dummy_nic/dummy_nic
 corundum/obj_dir
 corundum/corundum_verilator
 net_wire/net_wire
 net_tap/net_tap
+net_switch/net_switch
 i40e_bm/i40e_bm
 images/kernel/kheaders/
 images/mqnic/Module.symvers

Makefile

@@ -5,7 +5,8 @@ all: \
 	corundum_bm/corundum_bm \
 	i40e_bm/i40e_bm \
 	net_tap/net_tap \
-	net_wire/net_wire
+	net_wire/net_wire \
+	net_switch/net_switch
 
 clean:
 	$(MAKE) -C corundum/ clean
@@ -14,6 +15,7 @@ clean:
 	$(MAKE) -C dummy_nic/ clean
 	$(MAKE) -C net_tap/ clean
 	$(MAKE) -C net_wire/ clean
+	$(MAKE) -C net_switch/ clean
 	$(MAKE) -C nicsim_common/ clean
 	$(MAKE) -C netsim_common/ clean
 	$(MAKE) -C libnicbm/ clean
@@ -49,6 +51,9 @@ net_tap/net_tap: netsim_common/libnetsim_common.a
 net_wire/net_wire: netsim_common/libnetsim_common.a
 	$(MAKE) -C net_wire/
 
+net_switch/net_switch: netsim_common/libnetsim_common.a
+	$(MAKE) -C net_switch/
+
 nicsim_common/libnicsim_common.a:
 	$(MAKE) -C nicsim_common/
 

experiments/Makefile

@@ -62,6 +62,10 @@ GUESTS := \
     qemu-i40e-rpc-client \
     qemu-tas-server \
     qemu-tas-client \
+    qemu-echo-client \
+    qemu-echo-server-0 \
+    qemu-echo-server-1 \
+    qemu-echo-server-2 \
 
 # experiments we want for the paper
 EXPERIMENTS := \
@@ -97,7 +101,7 @@ EXP_CP := \
 
 #experiments not using check points
 EXP_NCP := \
-    qemu-ns3-nopaxos-endhost-sequencer \
+    qemu-corundum-bm-echo-switch \
     qemu-corundum-bm-pair \
     qemu-corundum-bm-switched-1 \
     qemu-i40e-bm-mtcp \
@@ -106,6 +110,7 @@ EXP_NCP := \
     qemu-i40e-bm-tas \
     qemu-i40e-bm-cc \
     qemu-ns3-nopaxos \
+    qemu-ns3-nopaxos-endhost-sequencer \
 
 # all experiments in repo
 ALL_EXPERIMENTS := \

experiments/common-functions.sh

@@ -182,6 +182,27 @@ run_wire() {
     return $pid
 }
 
+# Args:
+#   - Instance name
+#   - sim instance 1
+#   - sim instance 2
+#   - [sim instance 3, ...]
+run_switch() {
+    echo Starting switch $1
+
+    args=
+    for iface in ${@:2}
+    do
+        args="$args -s $WORKDIR/eth.$iface"
+    done
+
+    $EHSIM_BASE/net_switch/net_switch \
+        $args &>$OUTDIR/switch.$1.log &
+    pid=$!
+    ALL_PIDS="$ALL_PIDS $pid"
+    return $pid
+}
+
 # Args:
 #   - Instance name
 #   - Port names

experiments/experiments.py

+import modes.experiments as exp
+import modes.simulators as sim
+import modes.nodeconfig as node
+
+e = exp.Experiment('qemu-i40e-pair')
+net = sim.SwitchNet()
+e.add_network(net)
+
+nic_a = sim.I40eNIC()
+nic_a.set_network(net)
+e.add_nic(nic_a)
+
+host_a = sim.QemuHost()
+host_a.name = 'server'
+host_a.node_config = node.I40eLinuxNode()
+host_a.node_config.ip = '10.0.0.1'
+host_a.node_config.app = node.IperfTCPServer()
+host_a.add_nic(nic_a)
+e.add_host(host_a)
+
+for i in range (0, 2):
+    nic_b = sim.I40eNIC()
+    nic_b.set_network(net)
+    e.add_nic(nic_b)
+
+    host_b = sim.QemuHost()
+    host_b.name = 'client.%d' % i
+    host_b.wait = True
+    host_b.node_config = node.I40eLinuxNode()
+    host_b.node_config.ip = '10.0.0.%d' % (2 + i)
+    host_b.node_config.app = node.IperfTCPClient()
+    host_b.add_nic(nic_b)
+    e.add_host(host_b)
+
+env = exp.ExpEnv('..', './work')
+out = exp.run_exp_local(e, env)
+print(out.dumps())

experiments/experiments/qemu-corundum-bm-echo-switch.sh

+#!/bin/bash
+
+source common-functions.sh
+
+init_out qemu-corundum-bm-echo-switch $1
+run_corundum_bm a
+run_corundum_bm b
+run_corundum_bm c
+run_corundum_bm d
+sleep 2
+run_switch sw a b c d
+run_qemu a a build/qemu-echo-server-0.tar
+run_qemu b b build/qemu-echo-server-1.tar
+run_qemu c c build/qemu-echo-server-2.tar
+run_qemu d d build/qemu-echo-client.tar
+client_pid=$!
+wait $client_pid
+cleanup

experiments/guests/qemu-echo-client/run.sh

+#!/bin/bash
+insmod mqnic.ko
+ip link set dev eth0 up
+ip addr add 10.100.0.1/24 dev eth0
+sleep 2
+ping -c 5 10.100.0.2
+ping -c 5 10.100.0.3
+ping -c 5 10.100.0.4
+poweroff -f

experiments/guests/qemu-echo-server-0/run.sh

+#!/bin/bash
+insmod mqnic.ko
+ip link set dev eth0 up
+ip addr add 10.100.0.2/24 dev eth0
+sleep infinity
+poweroff -f

experiments/guests/qemu-echo-server-1/run.sh

+#!/bin/bash
+insmod mqnic.ko
+ip link set dev eth0 up
+ip addr add 10.100.0.3/24 dev eth0
+sleep infinity
+poweroff -f

experiments/guests/qemu-echo-server-2/run.sh

+#!/bin/bash
+insmod mqnic.ko
+ip link set dev eth0 up
+ip addr add 10.100.0.4/24 dev eth0
+sleep infinity
+poweroff -f

experiments/modes/exectools.py

+import asyncio
+import shlex
+import os
+import signal
+
+class HostConfig(object):
+    def __init__(self, name, ip, mac, sudopwd, other={}):
+        self.name = name
+        self.ip = ip
+        self.used_ip = ip
+        self.mac = mac
+        self.sudo_pwd = sudopwd
+        self.other = other.copy()
+
+class Component(object):
+    def __init__(self, cmd_parts, with_stdin=False):
+        self.is_ready = False
+        self.stdout = []
+        self.stdout_buf = bytearray()
+        self.stderr = []
+        self.stderr_buf = bytearray()
+        self.cmd_parts = cmd_parts
+        #print(cmd_parts)
+        self.with_stdin = with_stdin
+
+    def _parse_buf(self, buf, data):
+        if data is not None:
+            buf.extend(data)
+        lines = []
+        start = 0
+        for i in range(0, len(buf)):
+            if buf[i] == ord('\n'):
+                l = buf[start:i].decode('utf-8')
+                lines.append(l)
+                start = i + 1
+        del buf[0:start]
+
+        if len(data) == 0 and len(buf) > 0:
+            lines.append(buf.decode('utf-8'))
+        return lines
+
+    async def _consume_out(self, data):
+        eof = len(data) == 0
+        ls = self._parse_buf(self.stdout_buf, data)
+        if len(ls) > 0 or eof:
+            await self.process_out(ls, eof=eof)
+            self.stdout = self.stdout + ls
+
+    async def _consume_err(self, data):
+        eof = len(data) == 0
+        ls = self._parse_buf(self.stderr_buf, data)
+        if len(ls) > 0 or eof:
+            await self.process_err(ls, eof=eof)
+            self.stderr = self.stderr + ls
+
+    async def _read_stream(self, stream, fn):
+        while True:
+            bs = await stream.readline()
+            if bs:
+                await fn(bs)
+            else:
+                await fn(bs)
+                return
+
+    async def _waiter(self):
+        out_handlers = asyncio.ensure_future(asyncio.wait([
+            self._read_stream(self.proc.stdout, self._consume_out),
+            self._read_stream(self.proc.stderr, self._consume_err)]))
+        rc = await self.proc.wait()
+        await out_handlers
+        await self.terminated(rc)
+        return rc
+
+    async def send_input(self, bs, eof=False):
+        self.proc.stdin.write(bs)
+        if eof:
+            self.proc.stdin.close()
+
+    async def start(self):
+        if self.with_stdin:
+            stdin = asyncio.subprocess.PIPE
+        else:
+            stdin = None
+
+        self.proc = await asyncio.create_subprocess_exec(*self.cmd_parts,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+                stdin=stdin,
+                )
+        self.terminate_future = asyncio.ensure_future(self._waiter())
+        await self.started()
+
+    async def wait(self):
+        await self.terminate_future
+
+    async def interrupt(self):
+        if self.terminate_future.done():
+            return
+        self.proc.send_signal(signal.SIGINT)
+
+    async def terminate(self):
+        if self.terminate_future.done():
+            return
+        self.proc.terminate()
+
+    async def kill(self):
+        self.proc.terminate()
+
+    async def int_term_kill(self, delay=5):
+        await self.interrupt()
+        _,pending = await asyncio.wait([self.terminate_future], timeout=delay)
+        if len(pending) != 0:
+            print('terminating')
+            await self.terminate()
+            _,pending = await asyncio.wait([self.terminate_future],
+                    timeout=delay)
+            if len(pending) != 0:
+                print('killing')
+                await self.kill()
+
+    async def started(self):
+        pass
+
+    async def terminated(self, rc):
+        pass
+
+    async def process_out(self, lines, eof):
+        pass
+
+    async def process_err(self, lines, eof):
+        pass
+
+class RemoteComp(Component):
+    def __init__(self, host, cmd_parts, cwd=None, **kwargs):
+        if cwd is not None:
+            cmd_parts = ['cd', cwd, '&&',
+                    '(' + (' '.join(map(shlex.quote, cmd_parts))) + ')']
+        parts = [
+            'ssh',
+            '-o',
+            'UserKnownHostsFile=/dev/null',
+            '-o',
+            'StrictHostKeyChecking=no',
+            host.name,
+            '--'] + cmd_parts
+        #print(parts)
+        super().__init__(parts, **kwargs)
+
+
+class SimpleComponent(Component):
+    def __init__(self, label, cmd_parts, verbose=True, canfail=False,
+            *args, **kwargs):
+        self.label = label
+        self.verbose = verbose
+        self.canfail = canfail
+        self.cmd_parts = cmd_parts
+        super().__init__(cmd_parts, *args, **kwargs)
+
+    async def process_out(self, lines, eof):
+        if self.verbose:
+            for l in lines:
+                print(self.label, 'OUT:', lines)
+
+    async def process_err(self, lines, eof):
+        if self.verbose:
+            for l in lines:
+                print(self.label, 'ERR:', lines)
+
+    async def terminated(self, rc):
+        if self.verbose:
+            print(self.label, 'TERMINATED:', rc)
+        if not self.canfail and rc != 0:
+            raise Exception('Command Failed: ' + str(self.cmd_parts))
+
+
+# runs the list of commands as strings sequentially
+async def run_cmdlist(label, cmds, verbose=True):
+    i = 0
+    for cmd in cmds:
+        cmdC = SimpleComponent(label + '.' + str(i), shlex.split(cmd),
+                verbose=verbose)
+        await cmdC.start()
+        await cmdC.wait()
+
+async def await_file(path, delay=0.05):
+    print('await_file(%s)' % path)
+    while not os.path.exists(path):
+        await asyncio.sleep(delay)

experiments/modes/experiments.py

+import os
+import asyncio
+import modes.exectools as exectools
+import shlex
+import time
+import json
+
+class Experiment(object):
+    name = None
+    timeout = None
+    checkpoint = False
+
+    def __init__(self, name):
+        self.name = name
+        self.hosts = []
+        self.nics = []
+        self.networks = []
+
+    def add_host(self, sim):
+        self.hosts.append(sim)
+
+    def add_nic(self, sim):
+        self.nics.append(sim)
+
+    def add_network(self, sim):
+        self.networks.append(sim)
+
+    async def prepare(self, env):
+        # generate config tars
+        for host in self.hosts:
+            path = env.cfgtar_path(host)
+            print('preparing config tar:', path)
+            host.node_config.make_tar(path)
+
+        # prepare all simulators in parallel
+        sims = []
+        for sim in self.hosts + self.nics + self.networks:
+            prep_cmds = [pc for pc in sim.prep_cmds(env)]
+            sims.append(exectools.run_cmdlist('prepare_' + self.name, prep_cmds))
+        await asyncio.wait(sims)
+
+    async def run(self, env):
+        running = []
+        sockets = []
+        out = ExpOutput(self)
+        try:
+            out.set_start()
+
+            print('%s: starting NICS' % self.name)
+            for nic in self.nics:
+                print('start NIC:', nic.run_cmd(env))
+                sc = exectools.SimpleComponent(nic.full_name(),
+                        shlex.split(nic.run_cmd(env)))
+                await sc.start()
+                running.append((nic, sc))
+
+                sockets.append(env.nic_pci_path(nic))
+                sockets.append(env.nic_eth_path(nic))
+                sockets.append(env.nic_shm_path(nic))
+
+            print('%s: waiting for sockets' % self.name)
+            for s in sockets:
+                await exectools.await_file(s)
+
+            # start networks
+            for net in self.networks:
+                print('start Net:', net.run_cmd(env))
+                sc = exectools.SimpleComponent(net.full_name(),
+                        shlex.split(net.run_cmd(env)))
+                await sc.start()
+                running.append((net, sc))
+
+            # start hosts
+            wait_hosts = []
+            for host in self.hosts:
+                print('start Host:', host.run_cmd(env))
+                sc = exectools.SimpleComponent(host.full_name(),
+                        shlex.split(host.run_cmd(env)))
+                await sc.start()
+                running.append((host,sc))
+
+                if host.wait:
+                    wait_hosts.append(sc)
+
+            print('%s: waiting for hosts to terminate' % self.name)
+            for sc in wait_hosts:
+                await sc.wait()
+            # wait for necessary hosts to terminate
+        except:
+            out.set_failed()
+
+        finally:
+            out.set_end()
+
+            # shut things back down
+            print('%s: cleaning up' % self.name)
+            scs = []
+            for _,sc in running:
+                scs.append(sc.int_term_kill())
+            await asyncio.wait(scs)
+
+            for _,sc in running:
+                await sc.wait()
+
+            for sock in sockets:
+                os.remove(sock)
+
+            for sim,sc in running:
+                out.add_sim(sim, sc)
+        return out
+
+
+
+    def resreq_mem(self):
+        mem = 0
+        for h in self.hosts:
+            mem += h.resreq_mem()
+        for n in self.nics:
+            mem += n.resreq_mem()
+        for n in self.networks:
+            mem += n.resreq_mem()
+        return mem
+
+    def resreq_cores(self):
+        cores = 0
+        for h in self.hosts:
+            cores += h.resreq_cores()
+        for n in self.nics:
+            cores += n.resreq_cores()
+        for n in self.networks:
+            cores += n.resreq_cores()
+        return cores
+
+class ExpEnv(object):
+    def __init__(self, repo_path, workdir):
+        self.repodir = os.path.abspath(repo_path)
+        self.workdir = os.path.abspath(workdir)
+        self.qemu_img_path = self.repodir + '/qemu/qemu-img'
+        self.qemu_path = self.repodir + '/qemu/x86_64-softmmu/qemu-system-x86_64'
+        self.qemu_kernel_path = self.repodir + '/images/bzImage'
+
+    def hdcopy_path(self, sim):
+        return '%s/hdcopy.%s.%d' % (self.workdir, sim.name, id(sim))
+
+    def hd_path(self, hd_name):
+        return '%s/images/output-%s/%s' % (self.repodir, hd_name, hd_name)
+
+    def cfgtar_path(self, sim):
+        return '%s/cfg.%s.%d.tar' % (self.workdir, sim.name, id(sim))
+
+    def nic_pci_path(self, sim):
+        return '%s/nic.pci.%s.%d' % (self.workdir, sim.name, id(sim))
+
+    def nic_eth_path(self, sim):
+        return '%s/nic.eth.%s.%d' % (self.workdir, sim.name, id(sim))
+
+    def nic_shm_path(self, sim):
+        return '%s/nic.shm.%s.%d' % (self.workdir, sim.name, id(sim))
+
+class ExpOutput(object):
+    def __init__(self, exp):
+        self.exp_name = exp.name
+        self.start_time = None
+        self.end_time = None
+        self.sims = {}
+        self.success = True
+
+    def set_start(self):
+        self.start_time = time.time()
+
+    def set_end(self):
+        self.end = time.time()
+
+    def set_failed(self):
+        self.success = False
+
+    def add_sim(self, sim, comp):
+        obj = {
+            'class': sim.__class__.__name__,
+            'cmd': comp.cmd_parts,
+            'stdout': comp.stdout,
+            'stderr': comp.stderr,
+        }
+        self.sims[sim.full_name()] = obj
+
+    def dumps(self):
+        return json.dumps(self.__dict__)
+
+
+
+def run_exp_local(exp, env):
+    if os.path.exists(env.workdir):
+        raise Exception('Workdir already exists')
+
+    os.mkdir(env.workdir)
+    asyncio.run(exp.prepare(env))
+    return asyncio.run(exp.run(env))

experiments/modes/nodeconfig.py

+import tarfile
+import io
+
+class NodeConfig(object):
+    sim = 'qemu'
+    ip = '10.0.0.1'
+    prefix = 24
+    cores = 1
+    memory = 512
+    app = None
+
+    def config_str(self):
+        if self.sim == 'qemu':
+            cp_es = []
+            exit_es = ['poweroff -f']
+        else:
+            cp_es = ['m5 checkpoint']
+            exit_es = ['m5 exit']
+
+        es = self.prepare_pre_cp() + cp_es + self.prepare_post_cp() + \
+            self.run_cmds() + self.cleanup_cmds() + exit_es
+        return '\n'.join(es)
+
+    def make_tar(self, path):
+        tar = tarfile.open(path, 'w:')
+
+        # add main run script
+        cfg_i = tarfile.TarInfo('guest/run.sh')
+        cfg_i.mode = 0o777
+        cfg_f = self.strfile(self.config_str())
+        cfg_f.seek(0, io.SEEK_END)
+        cfg_i.size = cfg_f.tell()
+        cfg_f.seek(0, io.SEEK_SET)
+        tar.addfile(tarinfo=cfg_i, fileobj=cfg_f)
+        cfg_f.close()
+
+        # add additional config files
+        for (n,f) in self.config_files().items():
+            f_i = tarfile.TarInfo('guest/' + n)
+            f_i.mode = 0o777
+            f.seek(0, io.SEEK_END)
+            f_i.size = f.tell()
+            f.seek(0, io.SEEK_SET)
+            tar.addfile(tarinfo=f_i, fileobj=f)
+            f.close()
+
+        tar.close()
+
+    def prepare_pre_cp(self):
+        return [
+            'export HOME=/root',
+            'export LANG=en_US',
+            'export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:' + \
+                '/usr/bin:/sbin:/bin:/usr/games:/usr/local/games"'
+        ]
+
+    def prepare_post_cp(self):
+        return []
+
+    def run_cmds(self):
+        return self.app.run_cmds(self)
+
+    def cleanup_cmds(self):
+        return []
+
+    def config_files(self):
+        return {}
+
+    def strfile(self, s):
+        return io.BytesIO(bytes(s, encoding='UTF-8'))
+
+
+class AppConfig(object):
+    def run_cmds(self, node):
+        return []
+
+
+class LinuxNode(NodeConfig):
+    ifname = 'eth0'
+
+    def __init__(self):
+        self.drivers = []
+
+    def prepare_post_cp(self):
+        l = []
+        for d in self.drivers:
+            if d[0] == '/':
+                l.append('insmod ' + d)
+            else:
+                l.append('modprobe ' + d)
+        l.append('ip link set dev ' + self.ifname + ' up')
+        l.append('ip addr add %s/%d dev %s' %
+                (self.ip, self.prefix, self.ifname))
+        return super().prepare_post_cp() + l
+
+class I40eLinuxNode(LinuxNode):
+    def __init__(self):
+        super().__init__()
+        self.drivers.append('i40e')
+
+class CorundumLinuxNode(LinuxNode):
+    def __init__(self):
+        super().__init__()
+        self.drivers.append('/tmp/guest/mqnic.ko')
+
+    def config_files(self):
+        m = {'mqnic.ko': open('../images/mqnic/mqnic.ko', 'r')}
+        return {**m, **super().config_files()}
+
+
+
+class MtcpNode(NodeConfig):
+    pci_dev = '0000:00:02.0'
+    num_hugepages = 4096
+
+    def prepare_pre_cp(self):
+        return super().prepare_pre_cp() + [
+            'mount -t proc proc /proc',
+            'mount -t sysfs sysfs /sys',
+            'mkdir -p /dev/hugepages',
+            'mount -t hugetlbfs nodev /dev/hugepages',
+            'mkdir -p /dev/shm',
+            'mount -t tmpfs tmpfs /dev/shm'
+            'echo ' + str(self.num_hugepages) + ' > /sys/devices/system/' + \
+                    'node/node0/hugepages/hugepages-2048kB/nr_hugepages',
+        ]
+
+    def prepare_post_cp(self):
+        return super().prepare_post_cp() + [
+            'insmod /root/mtcp/dpdk/x86_64-native-linuxapp-gcc/kmod/igb_uio.ko',
+            '/root/mtcp/dpdk/usertools/dpdk-devbind.py -b igb_uio ' +
+                self.pci_dev,
+            'insmod /root/mtcp/dpdk-iface-kmod/dpdk_iface.ko',
+            '/root/mtcp/dpdk-iface-kmod/dpdk_iface_main',
+            'ip link set dev dpdk0 up',
+            'ip addr add %s/%d dev dpdk0' % (self.ip, self.prefix)
+        ]
+
+    def config_files(self):
+        m = {'mtcp.conf': self.strfile("io = dpdk\n"
+                "num_cores = " + str(self.cores) + "\n"
+                "num_mem_ch = 4\n"
+                "port = dpdk0\n"
+                "max_concurrency = 4096\n"
+                "max_num_buffers = 4096\n"
+                "rcvbuf = 8192\n"
+                "sndbuf = 8192\n"
+                "tcp_timeout = 10\n"
+                "tcp_timewait = 0\n"
+                "stat_print = dpdk0\n")}
+
+        return {**m, **super().config_files()}
+
+class TASNode(NodeConfig):
+    pci_dev = '0000:00:02.0'
+    num_hugepages = 4096
+
+    def prepare_pre_cp(self):
+        return super().prepare_pre_cp() + [
+            'mount -t proc proc /proc',
+            'mount -t sysfs sysfs /sys',
+            'mkdir -p /dev/hugepages',
+            'mount -t hugetlbfs nodev /dev/hugepages',
+            'mkdir -p /dev/shm',
+            'mount -t tmpfs tmpfs /dev/shm'
+            'echo ' + str(self.num_hugepages) + ' > /sys/devices/system/' + \
+                    'node/node0/hugepages/hugepages-2048kB/nr_hugepages',
+        ]
+
+    def prepare_post_cp(self):
+        return super().prepare_post_cp() + [
+            'insmod /root/dpdk/lib/modules/5.4.46/extra/dpdk/igb_uio.ko',
+            '/root/mtcp/dpdk/usertools/dpdk-devbind.py -b igb_uio ' +
+                self.pci_dev,
+            'insmod /root/mtcp/dpdk-iface-kmod/dpdk_iface.ko',
+            '/root/mtcp/dpdk-iface-kmod/dpdk_iface_main',
+            'ip link set dev dpdk0 up',
+            'ip addr add %s/%d dev dpdk0' % (self.ip, self.prefix)
+        ]
+
+
+class IperfTCPServer(AppConfig):
+    def run_cmds(self, node):
+        return ['iperf -s -l 32M -w 32M']
+
+class IperfUDPServer(AppConfig):
+    def run_cmds(self, node):
+        return ['iperf -s -u']
+
+class IperfTCPClient(AppConfig):
+    server_ip = '10.0.0.1'
+    procs = 1
+
+    def run_cmds(self, node):
+        return ['iperf -l 32M -w 32M  -c ' + self.server_ip +  ' -i 1 -P ' +
+                str(self.procs)]
+
+class IperfUDPClient(AppConfig):
+    server_ip = '10.0.0.1'
+    rate = '150m'
+    def run_cmds(self, node):
+        return ['iperf -c ' + self.server_ip + ' -u -b ' + self.rate]

experiments/modes/simulators.py

+class Simulator(object):
+    # number of cores required for this simulator
+    def resreq_cores(self):
+        return 1
+
+    # memory required for this simulator (in MB)
+    def resreq_mem(self):
+        return 64
+
+    def prep_cmds(self, env):
+        return []
+
+    def run_cmd(self, env):
+        pass
+
+class HostSim(Simulator):
+    node_config = None
+    disk_image = 'base'
+    name = ''
+    wait = False
+
+    def __init__(self):
+        self.nics = []
+
+    def full_name(self):
+        return 'host.%s.%d' % (self.name, id(self))
+
+    def add_nic(self, nic):
+        self.nics.append(nic)
+
+class NICSim(Simulator):
+    network = None
+    name = ''
+
+    def set_network(self, net):
+        self.network = net
+        net.nics.append(self)
+
+    def basic_run_cmd(self, env, name):
+        return '%s/%s %s %s %s' % \
+            (env.repodir, name, env.nic_pci_path(self), env.nic_eth_path(self),
+                    env.nic_shm_path(self))
+
+    def full_name(self):
+        return 'nic.%s.%d' % (self.name, id(self))
+
+class NetSim(Simulator):
+    name = ''
+
+    def __init__(self):
+        self.nics = []
+
+    def full_name(self):
+        return 'net.%s.%d' % (self.name, id(self))
+
+
+class QemuHost(HostSim):
+    mem = 16 * 1024 # 16G
+
+    def resreq_cores(self):
+        return self.node_config.cores + 1
+
+    def resreq_mem(self):
+        return 4096
+
+    def prep_cmds(self, env):
+        to_path = env.hdcopy_path(self)
+        return [f'{env.qemu_img_path} create -f qcow2 -o '
+            f'backing_file="{env.hd_path(self.disk_image)}" '
+            f'{env.hdcopy_path(self)}']
+
+    def run_cmd(self, env):
+        cmd = (f'{env.qemu_path} -machine q35 -cpu host -serial mon:stdio '
+            '-display none -enable-kvm -nic none '
+            f'-kernel {env.qemu_kernel_path} '
+            f'-drive file={env.hdcopy_path(self)},if=ide,index=0,media=disk '
+            f'-drive file={env.cfgtar_path(self)},if=ide,index=1,media=disk,'
+                'driver=raw '
+            '-append "earlyprintk=ttyS0 console=ttyS0 root=/dev/sda1 '
+                'init=/home/ubuntu/guestinit.sh rw" '
+            f'-m {self.mem} -smp {self.node_config.cores} ')
+        if len(self.nics) > 0:
+            assert len(self.nics) == 1
+            cmd += f'-chardev socket,path={env.nic_pci_path(self.nics[0])},'
+            cmd += 'id=cosimcd '
+            cmd += '-device cosim-pci,chardev=cosimcd '
+        return cmd
+
+class Gem5Host(HostSim):
+    mem = 16 * 1024 # 16G
+    cpu_type_cp = 'X86KvmCPU'
+    cpu_type = 'TimingSimpleCPU'
+
+    def resreq_cores(self):
+        return 1
+
+    def resreq_mem(self):
+        return 4096
+
+
+    def run_cmd(self, env):
+        cpu_type = self.cpu_type
+        if env.create_cp:
+            cpu_type = self.cpu_type_cp
+
+        self.cp_cpu_type
+        cmd = (f'{env.gem5_path} --outdir={env.gem5_outdir(self)} '
+            f'{env.gem5_py_path} --caches --l2cache --l3cache '
+            '--l1d_size=32kB --l1i_size=32kB --l2_size=2MB --l3_size=32MB '
+            '--cacheline_size=64 --cpu-clock=3GHz '
+            f'--checkpoint-dir={env.gem5_cpdir(self)} '
+            f'--kernel={env.gem5_kernel_path} '
+            f'--disk-image={env.hd_raw_path(self.disk_image)} '
+            f'--disk-image={env.cfgtar_path(self)} '
+            f'--cpu-type={cpu_type} --mem-size={self.mem}MB '
+            '--ddio-enabled --ddio-way-part=8 --mem-type=DDR4_2400_16x4 ')
+
+        if env.restore_cp:
+            cmd += '-r 0 '
+
+        if len(self.nics) > 0:
+            assert len(self.nics) == 1
+            nic = self.nics[0]
+            cmd += f'--cosim-pci={env.nic_pci_path(nic)} '
+            cmd += f'--cosim-shm={env.nic_shm_path(nic)} '
+            if cpu_type == 'TimingSimpleCPU':
+                cmd += '--cosim-sync '
+            if isinstance(nic, I40eNIC):
+                cmd += '--cosim-type=i40e '
+        return cmd
+
+
+
+class CorundumVerilatorNIC(NICSim):
+    def resreq_mem(self):
+        # this is a guess
+        return 512
+
+    def run_cmd(self, env):
+        return self.basic_run_cmd(env, 'corundum/corundum_verilator')
+
+class CorundumBMNIC(NICSim):
+    def run_cmd(self, env):
+        return self.basic_run_cmd(env, 'corundum_bm/corundum_bm')
+
+class I40eNIC(NICSim):
+    def run_cmd(self, env):
+        return self.basic_run_cmd(env, 'i40e_bm/i40e_bm')
+
+
+
+class WireNet(NetSim):
+    def run_cmd(self, env):
+        assert len(self.nics) == 2
+        return '%s/net_wire/net_wire %s %s' % \
+                (env.repodir, env.nic_eth_path(self.nics[0]),
+                        env.nic_eth_path(self.nics[1]))
+
+class SwitchNet(NetSim):
+    def run_cmd(self, env):
+        cmd = env.repodir + '/net_switch/net_switch'
+        for n in self.nics:
+            cmd += ' -s ' + env.nic_eth_path(n)
+        return cmd

net_switch/Makefile

+CPPFLAGS := -I../proto -I../netsim_common/include
+CFLAGS := -Wall -Wextra -O3
+LD := g++
+
+all: net_switch
+
+net_switch: net_switch.o ../netsim_common/libnetsim_common.a
+	$(LD) -o $@ $^
+
+clean:
+	rm -f *.o net_switch

net_switch/net_switch.cc

+#include <cstdio>
+#include <cstdlib>
+#include <csignal>
+#include <climits>
+#include <cstring>
+#include <unistd.h>
+#include <vector>
+#include <unordered_map>
+
+extern "C" {
+#include <netsim.h>
+};
+
+#define SYNC_PERIOD (500 * 1000ULL) // 500ns
+#define ETH_LATENCY (500 * 1000ULL) // 500ns
+
+/* MAC address type */
+struct MAC {
+    const volatile uint8_t *data;
+
+    MAC(const volatile uint8_t *data)
+        : data(data) {}
+
+    bool operator==(const MAC &other) const {
+        for (int i = 0; i < 6; i++) {
+            if (data[i] != other.data[i]) {
+                return false;
+            }
+        }
+        return true;
+    }
+};
+namespace std {
+    template <>
+    struct hash<MAC>
+    {
+        size_t operator()(const MAC &m) const {
+            size_t res = 0;
+            for (int i = 0; i < 6; i++) {
+                res = (res << 4) | (res ^ m.data[i]);
+            }
+            return res;
+        }
+    };
+} // namespace std
+
+/* Global variables */
+static uint64_t cur_ts = 0;
+static int exiting = 0;
+static const volatile uint8_t bcast[6] = {0xFF};
+static const MAC bcast_addr(bcast);
+static std::vector<struct netsim_interface> nsifs;
+static std::unordered_map<MAC, int> mac_table;
+
+static void sigint_handler(int dummy)
+{
+    exiting = 1;
+}
+
+static void forward_pkt(volatile struct cosim_eth_proto_d2n_send *tx, int port)
+{
+    volatile union cosim_eth_proto_n2d *msg_to;
+    msg_to = netsim_n2d_alloc(&nsifs[port], cur_ts, ETH_LATENCY);
+    if (msg_to != NULL) {
+        volatile struct cosim_eth_proto_n2d_recv *rx;
+        rx = &msg_to->recv;
+        rx->len = tx->len;
+        rx->port = 0;
+        memcpy((void *)rx->data, (void *)tx->data, tx->len);
+
+        // WMB();
+        rx->own_type = COSIM_ETH_PROTO_N2D_MSG_RECV |
+            COSIM_ETH_PROTO_N2D_OWN_DEV;
+    } else {
+        fprintf(stderr, "forward_pkt: dropping packet\n");
+    }
+}
+
+static void switch_pkt(struct netsim_interface *nsif, int iport)
+{
+    volatile union cosim_eth_proto_d2n *msg_from = netsim_d2n_poll(nsif, cur_ts);
+    if (msg_from == NULL) {
+        return;
+    }
+
+    uint8_t type = msg_from->dummy.own_type & COSIM_ETH_PROTO_D2N_MSG_MASK;
+    if (type == COSIM_ETH_PROTO_D2N_MSG_SEND) {
+        volatile struct cosim_eth_proto_d2n_send *tx;
+        tx = &msg_from->send;
+        // Get MAC addresses
+        MAC dst(tx->data), src(tx->data+6);
+        // MAC learning
+        if (!(src == bcast_addr)) {
+            mac_table[src] = iport;
+        }
+        // L2 forwarding
+        if (mac_table.count(dst) > 0) {
+            int eport = mac_table.at(dst);
+            forward_pkt(tx, eport);
+        } else {
+            // Broadcast
+            for (int eport = 0; eport < nsifs.size(); eport++) {
+                if (eport != iport) {
+                    // Do not forward to ingress port
+                    forward_pkt(tx, eport);
+                }
+            }
+        }
+    } else if (type == COSIM_ETH_PROTO_D2N_MSG_SYNC) {
+    } else {
+        fprintf(stderr, "switch_pkt: unsupported type=%u\n", type);
+        abort();
+    }
+    netsim_d2n_done(nsif, msg_from);
+}
+
+int main(int argc, char *argv[])
+{
+    int c;
+
+    // Parse command line argument
+    while ((c = getopt(argc, argv, "s:")) != -1) {
+        switch (c) {
+        case 's': {
+            struct netsim_interface nsif;
+            int sync = 1;
+            if (netsim_init(&nsif, optarg, &sync) != 0) {
+                return EXIT_FAILURE;
+            }
+            nsifs.push_back(nsif);
+        }
+        default:
+            fprintf(stderr, "unknown option %c\n", c);
+        }
+    }
+
+    if (nsifs.empty()) {
+        fprintf(stderr, "Usage: net_switch -s SOCKET-A [-s SOCKET-B ...]\n");
+        return EXIT_FAILURE;
+    }
+
+    signal(SIGINT, sigint_handler);
+    signal(SIGTERM, sigint_handler);
+
+    printf("start polling\n");
+    while (!exiting) {
+        // Sync all interfaces
+        for (auto &nsif : nsifs) {
+            if (netsim_n2d_sync(&nsif, cur_ts, ETH_LATENCY, SYNC_PERIOD) != 0) {
+                fprintf(stderr, "netsim_n2d_sync failed\n");
+                abort();
+            }
+        }
+        // Switch packets
+        uint64_t min_ts;
+        do {
+            min_ts = ULLONG_MAX;
+            for (int port = 0; port < nsifs.size(); port++) {
+                auto &nsif = nsifs.at(port);
+                switch_pkt(&nsif, port);
+                if (nsif.sync) {
+                    uint64_t ts = netsim_d2n_timestamp(&nsif);
+                    min_ts = ts < min_ts ? ts : min_ts;
+                }
+            }
+        } while (!exiting && (min_ts <= cur_ts));
+
+        // Update cur_ts
+        if (min_ts < ULLONG_MAX) {
+            cur_ts = min_ts;
+        }
+    }
+
+    return 0;
+}