Merge branch 'master' into experiments

corundum/corundum_verilator.cpp

@@ -19,13 +19,15 @@ extern "C" {
 #include "dma.h"
 #include "mem.h"
 
-#define CLOCK_PERIOD (4 * 1000ULL) // 4ns -> 2500MHz
-#define SYNC_PERIOD (500 * 1000ULL) // 500ns
-#define PCI_LATENCY (500 * 1000ULL) // 500ns
-#define ETH_LATENCY (500 * 1000ULL) // 500ns
-
 struct DMAOp;
 
+static uint64_t clock_period = 4 * 1000ULL; // 4ns -> 250MHz
+static uint64_t sync_period = 500 * 1000ULL; // 500ns
+static uint64_t pci_latency = 500 * 1000ULL; // 500ns
+static uint64_t eth_latency = 500 * 1000ULL; // 500ns
+
+
+
 static volatile int exiting = 0;
 uint64_t main_time = 0;
 static struct nicsim_params nsparams;
@@ -613,7 +615,7 @@ class EthernetTx {
             send = &msg->send;
             memcpy((void *) send->data, packet_buf, packet_len);
             send->len = packet_len;
-            send->timestamp = main_time + ETH_LATENCY;
+            send->timestamp = main_time + eth_latency;
 
             //WMB();
             send->own_type = COSIM_ETH_PROTO_D2N_MSG_SEND |
@@ -890,13 +892,22 @@ int main(int argc, char *argv[])
     Verilated::traceEverOn(true);
 #endif
 
-    if (argc != 4 && argc != 5) {
+    if (argc < 4 && argc > 9) {
         fprintf(stderr, "Usage: corundum_verilator PCI-SOCKET ETH-SOCKET "
-                "SHM [START-TICK]\n");
+                "SHM [START-TICK] [SYNC-PERIOD] [PCI-LATENCY] [ETH-LATENCY] "
+                "[CLOCK-FREQ-MHZ]\n");
         return EXIT_FAILURE;
     }
-    if (argc == 5)
+    if (argc >= 5)
         main_time = strtoull(argv[4], NULL, 0);
+    if (argc >= 6)
+        sync_period = strtoull(argv[5], NULL, 0) * 1000ULL;
+    if (argc >= 7)
+        pci_latency = strtoull(argv[6], NULL, 0) * 1000ULL;
+    if (argc >= 8)
+        eth_latency = strtoull(argv[7], NULL, 0) * 1000ULL;
+    if (argc >= 9)
+        clock_period = 1000000ULL / strtoull(argv[8], NULL, 0);
 
     struct cosim_pcie_proto_dev_intro di;
     memset(&di, 0, sizeof(di));
@@ -916,9 +927,9 @@ int main(int argc, char *argv[])
     nsparams.pci_socket_path = argv[1];
     nsparams.eth_socket_path = argv[2];
     nsparams.shm_path = argv[3];
-    nsparams.pci_latency = PCI_LATENCY;
-    nsparams.eth_latency = ETH_LATENCY;
-    nsparams.sync_delay = SYNC_PERIOD;
+    nsparams.pci_latency = pci_latency;
+    nsparams.eth_latency = eth_latency;
+    nsparams.sync_delay = sync_period;
     if (nicsim_init(&nsparams, &di)) {
         return EXIT_FAILURE;
     }
@@ -1054,7 +1065,7 @@ int main(int argc, char *argv[])
 
         /* falling edge */
         top->clk = !top->clk;
-        main_time += CLOCK_PERIOD / 2;
+        main_time += clock_period / 2;
         top->eval();
 
         mmio.step();
@@ -1076,7 +1087,7 @@ int main(int argc, char *argv[])
 
         /* raising edge */
         top->clk = !top->clk;
-        main_time += CLOCK_PERIOD / 2;
+        main_time += clock_period / 2;
 
         //top->s_axis_tx_ptp_ts_96 = main_time;
         top->s_axis_tx_ptp_ts_valid = 1;

experiments/modes/simulators.py

@@ -20,6 +20,9 @@ class HostSim(Simulator):
     sleep = 0
     cpu_freq = '3GHz'
 
+    sync_period = 500
+    pci_latency = 500
+
     def __init__(self):
         self.nics = []
 
@@ -37,14 +40,23 @@ class NICSim(Simulator):
     network = None
     name = ''
 
+    sync_period = 500
+    pci_latency = 500
+    eth_latency = 500
+
     def set_network(self, net):
         self.network = net
         net.nics.append(self)
 
-    def basic_run_cmd(self, env, name):
-        return '%s/%s %s %s %s' % \
+    def basic_run_cmd(self, env, name, extra=None):
+        cmd = '%s/%s %s %s %s 0 %d %d %d' % \
             (env.repodir, name, env.nic_pci_path(self), env.nic_eth_path(self),
-                    env.nic_shm_path(self))
+                    env.nic_shm_path(self), self.sync_period, self.pci_latency,
+                    self.eth_latency)
+
+        if extra is not None:
+            cmd += ' ' + extra
+        return cmd
 
     def full_name(self):
         return 'nic.' + self.name
@@ -52,6 +64,8 @@ class NICSim(Simulator):
 class NetSim(Simulator):
     name = ''
     opt = ''
+    sync_period = 500
+    eth_latency = 500
 
     def __init__(self):
         self.nics = []
@@ -63,6 +77,9 @@ class NetSim(Simulator):
 class QemuHost(HostSim):
     sync = False
     def resreq_cores(self):
+        if self.sync:
+            return 1
+        else:
             return self.node_config.cores + 1
 
     def resreq_mem(self):
@@ -94,8 +111,15 @@ class QemuHost(HostSim):
             assert len(self.nics) == 1
             cmd += f'-chardev socket,path={env.nic_pci_path(self.nics[0])},'
             cmd += 'id=cosimcd '
-            sync_onoff = 'on' if self.sync else 'off'
-            cmd += f'-device cosim-pci,chardev=cosimcd,sync={sync_onoff} '
+            cmd += f'-device cosim-pci,chardev=cosimcd'
+            if self.sync:
+                cmd += ',sync=on'
+                cmd += f',pci-latency={self.pci_latency}'
+                cmd += f',sync-period={self.sync_period}'
+            else:
+                cmd += ',sync=off'
+            cmd += ' '
+
         return cmd
 
 class Gem5Host(HostSim):
@@ -143,6 +167,8 @@ class Gem5Host(HostSim):
             cmd += f'--cosim-shm={env.nic_shm_path(nic)} '
             if cpu_type == 'TimingSimpleCPU':
                 cmd += '--cosim-sync '
+                cmd += f'--cosim-pci-lat={self.pci_latency} '
+                cmd += f'--cosim-sync-int={self.sync_period} '
             if isinstance(nic, I40eNIC):
                 cmd += '--cosim-type=i40e '
         return cmd
@@ -150,12 +176,15 @@ class Gem5Host(HostSim):
 
 
 class CorundumVerilatorNIC(NICSim):
+    clock_freq = 250 # MHz
+
     def resreq_mem(self):
         # this is a guess
         return 512
 
     def run_cmd(self, env):
-        return self.basic_run_cmd(env, 'corundum/corundum_verilator')
+        return self.basic_run_cmd(env, 'corundum/corundum_verilator',
+            str(self.clock_freq))
 
 class CorundumBMNIC(NICSim):
     def run_cmd(self, env):
@@ -170,13 +199,15 @@ class I40eNIC(NICSim):
 class WireNet(NetSim):
     def run_cmd(self, env):
         assert len(self.nics) == 2
-        return '%s/net_wire/net_wire %s %s' % \
+        return '%s/net_wire/net_wire %s %s %d %d' % \
                 (env.repodir, env.nic_eth_path(self.nics[0]),
-                        env.nic_eth_path(self.nics[1]))
+                        env.nic_eth_path(self.nics[1]),
+                        self.sync_period, self.eth_latency)
 
 class SwitchNet(NetSim):
     def run_cmd(self, env):
         cmd = env.repodir + '/net_switch/net_switch'
+        cmd += f' -S {self.sync_period} -E {self.eth_latency}'
         for n in self.nics:
             cmd += ' -s ' + env.nic_eth_path(n)
         return cmd

experiments/pyexps/mtcp_cores.py

+import modes.experiments as exp
+import modes.simulators as sim
+import modes.nodeconfig as node
+
+
+server_cores_configs = [1, 2, 4, 8]
+stacks = ['linux', 'mtcp']
+client_cores = 1
+num_clients = 1
+connections = 128
+msg_size = 64
+
+experiments = []
+for server_cores in server_cores_configs:
+  for stack in stacks:
+    e = exp.Experiment('qemu-ib-switch-mtcp_cores-%s-%d' % (stack,server_cores))
+    e.timeout = 5* 60
+    # add meta data for output file
+    e.metadata['msg_size'] = msg_size
+    e.metadata['stack'] = stack
+
+    net = sim.SwitchNet()
+    e.add_network(net)
+
+    if stack == 'tas':
+        n = node.TASNode
+    elif stack == 'mtcp':
+        n = node.MtcpNode
+    else:
+        n = node.I40eLinuxNode
+
+    servers = sim.create_basic_hosts(e, 1, 'server', net, sim.I40eNIC, sim.QemuHost,
+            n, node.RPCServer)
+
+    clients = sim.create_basic_hosts(e, num_clients, 'client', net, sim.I40eNIC,
+            sim.QemuHost, n, node.RPCClient, ip_start = 2)
+
+    for h in servers:
+        h.node_config.cores = server_cores
+        h.node_config.app.threads = server_cores
+        h.node_config.app.max_flows = connections * 4
+        h.sleep = 5
+
+
+
+    for c in clients:
+        c.wait = True
+        c.node_config.cores = client_cores
+        c.node_config.app.threads = client_cores
+
+        c.node_config.app.server_ip = servers[0].node_config.ip
+        c.node_config.app.max_msgs_conn = 1
+        c.node_config.app.max_flows = \
+            int(connections / num_clients / client_cores)
+
+    for h in servers + clients:
+        h.node_config.app.max_bytes = msg_size
+
+        if stack == 'linux':
+            h.node_config.disk_image = 'tas'
+        elif stack == 'tas':
+            c.node_config.cores += 2
+            c.node_config.fp_cores = 1
+    experiments.append(e)
+

experiments/pyexps/mtcp_mpcs.py

+import modes.experiments as exp
+import modes.simulators as sim
+import modes.nodeconfig as node
+
+
+mpcs = [1, 8, 128]
+stacks = ['linux', 'mtcp']
+server_cores = 8
+client_cores = 4
+num_clients = 4
+connections = 512
+msg_size = 64
+
+experiments = []
+for mpc in mpcs:
+  for stack in stacks:
+    e = exp.Experiment('qemu-ib-switch-mtcp_mpc-%s-%d' % (stack,mpc))
+    e.timeout = 5* 60
+    # add meta data for output file
+    e.metadata['mpc'] = mpc
+    e.metadata['stack'] = stack
+
+    net = sim.SwitchNet()
+    e.add_network(net)
+
+    if stack == 'tas':
+        n = node.TASNode
+    elif stack == 'mtcp':
+        n = node.MtcpNode
+    else:
+        n = node.I40eLinuxNode
+
+    servers = sim.create_basic_hosts(e, 1, 'server', net, sim.I40eNIC, sim.QemuHost,
+            n, node.RPCServer)
+
+    clients = sim.create_basic_hosts(e, num_clients, 'client', net, sim.I40eNIC,
+            sim.QemuHost, n, node.RPCClient, ip_start = 2)
+
+    for h in servers:
+        h.node_config.cores = server_cores
+        h.node_config.app.threads = server_cores
+        h.node_config.app.max_flows = connections * 4
+        h.sleep = 5
+
+
+
+    for c in clients:
+        c.wait = True
+        c.node_config.cores = client_cores
+        c.node_config.app.threads = client_cores
+
+        c.node_config.app.server_ip = servers[0].node_config.ip
+        c.node_config.app.max_msgs_conn = mpc
+        c.node_config.app.max_flows = \
+            int(connections / num_clients / client_cores)
+
+    for h in servers + clients:
+        h.node_config.app.max_bytes = msg_size
+
+        if stack == 'linux':
+            h.node_config.disk_image = 'tas'
+        elif stack == 'tas':
+            c.node_config.cores += 2
+            c.node_config.fp_cores = 1
+    experiments.append(e)
+

experiments/pyexps/mtcp_msgsz.py

+import modes.experiments as exp
+import modes.simulators as sim
+import modes.nodeconfig as node
+
+
+msg_sizes = [64, 1024, 8092]
+stacks = ['linux', 'mtcp']
+server_cores = 8
+client_cores = 4
+num_clients = 4
+connections = 512
+
+experiments = []
+for msg_size in msg_sizes:
+  for stack in stacks:
+    e = exp.Experiment('qemu-ib-switch-mtcp_msgsz-%s-%d' % (stack,msg_size))
+    e.timeout = 5* 60
+    # add meta data for output file
+    e.metadata['msg_size'] = msg_size
+    e.metadata['stack'] = stack
+
+    net = sim.SwitchNet()
+    e.add_network(net)
+
+    if stack == 'tas':
+        n = node.TASNode
+    elif stack == 'mtcp':
+        n = node.MtcpNode
+    else:
+        n = node.I40eLinuxNode
+
+    servers = sim.create_basic_hosts(e, 1, 'server', net, sim.I40eNIC, sim.QemuHost,
+            n, node.RPCServer)
+
+    clients = sim.create_basic_hosts(e, num_clients, 'client', net, sim.I40eNIC,
+            sim.QemuHost, n, node.RPCClient, ip_start = 2)
+
+    for h in servers:
+        h.node_config.cores = server_cores
+        h.node_config.app.threads = server_cores
+        h.node_config.app.max_flows = connections * 4
+        h.sleep = 5
+
+
+
+    for c in clients:
+        c.wait = True
+        c.node_config.cores = client_cores
+        c.node_config.app.threads = client_cores
+
+        c.node_config.app.server_ip = servers[0].node_config.ip
+        c.node_config.app.max_msgs_conn = 1
+        c.node_config.app.max_flows = \
+            int(connections / num_clients / client_cores)
+
+    for h in servers + clients:
+        h.node_config.app.max_bytes = msg_size
+
+        if stack == 'linux':
+            h.node_config.disk_image = 'tas'
+        elif stack == 'tas':
+            c.node_config.cores += 2
+            c.node_config.fp_cores = 1
+    experiments.append(e)
+

libnicbm/nicbm.cc

@@ -12,9 +12,6 @@
 
 //#define DEBUG_NICBM 1
 
-#define SYNC_PERIOD (100 * 1000ULL) // 100ns
-#define PCI_LATENCY (500 * 1000ULL) // 500ns
-#define ETH_LATENCY (500 * 1000ULL) // 500ns
 #define DMA_MAX_PENDING 64
 
 
@@ -395,14 +392,23 @@ int Runner::runMain(int argc, char *argv[])
 {
     uint64_t next_ts;
     uint64_t max_step = 10000;
+    uint64_t sync_period = 100 * 1000ULL;
+    uint64_t pci_latency = 500 * 1000ULL;
+    uint64_t eth_latency = 500 * 1000ULL;
 
-    if (argc != 4 && argc != 5) {
+    if (argc < 4 && argc > 8) {
         fprintf(stderr, "Usage: corundum_bm PCI-SOCKET ETH-SOCKET "
-                "SHM [START-TICK]\n");
+                "SHM [START-TICK] [SYNC-PERIOD] [PCI-LATENCY] [ETH-LATENCY]\n");
         return EXIT_FAILURE;
     }
-    if (argc == 5)
+    if (argc >= 5)
         main_time = strtoull(argv[4], NULL, 0);
+    if (argc >= 6)
+        sync_period = strtoull(argv[5], NULL, 0) * 1000ULL;
+    if (argc >= 7)
+        pci_latency = strtoull(argv[6], NULL, 0) * 1000ULL;
+    if (argc >= 8)
+        eth_latency = strtoull(argv[7], NULL, 0) * 1000ULL;
 
 
     signal(SIGINT, sigint_handler);
@@ -416,9 +422,9 @@ int Runner::runMain(int argc, char *argv[])
     nsparams.pci_socket_path = argv[1];
     nsparams.eth_socket_path = argv[2];
     nsparams.shm_path = argv[3];
-    nsparams.pci_latency = PCI_LATENCY;
-    nsparams.eth_latency = ETH_LATENCY;
-    nsparams.sync_delay = SYNC_PERIOD;
+    nsparams.pci_latency = pci_latency;
+    nsparams.eth_latency = eth_latency;
+    nsparams.sync_delay = sync_period;
     if (nicsim_init(&nsparams, &dintro)) {
         return EXIT_FAILURE;
     }

net_switch/net_switch.cc

@@ -11,8 +11,8 @@ extern "C" {
 #include <netsim.h>
 };
 
-#define SYNC_PERIOD (500 * 1000ULL) // 500ns
-#define ETH_LATENCY (500 * 1000ULL) // 500ns
+static uint64_t sync_period = (500 * 1000ULL); // 500ns
+static uint64_t eth_latency = (500 * 1000ULL); // 500ns
 
 /* MAC address type */
 struct MAC {
@@ -60,7 +60,7 @@ static void sigint_handler(int dummy)
 static void forward_pkt(volatile struct cosim_eth_proto_d2n_send *tx, int port)
 {
     volatile union cosim_eth_proto_n2d *msg_to;
-    msg_to = netsim_n2d_alloc(&nsifs[port], cur_ts, ETH_LATENCY);
+    msg_to = netsim_n2d_alloc(&nsifs[port], cur_ts, eth_latency);
     if (msg_to != NULL) {
         volatile struct cosim_eth_proto_n2d_recv *rx;
         rx = &msg_to->recv;
@@ -117,9 +117,10 @@ static void switch_pkt(struct netsim_interface *nsif, int iport)
 int main(int argc, char *argv[])
 {
     int c;
+    int bad_option = 0;
 
     // Parse command line argument
-    while ((c = getopt(argc, argv, "s:")) != -1) {
+    while ((c = getopt(argc, argv, "s:S:E:")) != -1 && !bad_option) {
         switch (c) {
         case 's': {
             struct netsim_interface nsif;
@@ -131,13 +132,25 @@ int main(int argc, char *argv[])
             nsifs.push_back(nsif);
             break;
         }
+
+        case 'S':
+            sync_period = strtoull(optarg, NULL, 0) * 1000ULL;
+            break;
+
+        case 'E':
+            eth_latency = strtoull(optarg, NULL, 0) * 1000ULL;
+            break;
+
         default:
             fprintf(stderr, "unknown option %c\n", c);
+            bad_option = 1;
+            break;
         }
     }
 
-    if (nsifs.empty()) {
-        fprintf(stderr, "Usage: net_switch -s SOCKET-A [-s SOCKET-B ...]\n");
+    if (nsifs.empty() || bad_option) {
+        fprintf(stderr, "Usage: net_switch [-S SYNC-PERIOD] [-E ETH-LATENCY] "
+                "-s SOCKET-A [-s SOCKET-B ...]\n");
         return EXIT_FAILURE;
     }
 
@@ -148,7 +161,7 @@ int main(int argc, char *argv[])
     while (!exiting) {
         // Sync all interfaces
         for (auto &nsif : nsifs) {
-            if (netsim_n2d_sync(&nsif, cur_ts, ETH_LATENCY, SYNC_PERIOD) != 0) {
+            if (netsim_n2d_sync(&nsif, cur_ts, eth_latency, sync_period) != 0) {
                 fprintf(stderr, "netsim_n2d_sync failed\n");
                 abort();
             }

net_wire/net_wire.c

@@ -36,9 +36,8 @@
 
 #include <netsim.h>
 
-#define SYNC_PERIOD (500 * 1000ULL) // 500ns
-#define ETH_LATENCY (500 * 1000ULL) // 500ns
-
+static uint64_t sync_period = (500 * 1000ULL); // 500ns
+static uint64_t eth_latency = (500 * 1000ULL); // 500ns
 static uint64_t cur_ts;
 static int exiting = 0;
 static pcap_dumper_t *dumpfile = NULL;
@@ -80,7 +79,7 @@ static void move_pkt(struct netsim_interface *from, struct netsim_interface *to)
                     (unsigned char *) tx->data);
         }
 
-        msg_to = netsim_n2d_alloc(to, cur_ts, ETH_LATENCY);
+        msg_to = netsim_n2d_alloc(to, cur_ts, eth_latency);
         if (msg_to != NULL) {
             rx = &msg_to->recv;
             rx->len = tx->len;
@@ -109,8 +108,9 @@ int main(int argc, char *argv[])
     int sync_a, sync_b;
     pcap_t *pc = NULL;
 
-    if (argc != 3 && argc != 4) {
-        fprintf(stderr, "Usage: net_tap SOCKET-A SOCKET-B [PCAP-FILE]\n");
+    if (argc < 3 && argc > 6) {
+        fprintf(stderr, "Usage: net_wire SOCKET-A SOCKET-B [SYNC-PERIOD] "
+                "[ETH-LATENCY] [PCAP-FILE]\n");
         return EXIT_FAILURE;
     }
 
@@ -118,7 +118,13 @@ int main(int argc, char *argv[])
     signal(SIGTERM, sigint_handler);
     signal(SIGUSR1, sigusr1_handler);
 
-    if (argc == 4) {
+    if (argc >= 4)
+        sync_period = strtoull(argv[3], NULL, 0) * 1000ULL;
+
+    if (argc >= 5)
+        eth_latency = strtoull(argv[4], NULL, 0) * 1000ULL;
+
+    if (argc >= 6) {
         pc = pcap_open_dead_with_tstamp_precision(DLT_EN10MB, 65535,
                 PCAP_TSTAMP_PRECISION_NANO);
         if (pc == NULL) {
@@ -126,7 +132,7 @@ int main(int argc, char *argv[])
             return EXIT_FAILURE;
         }
 
-        dumpfile = pcap_dump_open(pc, argv[3]);
+        dumpfile = pcap_dump_open(pc, argv[5]);
     }
 
     sync_a = sync_b = 1;
@@ -139,11 +145,11 @@ int main(int argc, char *argv[])
 
     printf("start polling\n");
     while (!exiting) {
-        if (netsim_n2d_sync(&nsif_a, cur_ts, ETH_LATENCY, SYNC_PERIOD) != 0) {
+        if (netsim_n2d_sync(&nsif_a, cur_ts, eth_latency, sync_period) != 0) {
             fprintf(stderr, "netsim_n2d_sync(nsif_a) failed\n");
             abort();
         }
-        if (netsim_n2d_sync(&nsif_b, cur_ts, ETH_LATENCY, SYNC_PERIOD) != 0) {
+        if (netsim_n2d_sync(&nsif_b, cur_ts, eth_latency, sync_period) != 0) {
             fprintf(stderr, "netsim_n2d_sync(nsif_a) failed\n");
             abort();
         }