Merge branch 'master' into experiments

corundum/corundum_verilator.cpp

@@ -888,26 +888,29 @@ int main(int argc, char *argv[])
 {
     char *vargs[2] = { argv[0], NULL };
     Verilated::commandArgs(1, vargs);
+    int sync_mode = SYNC_MODES;
 #ifdef TRACE_ENABLED
     Verilated::traceEverOn(true);
 #endif
 
-    if (argc < 4 && argc > 9) {
+    if (argc < 4 && argc > 10) {
         fprintf(stderr, "Usage: corundum_verilator PCI-SOCKET ETH-SOCKET "
-                "SHM [START-TICK] [SYNC-PERIOD] [PCI-LATENCY] [ETH-LATENCY] "
+                "SHM [SYNC-MODE] [START-TICK] [SYNC-PERIOD] [PCI-LATENCY] [ETH-LATENCY] "
                 "[CLOCK-FREQ-MHZ]\n");
         return EXIT_FAILURE;
     }
     if (argc >= 5)
-        main_time = strtoull(argv[4], NULL, 0);
+        sync_mode = strtol(argv[4], NULL, 0);
     if (argc >= 6)
-        sync_period = strtoull(argv[5], NULL, 0) * 1000ULL;
+        main_time = strtoull(argv[5], NULL, 0);
     if (argc >= 7)
-        pci_latency = strtoull(argv[6], NULL, 0) * 1000ULL;
+        sync_period = strtoull(argv[6], NULL, 0) * 1000ULL;
     if (argc >= 8)
-        eth_latency = strtoull(argv[7], NULL, 0) * 1000ULL;
+        pci_latency = strtoull(argv[7], NULL, 0) * 1000ULL;
     if (argc >= 9)
-        clock_period = 1000000ULL / strtoull(argv[8], NULL, 0);
+        eth_latency = strtoull(argv[8], NULL, 0) * 1000ULL;
+    if (argc >= 10)
+        clock_period = 1000000ULL / strtoull(argv[9], NULL, 0);
 
     struct cosim_pcie_proto_dev_intro di;
     memset(&di, 0, sizeof(di));
@@ -930,6 +933,9 @@ int main(int argc, char *argv[])
     nsparams.pci_latency = pci_latency;
     nsparams.eth_latency = eth_latency;
     nsparams.sync_delay = sync_period;
+    assert(sync_mode == SYNC_MODES || sync_mode == SYNC_BARRIER);
+    nsparams.sync_mode = sync_mode;
+
     if (nicsim_init(&nsparams, &di)) {
         return EXIT_FAILURE;
     }
@@ -1056,12 +1062,13 @@ int main(int argc, char *argv[])
             std::cerr << "warn: nicsim_sync failed (t=" << main_time << ")" <<
                 std::endl;
         }
+        nicsim_advance_epoch(&nsparams, main_time);
 
         do {
             poll_h2d(mmio);
             poll_n2d(rx);
         } while ((nsparams.sync_pci || nsparams.sync_eth) &&
-            netsim_next_timestamp(&nsparams) <= main_time && !exiting);
+            nicsim_next_timestamp(&nsparams) <= main_time && !exiting);
 
         /* falling edge */
         top->clk = !top->clk;

experiments/modes/simulators.py

@@ -22,6 +22,7 @@ class HostSim(Simulator):
     sleep = 0
     cpu_freq = '3GHz'
 
+    sync_mode = 0
     sync_period = 500
     pci_latency = 500
 
@@ -42,6 +43,7 @@ class NICSim(Simulator):
     network = None
     name = ''
 
+    sync_mode = 0
     sync_period = 500
     pci_latency = 500
     eth_latency = 500
@@ -51,10 +53,10 @@ class NICSim(Simulator):
         net.nics.append(self)
 
     def basic_run_cmd(self, env, name, extra=None):
-        cmd = '%s/%s %s %s %s 0 %d %d %d' % \
+        cmd = '%s/%s %s %s %s %d 0 %d %d %d' % \
             (env.repodir, name, env.nic_pci_path(self), env.nic_eth_path(self),
-                    env.nic_shm_path(self), self.sync_period, self.pci_latency,
-                    self.eth_latency)
+                    env.nic_shm_path(self), self.sync_mode, self.sync_period,
+                    self.pci_latency, self.eth_latency)
 
         if extra is not None:
             cmd += ' ' + extra
@@ -66,6 +68,7 @@ class NICSim(Simulator):
 class NetSim(Simulator):
     name = ''
     opt = ''
+    sync_mode = 0
     sync_period = 500
     eth_latency = 500
 
@@ -126,6 +129,7 @@ class QemuHost(HostSim):
             cmd += f'-device cosim-pci,chardev=cosimcd'
             if self.sync:
                 cmd += ',sync=on'
+                cmd += f',sync-mode={self.sync_mode}'
                 cmd += f',pci-latency={self.pci_latency}'
                 cmd += f',sync-period={self.sync_period}'
             else:
@@ -179,6 +183,7 @@ class Gem5Host(HostSim):
             cmd += f'--cosim-shm={env.nic_shm_path(nic)} '
             if cpu_type == 'TimingSimpleCPU':
                 cmd += '--cosim-sync '
+                cmd += f'--cosim-sync_mode={self.sync_mode} '
                 cmd += f'--cosim-pci-lat={self.pci_latency} '
                 cmd += f'--cosim-sync-int={self.sync_period} '
             if isinstance(nic, I40eNIC):
@@ -211,15 +216,15 @@ class I40eNIC(NICSim):
 class WireNet(NetSim):
     def run_cmd(self, env):
         assert len(self.nics) == 2
-        return '%s/net_wire/net_wire %s %s %d %d' % \
+        return '%s/net_wire/net_wire %s %s %d %d %d' % \
                 (env.repodir, env.nic_eth_path(self.nics[0]),
                         env.nic_eth_path(self.nics[1]),
-                        self.sync_period, self.eth_latency)
+                        self.sync_mode, self.sync_period, self.eth_latency)
 
 class SwitchNet(NetSim):
     def run_cmd(self, env):
         cmd = env.repodir + '/net_switch/net_switch'
-        cmd += f' -S {self.sync_period} -E {self.eth_latency}'
+        cmd += f' -m {self.sync_mode} -S {self.sync_period} -E {self.eth_latency}'
         for n in self.nics:
             cmd += ' -s ' + env.nic_eth_path(n)
         return cmd
@@ -315,4 +320,4 @@ def create_dctcp_hosts(e, num, name_prefix, net, nic_class, host_class,
 
         hosts.append(host)
 
-    return hosts
+    return hosts
\ No newline at end of file

libnicbm/nicbm.cc

@@ -391,21 +391,23 @@ int Runner::runMain(int argc, char *argv[])
     uint64_t sync_period = 100 * 1000ULL;
     uint64_t pci_latency = 500 * 1000ULL;
     uint64_t eth_latency = 500 * 1000ULL;
+    int sync_mode = SYNC_MODES;
 
-    if (argc < 4 && argc > 8) {
+    if (argc < 4 && argc > 9) {
         fprintf(stderr, "Usage: corundum_bm PCI-SOCKET ETH-SOCKET "
-                "SHM [START-TICK] [SYNC-PERIOD] [PCI-LATENCY] [ETH-LATENCY]\n");
+                "SHM [SYNC-MODE] [START-TICK] [SYNC-PERIOD] [PCI-LATENCY] [ETH-LATENCY]\n");
         return EXIT_FAILURE;
     }
     if (argc >= 5)
-        main_time = strtoull(argv[4], NULL, 0);
+        sync_mode = strtol(argv[4], NULL, 0);
     if (argc >= 6)
-        sync_period = strtoull(argv[5], NULL, 0) * 1000ULL;
+        main_time = strtoull(argv[5], NULL, 0);
     if (argc >= 7)
-        pci_latency = strtoull(argv[6], NULL, 0) * 1000ULL;
+        sync_period = strtoull(argv[6], NULL, 0) * 1000ULL;
     if (argc >= 8)
-        eth_latency = strtoull(argv[7], NULL, 0) * 1000ULL;
-
+        pci_latency = strtoull(argv[7], NULL, 0) * 1000ULL;
+    if (argc >= 9)
+        eth_latency = strtoull(argv[8], NULL, 0) * 1000ULL;
 
     signal(SIGINT, sigint_handler);
     signal(SIGUSR1, sigusr1_handler);
@@ -421,6 +423,9 @@ int Runner::runMain(int argc, char *argv[])
     nsparams.pci_latency = pci_latency;
     nsparams.eth_latency = eth_latency;
     nsparams.sync_delay = sync_period;
+    assert(sync_mode == SYNC_MODES || sync_mode == SYNC_BARRIER);
+    nsparams.sync_mode = sync_mode;
+
     if (nicsim_init(&nsparams, &dintro)) {
         return EXIT_FAILURE;
     }
@@ -433,6 +438,7 @@ int Runner::runMain(int argc, char *argv[])
         while (nicsim_sync(&nsparams, main_time)) {
             fprintf(stderr, "warn: nicsim_sync failed (t=%lu)\n", main_time);
         }
+        nicsim_advance_epoch(&nsparams, main_time);
 
         do {
             poll_h2d();
@@ -440,7 +446,7 @@ int Runner::runMain(int argc, char *argv[])
             event_trigger();
 
             if (is_sync) {
-                next_ts = netsim_next_timestamp(&nsparams);
+                next_ts = nicsim_next_timestamp(&nsparams);
                 if (next_ts > main_time + max_step)
                     next_ts = main_time + max_step;
             } else {
@@ -452,7 +458,7 @@ int Runner::runMain(int argc, char *argv[])
                 next_ts = ev_ts;
 
         } while (next_ts <= main_time && !exiting);
-        main_time = next_ts;
+        main_time = nicsim_advance_time(&nsparams, next_ts);
     }
 
     fprintf(stderr, "exit main_time: %lu\n", main_time);

net_switch/net_switch.cc

@@ -6,6 +6,7 @@
 #include <unistd.h>
 #include <vector>
 #include <unordered_map>
+#include <cassert>
 
 extern "C" {
 #include <netsim.h>
@@ -118,9 +119,10 @@ int main(int argc, char *argv[])
 {
     int c;
     int bad_option = 0;
+    int sync_mode = SYNC_MODES;
 
     // Parse command line argument
-    while ((c = getopt(argc, argv, "s:S:E:")) != -1 && !bad_option) {
+    while ((c = getopt(argc, argv, "s:S:E:m:")) != -1 && !bad_option) {
         switch (c) {
         case 's': {
             struct netsim_interface nsif;
@@ -141,6 +143,11 @@ int main(int argc, char *argv[])
             eth_latency = strtoull(optarg, NULL, 0) * 1000ULL;
             break;
 
+        case 'm':
+            sync_mode = strtol(optarg, NULL, 0);
+            assert(sync_mode == SYNC_MODES || sync_mode == SYNC_BARRIER);
+            break;
+
         default:
             fprintf(stderr, "unknown option %c\n", c);
             bad_option = 1;
@@ -161,11 +168,14 @@ int main(int argc, char *argv[])
     while (!exiting) {
         // Sync all interfaces
         for (auto &nsif : nsifs) {
-            if (netsim_n2d_sync(&nsif, cur_ts, eth_latency, sync_period) != 0) {
+            if (netsim_n2d_sync(&nsif, cur_ts, eth_latency,
+                        sync_period, sync_mode) != 0) {
                 fprintf(stderr, "netsim_n2d_sync failed\n");
                 abort();
             }
         }
+        netsim_advance_epoch(cur_ts, sync_period, sync_mode);
+
         // Switch packets
         uint64_t min_ts;
         do {
@@ -182,7 +192,7 @@ int main(int argc, char *argv[])
 
         // Update cur_ts
         if (min_ts < ULLONG_MAX) {
-            cur_ts = min_ts;
+            cur_ts = netsim_advance_time(min_ts, sync_period, sync_mode);
         }
     }
 

net_wire/net_wire.c

@@ -33,6 +33,7 @@
 #include <linux/if.h>
 #include <linux/if_tun.h>
 #include <pcap/pcap.h>
+#include <assert.h>
 
 #include <netsim.h>
 
@@ -107,9 +108,10 @@ int main(int argc, char *argv[])
     uint64_t ts_a, ts_b;
     int sync_a, sync_b;
     pcap_t *pc = NULL;
+    int sync_mode = SYNC_MODES;
 
-    if (argc < 3 && argc > 6) {
-        fprintf(stderr, "Usage: net_wire SOCKET-A SOCKET-B [SYNC-PERIOD] "
+    if (argc < 3 && argc > 7) {
+        fprintf(stderr, "Usage: net_wire SOCKET-A SOCKET-B [SYNC-MODE] [SYNC-PERIOD] "
                 "[ETH-LATENCY] [PCAP-FILE]\n");
         return EXIT_FAILURE;
     }
@@ -119,12 +121,15 @@ int main(int argc, char *argv[])
     signal(SIGUSR1, sigusr1_handler);
 
     if (argc >= 4)
-        sync_period = strtoull(argv[3], NULL, 0) * 1000ULL;
+        sync_mode = strtol(argv[3], NULL, 0);
 
     if (argc >= 5)
-        eth_latency = strtoull(argv[4], NULL, 0) * 1000ULL;
+        sync_period = strtoull(argv[4], NULL, 0) * 1000ULL;
 
-    if (argc >= 6) {
+    if (argc >= 6)
+        eth_latency = strtoull(argv[5], NULL, 0) * 1000ULL;
+
+    if (argc >= 7) {
         pc = pcap_open_dead_with_tstamp_precision(DLT_EN10MB, 65535,
                 PCAP_TSTAMP_PRECISION_NANO);
         if (pc == NULL) {
@@ -132,9 +137,11 @@ int main(int argc, char *argv[])
             return EXIT_FAILURE;
         }
 
-        dumpfile = pcap_dump_open(pc, argv[5]);
+        dumpfile = pcap_dump_open(pc, argv[6]);
     }
 
+    assert(sync_mode == SYNC_MODES || sync_mode == SYNC_BARRIER);
+
     sync_a = sync_b = 1;
     if (netsim_init(&nsif_a, argv[1], &sync_a) != 0) {
         return -1;
@@ -145,14 +152,15 @@ int main(int argc, char *argv[])
 
     printf("start polling\n");
     while (!exiting) {
-        if (netsim_n2d_sync(&nsif_a, cur_ts, eth_latency, sync_period) != 0) {
+        if (netsim_n2d_sync(&nsif_a, cur_ts, eth_latency, sync_period, sync_mode) != 0) {
             fprintf(stderr, "netsim_n2d_sync(nsif_a) failed\n");
             abort();
         }
-        if (netsim_n2d_sync(&nsif_b, cur_ts, eth_latency, sync_period) != 0) {
+        if (netsim_n2d_sync(&nsif_b, cur_ts, eth_latency, sync_period, sync_mode) != 0) {
             fprintf(stderr, "netsim_n2d_sync(nsif_a) failed\n");
             abort();
         }
+        netsim_advance_epoch(cur_ts, sync_period, sync_mode);
 
         do {
             move_pkt(&nsif_a, &nsif_b);
@@ -164,11 +172,12 @@ int main(int argc, char *argv[])
                 (sync_b && ts_b <= cur_ts)));
 
         if (sync_a && sync_b)
-            cur_ts = (ts_a <= ts_b ? ts_a : ts_b);
+            cur_ts = netsim_advance_time(ts_a <= ts_b ? ts_a : ts_b,
+                    sync_period, sync_mode);
         else if (sync_a)
-            cur_ts = ts_a;
+            cur_ts = netsim_advance_time(ts_a, sync_period, sync_mode);
         else if (sync_b)
-            cur_ts = ts_b;
+            cur_ts = netsim_advance_time(ts_b, sync_period, sync_mode);
     }
 
     if (dumpfile)

netsim_common/include/netsim.h

@@ -28,6 +28,9 @@
 #include <stdint.h>
 #include <cosim_eth_proto.h>
 
+#define SYNC_MODES 0
+#define SYNC_BARRIER 1
+
 struct netsim_interface {
     uint8_t *d2n_queue;
     size_t d2n_pos;
@@ -61,6 +64,8 @@ volatile union cosim_eth_proto_n2d *netsim_n2d_alloc(
         struct netsim_interface *nsif, uint64_t timestamp,
         uint64_t latency);
 int netsim_n2d_sync(struct netsim_interface *nsif, uint64_t timestamp,
-        uint64_t latency, uint64_t sync_delay);
+        uint64_t latency, uint64_t sync_delay, int sync_mode);
+void netsim_advance_epoch(uint64_t timestamp, uint64_t sync_delay, int sync_mode);
+uint64_t netsim_advance_time(uint64_t timestamp, uint64_t sync_delay, int sync_mode);
 
 #endif /* ndef COSIM_NETSIM_H_ */

netsim_common/netsim.c

@@ -31,6 +31,8 @@
 #include <netsim.h>
 #include "internal.h"
 
+static uint64_t current_epoch = 0;
+
 int netsim_init(struct netsim_interface *nsif,
         const char *eth_socket_path, int *sync_eth)
 {
@@ -137,17 +139,32 @@ volatile union cosim_eth_proto_n2d *netsim_n2d_alloc(
 }
 
 int netsim_n2d_sync(struct netsim_interface *nsif, uint64_t timestamp,
-        uint64_t latency, uint64_t sync_delay)
+        uint64_t latency, uint64_t sync_delay, int sync_mode)
 {
     volatile union cosim_eth_proto_n2d *msg;
     volatile struct cosim_eth_proto_n2d_sync *sync;
+    int do_sync;
 
     if (!nsif->sync)
         return 0;
 
-    if (nsif->n2d_timestamp != 0 &&
-            timestamp - nsif->n2d_timestamp < sync_delay)
+    switch (sync_mode) {
+    case SYNC_MODES:
+        do_sync = nsif->n2d_timestamp == 0 ||
+            timestamp - nsif->n2d_timestamp >= sync_delay;
+        break;
+    case SYNC_BARRIER:
+        do_sync = current_epoch == 0 ||
+            timestamp - current_epoch >= sync_delay;
+        break;
+    default:
+        fprintf(stderr, "unsupported sync mode=%u\n", sync_mode);
         return 0;
+    }
+
+    if (!do_sync) {
+        return 0;
+    }
 
     msg = netsim_n2d_alloc(nsif, timestamp, latency);
     if (msg == NULL)
@@ -159,3 +176,26 @@ int netsim_n2d_sync(struct netsim_interface *nsif, uint64_t timestamp,
 
     return 0;
 }
+
+void netsim_advance_epoch(uint64_t timestamp, uint64_t sync_delay, int sync_mode)
+{
+    if (sync_mode == SYNC_BARRIER) {
+        if (timestamp - current_epoch >= sync_delay) {
+            current_epoch = timestamp;
+        }
+    }
+}
+
+uint64_t netsim_advance_time(uint64_t timestamp, uint64_t sync_delay, int sync_mode)
+{
+    switch (sync_mode) {
+    case SYNC_MODES:
+        return timestamp;
+    case SYNC_BARRIER:
+        return timestamp < current_epoch + sync_delay ?
+            timestamp : current_epoch + sync_delay;
+    default:
+        fprintf(stderr, "unsupported sync mode=%u\n", sync_mode);
+        return timestamp;
+    }
+}

nicsim_common/include/nicsim.h

@@ -27,6 +27,9 @@
 #include <cosim_pcie_proto.h>
 #include <cosim_eth_proto.h>
 
+#define SYNC_MODES 0 // ModES style synchronization
+#define SYNC_BARRIER 1 // Global barrier style synchronization
+
 struct nicsim_params {
     const char *pci_socket_path;
     const char *eth_socket_path;
@@ -38,6 +41,7 @@ struct nicsim_params {
 
     int sync_pci;
     int sync_eth;
+    int sync_mode;
 };
 
 int nicsim_init(struct nicsim_params *params,
@@ -45,7 +49,9 @@ int nicsim_init(struct nicsim_params *params,
 void nicsim_cleanup(void);
 
 int nicsim_sync(struct nicsim_params *params, uint64_t timestamp);
-uint64_t netsim_next_timestamp(struct nicsim_params *params);
+void nicsim_advance_epoch(struct nicsim_params *params, uint64_t timestamp);
+uint64_t nicsim_advance_time(struct nicsim_params *params, uint64_t timestamp);
+uint64_t nicsim_next_timestamp(struct nicsim_params *params);
 
 volatile union cosim_pcie_proto_h2d *nicif_h2d_poll(
         struct nicsim_params *params, uint64_t timestamp);

nicsim_common/nicsim.c

@@ -66,6 +66,8 @@ static uint64_t pci_last_tx_time = 0;
 static uint64_t eth_last_rx_time = 0;
 static uint64_t eth_last_tx_time = 0;
 
+static uint64_t current_epoch = 0;
+
 static int shm_fd = -1;
 static int pci_cfd = -1;
 static int eth_cfd = -1;
@@ -266,35 +268,91 @@ int nicsim_sync(struct nicsim_params *params, uint64_t timestamp)
     volatile union cosim_eth_proto_d2n *d2n;
 
     /* sync PCI if necessary */
-    if (params->sync_pci && (pci_last_tx_time == 0 ||
-            timestamp - pci_last_tx_time >= params->sync_delay))
-    {
-        d2h = nicsim_d2h_alloc(params, timestamp);
-        if (d2h == NULL) {
-            ret = -1;
-        } else {
-            d2h->sync.own_type = COSIM_PCIE_PROTO_D2H_MSG_SYNC |
-                COSIM_PCIE_PROTO_D2H_OWN_HOST;
+    if (params->sync_pci) {
+        int sync;
+        switch (params->sync_mode) {
+        case SYNC_MODES:
+            sync = pci_last_tx_time == 0 ||
+                timestamp - pci_last_tx_time >= params->sync_delay;
+            break;
+        case SYNC_BARRIER:
+            sync = current_epoch == 0 ||
+                timestamp - current_epoch >= params->sync_delay;
+            break;
+        default:
+            fprintf(stderr, "unsupported sync mode=%u\n", params->sync_mode);
+            return ret;
+        }
+
+        if (sync)
+        {
+            d2h = nicsim_d2h_alloc(params, timestamp);
+            if (d2h == NULL) {
+                ret = -1;
+            } else {
+                d2h->sync.own_type = COSIM_PCIE_PROTO_D2H_MSG_SYNC |
+                    COSIM_PCIE_PROTO_D2H_OWN_HOST;
+            }
         }
     }
 
     /* sync Ethernet if necessary */
-    if (params->sync_eth && (eth_last_tx_time == 0 ||
-            timestamp - eth_last_tx_time >= params->sync_delay))
-    {
-        d2n = nicsim_d2n_alloc(params, timestamp);
-        if (d2n == NULL) {
-            ret = -1;
-        } else {
-            d2n->sync.own_type = COSIM_ETH_PROTO_D2N_MSG_SYNC |
-                COSIM_ETH_PROTO_D2N_OWN_NET;
+    if (params->sync_eth) {
+        int sync;
+        switch (params->sync_mode) {
+        case SYNC_MODES:
+            sync = eth_last_tx_time == 0 ||
+                timestamp - eth_last_tx_time >= params->sync_delay;
+            break;
+        case SYNC_BARRIER:
+            sync = current_epoch == 0 ||
+                timestamp - current_epoch >= params->sync_delay;
+            break;
+        default:
+            fprintf(stderr, "unsupported sync mode=%u\n", params->sync_mode);
+            return ret;
+        }
+
+        if (sync)
+        {
+            d2n = nicsim_d2n_alloc(params, timestamp);
+            if (d2n == NULL) {
+                ret = -1;
+            } else {
+                d2n->sync.own_type = COSIM_ETH_PROTO_D2N_MSG_SYNC |
+                    COSIM_ETH_PROTO_D2N_OWN_NET;
+            }
         }
     }
 
     return ret;
 }
 
-uint64_t netsim_next_timestamp(struct nicsim_params *params)
+void nicsim_advance_epoch(struct nicsim_params *params, uint64_t timestamp)
+{
+    if (params->sync_mode == SYNC_BARRIER) {
+        if ((params->sync_pci || params->sync_eth) &&
+                timestamp - current_epoch >= params->sync_delay) {
+            current_epoch = timestamp;
+        }
+    }
+}
+
+uint64_t nicsim_advance_time(struct nicsim_params *params, uint64_t timestamp)
+{
+    switch (params->sync_mode) {
+    case SYNC_MODES:
+        return timestamp;
+    case SYNC_BARRIER:
+        return timestamp < current_epoch + params->sync_delay ?
+            timestamp : current_epoch + params->sync_delay;
+    default:
+        fprintf(stderr, "unsupported sync mode=%u\n", params->sync_mode);
+        return timestamp;
+    }
+}
+
+uint64_t nicsim_next_timestamp(struct nicsim_params *params)
 {
     if (params->sync_pci && params->sync_eth) {
         return (pci_last_rx_time <= eth_last_rx_time ? pci_last_rx_time :