netsim/nicsim: add global barrier synchronization mode

libnicbm/nicbm.cc

@@ -391,21 +391,23 @@ int Runner::runMain(int argc, char *argv[])
     uint64_t sync_period = 100 * 1000ULL;
     uint64_t pci_latency = 500 * 1000ULL;
     uint64_t eth_latency = 500 * 1000ULL;
+    int sync_mode = SYNC_MODES;
 
-    if (argc < 4 && argc > 8) {
+    if (argc < 4 && argc > 9) {
         fprintf(stderr, "Usage: corundum_bm PCI-SOCKET ETH-SOCKET "
-                "SHM [START-TICK] [SYNC-PERIOD] [PCI-LATENCY] [ETH-LATENCY]\n");
+                "SHM [SYNC-MODE] [START-TICK] [SYNC-PERIOD] [PCI-LATENCY] [ETH-LATENCY]\n");
         return EXIT_FAILURE;
     }
     if (argc >= 5)
-        main_time = strtoull(argv[4], NULL, 0);
+        sync_mode = strtol(argv[4], NULL, 0);
     if (argc >= 6)
-        sync_period = strtoull(argv[5], NULL, 0) * 1000ULL;
+        main_time = strtoull(argv[5], NULL, 0);
     if (argc >= 7)
-        pci_latency = strtoull(argv[6], NULL, 0) * 1000ULL;
+        sync_period = strtoull(argv[6], NULL, 0) * 1000ULL;
     if (argc >= 8)
-        eth_latency = strtoull(argv[7], NULL, 0) * 1000ULL;
-
+        pci_latency = strtoull(argv[7], NULL, 0) * 1000ULL;
+    if (argc >= 9)
+        eth_latency = strtoull(argv[8], NULL, 0) * 1000ULL;
 
     signal(SIGINT, sigint_handler);
     signal(SIGUSR1, sigusr1_handler);
@@ -421,6 +423,9 @@ int Runner::runMain(int argc, char *argv[])
     nsparams.pci_latency = pci_latency;
     nsparams.eth_latency = eth_latency;
     nsparams.sync_delay = sync_period;
+    assert(sync_mode == SYNC_MODES || sync_mode == SYNC_BARRIER);
+    nsparams.sync_mode = sync_mode;
+
     if (nicsim_init(&nsparams, &dintro)) {
         return EXIT_FAILURE;
     }
@@ -433,6 +438,7 @@ int Runner::runMain(int argc, char *argv[])
         while (nicsim_sync(&nsparams, main_time)) {
             fprintf(stderr, "warn: nicsim_sync failed (t=%lu)\n", main_time);
         }
+        nicsim_advance_epoch(&nsparams, main_time);
 
         do {
             poll_h2d();
@@ -440,7 +446,7 @@ int Runner::runMain(int argc, char *argv[])
             event_trigger();
 
             if (is_sync) {
-                next_ts = netsim_next_timestamp(&nsparams);
+                next_ts = nicsim_next_timestamp(&nsparams);
                 if (next_ts > main_time + max_step)
                     next_ts = main_time + max_step;
             } else {
@@ -452,7 +458,7 @@ int Runner::runMain(int argc, char *argv[])
                 next_ts = ev_ts;
 
         } while (next_ts <= main_time && !exiting);
-        main_time = next_ts;
+        main_time = nicsim_advance_time(&nsparams, next_ts);
     }
 
     fprintf(stderr, "exit main_time: %lu\n", main_time);

net_switch/net_switch.cc

@@ -6,6 +6,7 @@
 #include <unistd.h>
 #include <vector>
 #include <unordered_map>
+#include <cassert>
 
 extern "C" {
 #include <netsim.h>
@@ -118,9 +119,10 @@ int main(int argc, char *argv[])
 {
     int c;
     int bad_option = 0;
+    int sync_mode = SYNC_MODES;
 
     // Parse command line argument
-    while ((c = getopt(argc, argv, "s:S:E:")) != -1 && !bad_option) {
+    while ((c = getopt(argc, argv, "s:S:E:m:")) != -1 && !bad_option) {
         switch (c) {
         case 's': {
             struct netsim_interface nsif;
@@ -141,6 +143,11 @@ int main(int argc, char *argv[])
             eth_latency = strtoull(optarg, NULL, 0) * 1000ULL;
             break;
 
+        case 'm':
+            sync_mode = strtol(optarg, NULL, 0);
+            assert(sync_mode == SYNC_MODES || sync_mode == SYNC_BARRIER);
+            break;
+
         default:
             fprintf(stderr, "unknown option %c\n", c);
             bad_option = 1;
@@ -161,11 +168,14 @@ int main(int argc, char *argv[])
     while (!exiting) {
         // Sync all interfaces
         for (auto &nsif : nsifs) {
-            if (netsim_n2d_sync(&nsif, cur_ts, eth_latency, sync_period) != 0) {
+            if (netsim_n2d_sync(&nsif, cur_ts, eth_latency,
+                        sync_period, sync_mode) != 0) {
                 fprintf(stderr, "netsim_n2d_sync failed\n");
                 abort();
             }
         }
+        netsim_advance_epoch(cur_ts, sync_period, sync_mode);
+
         // Switch packets
         uint64_t min_ts;
         do {
@@ -182,7 +192,7 @@ int main(int argc, char *argv[])
 
         // Update cur_ts
         if (min_ts < ULLONG_MAX) {
-            cur_ts = min_ts;
+            cur_ts = netsim_advance_time(min_ts, sync_period, sync_mode);
         }
     }
 

net_wire/net_wire.c

@@ -33,6 +33,7 @@
 #include <linux/if.h>
 #include <linux/if_tun.h>
 #include <pcap/pcap.h>
+#include <assert.h>
 
 #include <netsim.h>
 
@@ -107,9 +108,10 @@ int main(int argc, char *argv[])
     uint64_t ts_a, ts_b;
     int sync_a, sync_b;
     pcap_t *pc = NULL;
+    int sync_mode = SYNC_MODES;
 
-    if (argc < 3 && argc > 6) {
-        fprintf(stderr, "Usage: net_wire SOCKET-A SOCKET-B [SYNC-PERIOD] "
+    if (argc < 3 && argc > 7) {
+        fprintf(stderr, "Usage: net_wire SOCKET-A SOCKET-B [SYNC-MODE] [SYNC-PERIOD] "
                 "[ETH-LATENCY] [PCAP-FILE]\n");
         return EXIT_FAILURE;
     }
@@ -119,12 +121,15 @@ int main(int argc, char *argv[])
     signal(SIGUSR1, sigusr1_handler);
 
     if (argc >= 4)
-        sync_period = strtoull(argv[3], NULL, 0) * 1000ULL;
+        sync_mode = strtol(argv[3], NULL, 0);
 
     if (argc >= 5)
-        eth_latency = strtoull(argv[4], NULL, 0) * 1000ULL;
+        sync_period = strtoull(argv[4], NULL, 0) * 1000ULL;
 
-    if (argc >= 6) {
+    if (argc >= 6)
+        eth_latency = strtoull(argv[5], NULL, 0) * 1000ULL;
+
+    if (argc >= 7) {
         pc = pcap_open_dead_with_tstamp_precision(DLT_EN10MB, 65535,
                 PCAP_TSTAMP_PRECISION_NANO);
         if (pc == NULL) {
@@ -132,9 +137,11 @@ int main(int argc, char *argv[])
             return EXIT_FAILURE;
         }
 
-        dumpfile = pcap_dump_open(pc, argv[5]);
+        dumpfile = pcap_dump_open(pc, argv[6]);
     }
 
+    assert(sync_mode == SYNC_MODES || sync_mode == SYNC_BARRIER);
+
     sync_a = sync_b = 1;
     if (netsim_init(&nsif_a, argv[1], &sync_a) != 0) {
         return -1;
@@ -145,14 +152,15 @@ int main(int argc, char *argv[])
 
     printf("start polling\n");
     while (!exiting) {
-        if (netsim_n2d_sync(&nsif_a, cur_ts, eth_latency, sync_period) != 0) {
+        if (netsim_n2d_sync(&nsif_a, cur_ts, eth_latency, sync_period, sync_mode) != 0) {
             fprintf(stderr, "netsim_n2d_sync(nsif_a) failed\n");
             abort();
         }
-        if (netsim_n2d_sync(&nsif_b, cur_ts, eth_latency, sync_period) != 0) {
+        if (netsim_n2d_sync(&nsif_b, cur_ts, eth_latency, sync_period, sync_mode) != 0) {
             fprintf(stderr, "netsim_n2d_sync(nsif_a) failed\n");
             abort();
         }
+        netsim_advance_epoch(cur_ts, sync_period, sync_mode);
 
         do {
             move_pkt(&nsif_a, &nsif_b);
@@ -164,11 +172,12 @@ int main(int argc, char *argv[])
                 (sync_b && ts_b <= cur_ts)));
 
         if (sync_a && sync_b)
-            cur_ts = (ts_a <= ts_b ? ts_a : ts_b);
+            cur_ts = netsim_advance_time(ts_a <= ts_b ? ts_a : ts_b,
+                    sync_period, sync_mode);
         else if (sync_a)
-            cur_ts = ts_a;
+            cur_ts = netsim_advance_time(ts_a, sync_period, sync_mode);
         else if (sync_b)
-            cur_ts = ts_b;
+            cur_ts = netsim_advance_time(ts_b, sync_period, sync_mode);
     }
 
     if (dumpfile)

netsim_common/include/netsim.h

@@ -28,6 +28,9 @@
 #include <stdint.h>
 #include <cosim_eth_proto.h>
 
+#define SYNC_MODES 0
+#define SYNC_BARRIER 1
+
 struct netsim_interface {
     uint8_t *d2n_queue;
     size_t d2n_pos;
@@ -61,6 +64,8 @@ volatile union cosim_eth_proto_n2d *netsim_n2d_alloc(
         struct netsim_interface *nsif, uint64_t timestamp,
         uint64_t latency);
 int netsim_n2d_sync(struct netsim_interface *nsif, uint64_t timestamp,
-        uint64_t latency, uint64_t sync_delay);
+        uint64_t latency, uint64_t sync_delay, int sync_mode);
+void netsim_advance_epoch(uint64_t timestamp, uint64_t sync_delay, int sync_mode);
+uint64_t netsim_advance_time(uint64_t timestamp, uint64_t sync_delay, int sync_mode);
 
 #endif /* ndef COSIM_NETSIM_H_ */

netsim_common/netsim.c

@@ -31,6 +31,8 @@
 #include <netsim.h>
 #include "internal.h"
 
+static uint64_t current_epoch = 0;
+
 int netsim_init(struct netsim_interface *nsif,
         const char *eth_socket_path, int *sync_eth)
 {
@@ -137,17 +139,32 @@ volatile union cosim_eth_proto_n2d *netsim_n2d_alloc(
 }
 
 int netsim_n2d_sync(struct netsim_interface *nsif, uint64_t timestamp,
-        uint64_t latency, uint64_t sync_delay)
+        uint64_t latency, uint64_t sync_delay, int sync_mode)
 {
     volatile union cosim_eth_proto_n2d *msg;
     volatile struct cosim_eth_proto_n2d_sync *sync;
+    int do_sync;
 
     if (!nsif->sync)
         return 0;
 
-    if (nsif->n2d_timestamp != 0 &&
-            timestamp - nsif->n2d_timestamp < sync_delay)
+    switch (sync_mode) {
+    case SYNC_MODES:
+        do_sync = nsif->n2d_timestamp == 0 ||
+            timestamp - nsif->n2d_timestamp >= sync_delay;
+        break;
+    case SYNC_BARRIER:
+        do_sync = current_epoch == 0 ||
+            timestamp - current_epoch >= sync_delay;
+        break;
+    default:
+        fprintf(stderr, "unsupported sync mode=%u\n", sync_mode);
         return 0;
+    }
+
+    if (!do_sync) {
+        return 0;
+    }
 
     msg = netsim_n2d_alloc(nsif, timestamp, latency);
     if (msg == NULL)
@@ -159,3 +176,26 @@ int netsim_n2d_sync(struct netsim_interface *nsif, uint64_t timestamp,
 
     return 0;
 }
+
+void netsim_advance_epoch(uint64_t timestamp, uint64_t sync_delay, int sync_mode)
+{
+    if (sync_mode == SYNC_BARRIER) {
+        if (timestamp - current_epoch >= sync_delay) {
+            current_epoch = timestamp;
+        }
+    }
+}
+
+uint64_t netsim_advance_time(uint64_t timestamp, uint64_t sync_delay, int sync_mode)
+{
+    switch (sync_mode) {
+    case SYNC_MODES:
+        return timestamp;
+    case SYNC_BARRIER:
+        return timestamp < current_epoch + sync_delay ?
+            timestamp : current_epoch + sync_delay;
+    default:
+        fprintf(stderr, "unsupported sync mode=%u\n", sync_mode);
+        return timestamp;
+    }
+}

nicsim_common/include/nicsim.h

@@ -27,6 +27,9 @@
 #include <cosim_pcie_proto.h>
 #include <cosim_eth_proto.h>
 
+#define SYNC_MODES 0 // ModES style synchronization
+#define SYNC_BARRIER 1 // Global barrier style synchronization
+
 struct nicsim_params {
     const char *pci_socket_path;
     const char *eth_socket_path;
@@ -38,6 +41,7 @@ struct nicsim_params {
 
     int sync_pci;
     int sync_eth;
+    int sync_mode;
 };
 
 int nicsim_init(struct nicsim_params *params,
@@ -45,7 +49,9 @@ int nicsim_init(struct nicsim_params *params,
 void nicsim_cleanup(void);
 
 int nicsim_sync(struct nicsim_params *params, uint64_t timestamp);
-uint64_t netsim_next_timestamp(struct nicsim_params *params);
+void nicsim_advance_epoch(struct nicsim_params *params, uint64_t timestamp);
+uint64_t nicsim_advance_time(struct nicsim_params *params, uint64_t timestamp);
+uint64_t nicsim_next_timestamp(struct nicsim_params *params);
 
 volatile union cosim_pcie_proto_h2d *nicif_h2d_poll(
         struct nicsim_params *params, uint64_t timestamp);

nicsim_common/nicsim.c

@@ -66,6 +66,8 @@ static uint64_t pci_last_tx_time = 0;
 static uint64_t eth_last_rx_time = 0;
 static uint64_t eth_last_tx_time = 0;
 
+static uint64_t current_epoch = 0;
+
 static int shm_fd = -1;
 static int pci_cfd = -1;
 static int eth_cfd = -1;
@@ -266,35 +268,91 @@ int nicsim_sync(struct nicsim_params *params, uint64_t timestamp)
     volatile union cosim_eth_proto_d2n *d2n;
 
     /* sync PCI if necessary */
-    if (params->sync_pci && (pci_last_tx_time == 0 ||
-            timestamp - pci_last_tx_time >= params->sync_delay))
-    {
-        d2h = nicsim_d2h_alloc(params, timestamp);
-        if (d2h == NULL) {
-            ret = -1;
-        } else {
-            d2h->sync.own_type = COSIM_PCIE_PROTO_D2H_MSG_SYNC |
-                COSIM_PCIE_PROTO_D2H_OWN_HOST;
+    if (params->sync_pci) {
+        int sync;
+        switch (params->sync_mode) {
+        case SYNC_MODES:
+            sync = pci_last_tx_time == 0 ||
+                timestamp - pci_last_tx_time >= params->sync_delay;
+            break;
+        case SYNC_BARRIER:
+            sync = current_epoch == 0 ||
+                timestamp - current_epoch >= params->sync_delay;
+            break;
+        default:
+            fprintf(stderr, "unsupported sync mode=%u\n", params->sync_mode);
+            return ret;
+        }
+
+        if (sync)
+        {
+            d2h = nicsim_d2h_alloc(params, timestamp);
+            if (d2h == NULL) {
+                ret = -1;
+            } else {
+                d2h->sync.own_type = COSIM_PCIE_PROTO_D2H_MSG_SYNC |
+                    COSIM_PCIE_PROTO_D2H_OWN_HOST;
+            }
         }
     }
 
     /* sync Ethernet if necessary */
-    if (params->sync_eth && (eth_last_tx_time == 0 ||
-            timestamp - eth_last_tx_time >= params->sync_delay))
-    {
-        d2n = nicsim_d2n_alloc(params, timestamp);
-        if (d2n == NULL) {
-            ret = -1;
-        } else {
-            d2n->sync.own_type = COSIM_ETH_PROTO_D2N_MSG_SYNC |
-                COSIM_ETH_PROTO_D2N_OWN_NET;
+    if (params->sync_eth) {
+        int sync;
+        switch (params->sync_mode) {
+        case SYNC_MODES:
+            sync = eth_last_tx_time == 0 ||
+                timestamp - eth_last_tx_time >= params->sync_delay;
+            break;
+        case SYNC_BARRIER:
+            sync = current_epoch == 0 ||
+                timestamp - current_epoch >= params->sync_delay;
+            break;
+        default:
+            fprintf(stderr, "unsupported sync mode=%u\n", params->sync_mode);
+            return ret;
+        }
+
+        if (sync)
+        {
+            d2n = nicsim_d2n_alloc(params, timestamp);
+            if (d2n == NULL) {
+                ret = -1;
+            } else {
+                d2n->sync.own_type = COSIM_ETH_PROTO_D2N_MSG_SYNC |
+                    COSIM_ETH_PROTO_D2N_OWN_NET;
+            }
         }
     }
 
     return ret;
 }
 
-uint64_t netsim_next_timestamp(struct nicsim_params *params)
+void nicsim_advance_epoch(struct nicsim_params *params, uint64_t timestamp)
+{
+    if (params->sync_mode == SYNC_BARRIER) {
+        if ((params->sync_pci || params->sync_eth) &&
+                timestamp - current_epoch >= params->sync_delay) {
+            current_epoch = timestamp;
+        }
+    }
+}
+
+uint64_t nicsim_advance_time(struct nicsim_params *params, uint64_t timestamp)
+{
+    switch (params->sync_mode) {
+    case SYNC_MODES:
+        return timestamp;
+    case SYNC_BARRIER:
+        return timestamp < current_epoch + params->sync_delay ?
+            timestamp : current_epoch + params->sync_delay;
+    default:
+        fprintf(stderr, "unsupported sync mode=%u\n", params->sync_mode);
+        return timestamp;
+    }
+}
+
+uint64_t nicsim_next_timestamp(struct nicsim_params *params)
 {
     if (params->sync_pci && params->sync_eth) {
         return (pci_last_rx_time <= eth_last_rx_time ? pci_last_rx_time :