lib/nicif: move state into struct from global variables

To support use-cases with multiple NIC interfaces.

lib/simbricks/nicbm/nicbm.cc

@@ -84,7 +84,7 @@ static void sigusr2_handler(int dummy) {
 
 volatile union SimbricksProtoPcieD2H *Runner::D2HAlloc() {
   volatile union SimbricksProtoPcieD2H *msg;
-  while ((msg = SimbricksNicIfD2HAlloc(&nsparams_, main_time)) == NULL) {
+  while ((msg = SimbricksNicIfD2HAlloc(&nicif_, main_time)) == NULL) {
     fprintf(stderr, "D2HAlloc: no entry available\n");
   }
   return msg;
@@ -92,7 +92,7 @@ volatile union SimbricksProtoPcieD2H *Runner::D2HAlloc() {
 
 volatile union SimbricksProtoNetD2N *Runner::D2NAlloc() {
   volatile union SimbricksProtoNetD2N *msg;
-  while ((msg = SimbricksNicIfD2NAlloc(&nsparams_, main_time)) == NULL) {
+  while ((msg = SimbricksNicIfD2NAlloc(&nicif_, main_time)) == NULL) {
     fprintf(stderr, "D2NAlloc: no entry available\n");
   }
   return msg;
@@ -318,7 +318,7 @@ void Runner::EthSend(const void *data, size_t len) {
 
 void Runner::PollH2D() {
   volatile union SimbricksProtoPcieH2D *msg =
-      SimbricksNicIfH2DPoll(&nsparams_, main_time);
+      SimbricksNicIfH2DPoll(&nicif_, main_time);
   uint8_t type;
 
 #ifdef STAT_NICBM
@@ -373,13 +373,13 @@ void Runner::PollH2D() {
       fprintf(stderr, "poll_h2d: unsupported type=%u\n", type);
   }
 
-  SimbricksNicIfH2DDone(msg);
-  SimbricksNicIfH2DNext();
+  SimbricksNicIfH2DDone(&nicif_, msg);
+  SimbricksNicIfH2DNext(&nicif_);
 }
 
 void Runner::PollN2D() {
   volatile union SimbricksProtoNetN2D *msg =
-      SimbricksNicIfN2DPoll(&nsparams_, main_time);
+      SimbricksNicIfN2DPoll(&nicif_, main_time);
   uint8_t t;
 
 #ifdef STAT_NICBM
@@ -418,8 +418,8 @@ void Runner::PollN2D() {
       fprintf(stderr, "poll_n2d: unsupported type=%u", t);
   }
 
-  SimbricksNicIfN2DDone(msg);
-  SimbricksNicIfN2DNext();
+  SimbricksNicIfN2DDone(&nicif_, msg);
+  SimbricksNicIfN2DNext(&nicif_);
 }
 
 uint64_t Runner::TimePs() const {
@@ -500,31 +500,32 @@ int Runner::RunMain(int argc, char *argv[]) {
   memset(&dintro_, 0, sizeof(dintro_));
   dev_.SetupIntro(dintro_);
 
-  nsparams_.sync_pci = 1;
-  nsparams_.sync_eth = 1;
-  nsparams_.pci_socket_path = argv[1];
-  nsparams_.eth_socket_path = argv[2];
-  nsparams_.shm_path = argv[3];
-  nsparams_.pci_latency = pci_latency;
-  nsparams_.eth_latency = eth_latency;
-  nsparams_.sync_delay = sync_period;
+  struct SimbricksNicIfParams nsparams;
+  nsparams.sync_pci = 1;
+  nsparams.sync_eth = 1;
+  nsparams.pci_socket_path = argv[1];
+  nsparams.eth_socket_path = argv[2];
+  nsparams.shm_path = argv[3];
+  nsparams.pci_latency = pci_latency;
+  nsparams.eth_latency = eth_latency;
+  nsparams.sync_delay = sync_period;
   assert(sync_mode == SIMBRICKS_PROTO_SYNC_SIMBRICKS ||
          sync_mode == SIMBRICKS_PROTO_SYNC_BARRIER);
-  nsparams_.sync_mode = sync_mode;
+  nsparams.sync_mode = sync_mode;
 
-  if (SimbricksNicIfInit(&nsparams_, &dintro_)) {
+  if (SimbricksNicIfInit(&nicif_, &nsparams, &dintro_)) {
     return EXIT_FAILURE;
   }
-  fprintf(stderr, "sync_pci=%d sync_eth=%d\n", nsparams_.sync_pci,
-          nsparams_.sync_eth);
+  fprintf(stderr, "sync_pci=%d sync_eth=%d\n", nsparams.sync_pci,
+          nsparams.sync_eth);
 
-  bool is_sync = nsparams_.sync_pci || nsparams_.sync_eth;
+  bool is_sync = nsparams.sync_pci || nsparams.sync_eth;
 
   while (!exiting) {
-    while (SimbricksNicIfSync(&nsparams_, main_time)) {
+    while (SimbricksNicIfSync(&nicif_, main_time)) {
       fprintf(stderr, "warn: SimbricksNicIfSync failed (t=%lu)\n", main_time);
     }
-    SimbricksNicIfAdvanceEpoch(&nsparams_, main_time);
+    SimbricksNicIfAdvanceEpoch(&nicif_, main_time);
 
     do {
       PollH2D();
@@ -532,7 +533,7 @@ int Runner::RunMain(int argc, char *argv[]) {
       EventTrigger();
 
       if (is_sync) {
-        next_ts = SimbricksNicIfNextTimestamp(&nsparams_);
+        next_ts = SimbricksNicIfNextTimestamp(&nicif_);
         if (next_ts > main_time + max_step)
           next_ts = main_time + max_step;
       } else {
@@ -543,7 +544,7 @@ int Runner::RunMain(int argc, char *argv[]) {
       if (EventNext(ev_ts) && ev_ts < next_ts)
         next_ts = ev_ts;
     } while (next_ts <= main_time && !exiting);
-    main_time = SimbricksNicIfAdvanceTime(&nsparams_, next_ts);
+    main_time = SimbricksNicIfAdvanceTime(&nicif_, next_ts);
   }
 
   fprintf(stderr, "exit main_time: %lu\n", main_time);
@@ -587,7 +588,7 @@ int Runner::RunMain(int argc, char *argv[]) {
       (double)(s_h2d_poll_sync + s_n2d_poll_sync) / (s_h2d_poll_suc + s_n2d_poll_suc));
 #endif
 
-  SimbricksNicIfCleanup();
+  SimbricksNicIfCleanup(&nicif_);
   return 0;
 }
 

lib/simbricks/nicbm/nicbm.h

@@ -122,7 +122,7 @@ class Runner {
   std::deque<DMAOp *> dma_queue_;
   size_t dma_pending_;
   uint64_t mac_addr_;
-  struct SimbricksNicIfParams nsparams_;
+  struct SimbricksNicIf nicif_;
   struct SimbricksProtoPcieDevIntro dintro_;
 
   volatile union SimbricksProtoPcieD2H *D2HAlloc();

lib/simbricks/nicif/nicif.c

@@ -47,46 +47,26 @@
 #define N2D_ELEN (9024 + 64)
 #define N2D_ENUM 8192
 
-static uint8_t *d2h_queue;
-static size_t d2h_pos;
-static size_t d2h_off; /* offset in shm region */
 
-static uint8_t *h2d_queue;
-static size_t h2d_pos;
-static size_t h2d_off; /* offset in shm region */
 
-static uint8_t *d2n_queue;
-static size_t d2n_pos;
-static size_t d2n_off; /* offset in shm region */
 
-static uint8_t *n2d_queue;
-static size_t n2d_pos;
-static size_t n2d_off; /* offset in shm region */
 
-static uint64_t pci_last_rx_time = 0;
-static uint64_t pci_last_tx_time = 0;
-static uint64_t eth_last_rx_time = 0;
-static uint64_t eth_last_tx_time = 0;
 
-static uint64_t current_epoch = 0;
-
-static int shm_fd = -1;
-static int pci_cfd = -1;
-static int eth_cfd = -1;
-
-static int accept_pci(struct SimbricksProtoPcieDevIntro *di, int pci_lfd,
+static int accept_pci(struct SimbricksNicIf *nicif,
+                      struct SimbricksProtoPcieDevIntro *di,
+                      int pci_lfd,
                       int *sync_pci) {
-  if ((pci_cfd = accept(pci_lfd, NULL, NULL)) < 0) {
+  if ((nicif->pci_cfd = accept(pci_lfd, NULL, NULL)) < 0) {
     return -1;
   }
   close(pci_lfd);
   printf("pci connection accepted\n");
 
-  di->d2h_offset = d2h_off;
+  di->d2h_offset = nicif->d2h_off;
   di->d2h_elen = D2H_ELEN;
   di->d2h_nentries = D2H_ENUM;
 
-  di->h2d_offset = h2d_off;
+  di->h2d_offset = nicif->h2d_off;
   di->h2d_elen = H2D_ELEN;
   di->h2d_nentries = H2D_ENUM;
 
@@ -95,17 +75,19 @@ static int accept_pci(struct SimbricksProtoPcieDevIntro *di, int pci_lfd,
   else
     di->flags &= ~((uint64_t)SIMBRICKS_PROTO_PCIE_FLAGS_DI_SYNC);
 
-  if (uxsocket_send(pci_cfd, di, sizeof(*di), shm_fd)) {
+  if (uxsocket_send(nicif->pci_cfd, di, sizeof(*di), nicif->shm_fd)) {
     return -1;
   }
   printf("pci intro sent\n");
   return 0;
 }
 
-static int accept_eth(int eth_lfd, int *sync_eth) {
+static int accept_eth(struct SimbricksNicIf *nicif,
+                      int eth_lfd,
+                      int *sync_eth) {
   struct SimbricksProtoNetDevIntro di;
 
-  if ((eth_cfd = accept(eth_lfd, NULL, NULL)) < 0) {
+  if ((nicif->eth_cfd = accept(eth_lfd, NULL, NULL)) < 0) {
     return -1;
   }
   close(eth_lfd);
@@ -116,22 +98,23 @@ static int accept_eth(int eth_lfd, int *sync_eth) {
   if (*sync_eth)
     di.flags |= SIMBRICKS_PROTO_NET_FLAGS_DI_SYNC;
 
-  di.d2n_offset = d2n_off;
+  di.d2n_offset = nicif->d2n_off;
   di.d2n_elen = D2N_ELEN;
   di.d2n_nentries = D2N_ENUM;
 
-  di.n2d_offset = n2d_off;
+  di.n2d_offset = nicif->n2d_off;
   di.n2d_elen = N2D_ELEN;
   di.n2d_nentries = N2D_ENUM;
 
-  if (uxsocket_send(eth_cfd, &di, sizeof(di), shm_fd)) {
+  if (uxsocket_send(nicif->eth_cfd, &di, sizeof(di), nicif->shm_fd)) {
     return -1;
   }
   printf("eth intro sent\n");
   return 0;
 }
 
-static int accept_conns(struct SimbricksProtoPcieDevIntro *di, int pci_lfd,
+static int accept_conns(struct SimbricksNicIf *nicif,
+                        struct SimbricksProtoPcieDevIntro *di, int pci_lfd,
                         int *sync_pci, int eth_lfd, int *sync_eth) {
   struct pollfd pfds[2];
   int await_pci = pci_lfd != -1;
@@ -153,23 +136,23 @@ static int accept_conns(struct SimbricksProtoPcieDevIntro *di, int pci_lfd,
       }
 
       if (pfds[0].revents) {
-        if (accept_pci(di, pci_lfd, sync_pci) != 0)
+        if (accept_pci(nicif, di, pci_lfd, sync_pci) != 0)
           return -1;
         await_pci = 0;
       }
       if (pfds[1].revents) {
-        if (accept_eth(eth_lfd, sync_eth) != 0)
+        if (accept_eth(nicif, eth_lfd, sync_eth) != 0)
           return -1;
         await_eth = 0;
       }
     } else if (await_pci) {
       /* waiting just on pci */
-      if (accept_pci(di, pci_lfd, sync_pci) != 0)
+      if (accept_pci(nicif, di, pci_lfd, sync_pci) != 0)
         return -1;
       await_pci = 0;
     } else {
       /* waiting just on ethernet */
-      if (accept_eth(eth_lfd, sync_eth) != 0)
+      if (accept_eth(nicif, eth_lfd, sync_eth) != 0)
         return -1;
       await_eth = 0;
     }
@@ -178,30 +161,36 @@ static int accept_conns(struct SimbricksProtoPcieDevIntro *di, int pci_lfd,
   return 0;
 }
 
-int SimbricksNicIfInit(struct SimbricksNicIfParams *params,
+int SimbricksNicIfInit(struct SimbricksNicIf *nicif,
+                       struct SimbricksNicIfParams *params,
                        struct SimbricksProtoPcieDevIntro *di) {
   int pci_lfd = -1, eth_lfd = -1;
   void *shmptr;
   size_t shm_size;
 
+  /* initialize nicif struct */
+  memset(nicif, 0, sizeof(*nicif));
+  nicif->params = *params;
+  nicif->pci_cfd = nicif->eth_cfd = -1;
+
   /* ready in memory queues */
   shm_size = (uint64_t)D2H_ELEN * D2H_ENUM + (uint64_t)H2D_ELEN * H2D_ENUM +
              (uint64_t)D2N_ELEN * D2N_ENUM + (uint64_t)N2D_ELEN * N2D_ENUM;
-  if ((shm_fd = shm_create(params->shm_path, shm_size, &shmptr)) < 0) {
+  if ((nicif->shm_fd = shm_create(params->shm_path, shm_size, &shmptr)) < 0) {
     return -1;
   }
 
-  d2h_off = 0;
-  h2d_off = d2h_off + (uint64_t)D2H_ELEN * D2H_ENUM;
-  d2n_off = h2d_off + (uint64_t)H2D_ELEN * H2D_ENUM;
-  n2d_off = d2n_off + (uint64_t)D2N_ELEN * D2N_ENUM;
+  nicif->d2h_off = 0;
+  nicif->h2d_off = nicif->d2h_off + (uint64_t)D2H_ELEN * D2H_ENUM;
+  nicif->d2n_off = nicif->h2d_off + (uint64_t)H2D_ELEN * H2D_ENUM;
+  nicif->n2d_off = nicif->d2n_off + (uint64_t)D2N_ELEN * D2N_ENUM;
 
-  d2h_queue = (uint8_t *)shmptr + d2h_off;
-  h2d_queue = (uint8_t *)shmptr + h2d_off;
-  d2n_queue = (uint8_t *)shmptr + d2n_off;
-  n2d_queue = (uint8_t *)shmptr + n2d_off;
+  nicif->d2h_queue = (uint8_t *)shmptr + nicif->d2h_off;
+  nicif->h2d_queue = (uint8_t *)shmptr + nicif->h2d_off;
+  nicif->d2n_queue = (uint8_t *)shmptr + nicif->d2n_off;
+  nicif->n2d_queue = (uint8_t *)shmptr + nicif->n2d_off;
 
-  d2h_pos = h2d_pos = d2n_pos = n2d_pos = 0;
+  nicif->d2h_pos = nicif->h2d_pos = nicif->d2n_pos = nicif->n2d_pos = 0;
 
   /* get listening sockets ready */
   if (params->pci_socket_path != NULL) {
@@ -216,7 +205,7 @@ int SimbricksNicIfInit(struct SimbricksNicIfParams *params,
   }
 
   /* accept connection fds */
-  if (accept_conns(di, pci_lfd, &params->sync_pci, eth_lfd,
+  if (accept_conns(nicif, di, pci_lfd, &params->sync_pci, eth_lfd,
                    &params->sync_eth) != 0) {
     return -1;
   }
@@ -224,7 +213,7 @@ int SimbricksNicIfInit(struct SimbricksNicIfParams *params,
   /* receive introductions from other end */
   if (params->pci_socket_path != NULL) {
     struct SimbricksProtoPcieHostIntro hi;
-    if (recv(pci_cfd, &hi, sizeof(hi), 0) != sizeof(hi)) {
+    if (recv(nicif->pci_cfd, &hi, sizeof(hi), 0) != sizeof(hi)) {
       return -1;
     }
     if ((hi.flags & SIMBRICKS_PROTO_PCIE_FLAGS_HI_SYNC) == 0)
@@ -233,7 +222,7 @@ int SimbricksNicIfInit(struct SimbricksNicIfParams *params,
   }
   if (params->eth_socket_path != NULL) {
     struct SimbricksProtoNetNetIntro ni;
-    if (recv(eth_cfd, &ni, sizeof(ni), 0) != sizeof(ni)) {
+    if (recv(nicif->eth_cfd, &ni, sizeof(ni), 0) != sizeof(ni)) {
       return -1;
     }
     if ((ni.flags & SIMBRICKS_PROTO_NET_FLAGS_NI_SYNC) == 0)
@@ -241,20 +230,23 @@ int SimbricksNicIfInit(struct SimbricksNicIfParams *params,
     printf("eth net info received\n");
   }
 
+  nicif->params.sync_pci = params->sync_pci;
+  nicif->params.sync_eth = params->sync_eth;
   return 0;
 }
 
-void SimbricksNicIfCleanup(void) {
-  close(pci_cfd);
-  close(eth_cfd);
+void SimbricksNicIfCleanup(struct SimbricksNicIf *nicif) {
+  close(nicif->pci_cfd);
+  close(nicif->eth_cfd);
 }
 
 /******************************************************************************/
 /* Sync */
 
-int SimbricksNicIfSync(struct SimbricksNicIfParams *params,
+int SimbricksNicIfSync(struct SimbricksNicIf *nicif,
                        uint64_t timestamp) {
   int ret = 0;
+  struct SimbricksNicIfParams *params = &nicif->params;
   volatile union SimbricksProtoPcieD2H *d2h;
   volatile union SimbricksProtoNetD2N *d2n;
 
@@ -263,12 +255,12 @@ int SimbricksNicIfSync(struct SimbricksNicIfParams *params,
     int sync;
     switch (params->sync_mode) {
       case SIMBRICKS_PROTO_SYNC_SIMBRICKS:
-        sync = pci_last_tx_time == 0 ||
-               timestamp - pci_last_tx_time >= params->sync_delay;
+        sync = nicif->pci_last_tx_time == 0 ||
+               timestamp - nicif->pci_last_tx_time >= params->sync_delay;
         break;
       case SIMBRICKS_PROTO_SYNC_BARRIER:
-        sync = current_epoch == 0 ||
-               timestamp - current_epoch >= params->sync_delay;
+        sync = nicif->current_epoch == 0 ||
+               timestamp - nicif->current_epoch >= params->sync_delay;
         break;
       default:
         fprintf(stderr, "unsupported sync mode=%u\n", params->sync_mode);
@@ -276,7 +268,7 @@ int SimbricksNicIfSync(struct SimbricksNicIfParams *params,
     }
 
     if (sync) {
-      d2h = SimbricksNicIfD2HAlloc(params, timestamp);
+      d2h = SimbricksNicIfD2HAlloc(nicif, timestamp);
       if (d2h == NULL) {
         ret = -1;
       } else {
@@ -291,12 +283,12 @@ int SimbricksNicIfSync(struct SimbricksNicIfParams *params,
     int sync;
     switch (params->sync_mode) {
       case SIMBRICKS_PROTO_SYNC_SIMBRICKS:
-        sync = eth_last_tx_time == 0 ||
-               timestamp - eth_last_tx_time >= params->sync_delay;
+        sync = nicif->eth_last_tx_time == 0 ||
+               timestamp - nicif->eth_last_tx_time >= params->sync_delay;
         break;
       case SIMBRICKS_PROTO_SYNC_BARRIER:
-        sync = current_epoch == 0 ||
-               timestamp - current_epoch >= params->sync_delay;
+        sync = nicif->current_epoch == 0 ||
+               timestamp - nicif->current_epoch >= params->sync_delay;
         break;
       default:
         fprintf(stderr, "unsupported sync mode=%u\n", params->sync_mode);
@@ -304,7 +296,7 @@ int SimbricksNicIfSync(struct SimbricksNicIfParams *params,
     }
 
     if (sync) {
-      d2n = SimbricksNicIfD2NAlloc(params, timestamp);
+      d2n = SimbricksNicIfD2NAlloc(nicif, timestamp);
       if (d2n == NULL) {
         ret = -1;
       } else {
@@ -317,39 +309,43 @@ int SimbricksNicIfSync(struct SimbricksNicIfParams *params,
   return ret;
 }
 
-void SimbricksNicIfAdvanceEpoch(struct SimbricksNicIfParams *params,
+void SimbricksNicIfAdvanceEpoch(struct SimbricksNicIf *nicif,
                                 uint64_t timestamp) {
+  struct SimbricksNicIfParams *params = &nicif->params;
   if (params->sync_mode == SIMBRICKS_PROTO_SYNC_BARRIER) {
     if ((params->sync_pci || params->sync_eth) &&
-        timestamp - current_epoch >= params->sync_delay) {
-      current_epoch = timestamp;
+        timestamp - nicif->current_epoch >= params->sync_delay) {
+      nicif->current_epoch = timestamp;
     }
   }
 }
 
-uint64_t SimbricksNicIfAdvanceTime(struct SimbricksNicIfParams *params,
+uint64_t SimbricksNicIfAdvanceTime(struct SimbricksNicIf *nicif,
                                    uint64_t timestamp) {
+  struct SimbricksNicIfParams *params = &nicif->params;
   switch (params->sync_mode) {
     case SIMBRICKS_PROTO_SYNC_SIMBRICKS:
       return timestamp;
     case SIMBRICKS_PROTO_SYNC_BARRIER:
-      return timestamp < current_epoch + params->sync_delay
+      return timestamp < nicif->current_epoch + params->sync_delay
                  ? timestamp
-                 : current_epoch + params->sync_delay;
+                 : nicif->current_epoch + params->sync_delay;
     default:
       fprintf(stderr, "unsupported sync mode=%u\n", params->sync_mode);
       return timestamp;
   }
 }
 
-uint64_t SimbricksNicIfNextTimestamp(struct SimbricksNicIfParams *params) {
+uint64_t SimbricksNicIfNextTimestamp(struct SimbricksNicIf *nicif) {
+  struct SimbricksNicIfParams *params = &nicif->params;
   if (params->sync_pci && params->sync_eth) {
-    return (pci_last_rx_time <= eth_last_rx_time ? pci_last_rx_time
-                                                 : eth_last_rx_time);
+    return (nicif->pci_last_rx_time <= nicif->eth_last_rx_time ?
+              nicif->pci_last_rx_time :
+              nicif->eth_last_rx_time);
   } else if (params->sync_pci) {
-    return pci_last_rx_time;
+    return nicif->pci_last_rx_time;
   } else if (params->sync_eth) {
-    return eth_last_rx_time;
+    return nicif->eth_last_rx_time;
   } else {
     return 0;
   }
@@ -359,9 +355,10 @@ uint64_t SimbricksNicIfNextTimestamp(struct SimbricksNicIfParams *params) {
 /* PCI */
 
 volatile union SimbricksProtoPcieH2D *SimbricksNicIfH2DPoll(
-    struct SimbricksNicIfParams *params, uint64_t timestamp) {
+    struct SimbricksNicIf *nicif, uint64_t timestamp) {
   volatile union SimbricksProtoPcieH2D *msg =
-      (volatile union SimbricksProtoPcieH2D *)(h2d_queue + h2d_pos * H2D_ELEN);
+      (volatile union SimbricksProtoPcieH2D *)
+      (nicif->h2d_queue + nicif->h2d_pos * H2D_ELEN);
 
   /* message not ready */
   if ((msg->dummy.own_type & SIMBRICKS_PROTO_PCIE_H2D_OWN_MASK) !=
@@ -369,37 +366,39 @@ volatile union SimbricksProtoPcieH2D *SimbricksNicIfH2DPoll(
     return NULL;
 
   /* if in sync mode, wait till message is ready */
-  pci_last_rx_time = msg->dummy.timestamp;
-  if (params->sync_pci && pci_last_rx_time > timestamp)
+  nicif->pci_last_rx_time = msg->dummy.timestamp;
+  if (nicif->params.sync_pci && nicif->pci_last_rx_time > timestamp)
     return NULL;
 
   return msg;
 }
 
-void SimbricksNicIfH2DDone(volatile union SimbricksProtoPcieH2D *msg) {
+void SimbricksNicIfH2DDone(struct SimbricksNicIf *nicif,
+                           volatile union SimbricksProtoPcieH2D *msg) {
   msg->dummy.own_type =
       (msg->dummy.own_type & SIMBRICKS_PROTO_PCIE_H2D_MSG_MASK) |
       SIMBRICKS_PROTO_PCIE_H2D_OWN_HOST;
 }
 
-void SimbricksNicIfH2DNext(void) {
-  h2d_pos = (h2d_pos + 1) % H2D_ENUM;
+void SimbricksNicIfH2DNext(struct SimbricksNicIf *nicif) {
+  nicif->h2d_pos = (nicif->h2d_pos + 1) % H2D_ENUM;
 }
 
 volatile union SimbricksProtoPcieD2H *SimbricksNicIfD2HAlloc(
-    struct SimbricksNicIfParams *params, uint64_t timestamp) {
+    struct SimbricksNicIf *nicif, uint64_t timestamp) {
   volatile union SimbricksProtoPcieD2H *msg =
-      (volatile union SimbricksProtoPcieD2H *)(d2h_queue + d2h_pos * D2H_ELEN);
+      (volatile union SimbricksProtoPcieD2H *)
+      (nicif->d2h_queue + nicif->d2h_pos * D2H_ELEN);
 
   if ((msg->dummy.own_type & SIMBRICKS_PROTO_PCIE_D2H_OWN_MASK) !=
       SIMBRICKS_PROTO_PCIE_D2H_OWN_DEV) {
     return NULL;
   }
 
-  msg->dummy.timestamp = timestamp + params->pci_latency;
-  pci_last_tx_time = timestamp;
+  msg->dummy.timestamp = timestamp + nicif->params.pci_latency;
+  nicif->pci_last_tx_time = timestamp;
 
-  d2h_pos = (d2h_pos + 1) % D2H_ENUM;
+  nicif->d2h_pos = (nicif->d2h_pos + 1) % D2H_ENUM;
   return msg;
 }
 
@@ -407,9 +406,10 @@ volatile union SimbricksProtoPcieD2H *SimbricksNicIfD2HAlloc(
 /* Ethernet */
 
 volatile union SimbricksProtoNetN2D *SimbricksNicIfN2DPoll(
-    struct SimbricksNicIfParams *params, uint64_t timestamp) {
+    struct SimbricksNicIf *nicif, uint64_t timestamp) {
   volatile union SimbricksProtoNetN2D *msg =
-      (volatile union SimbricksProtoNetN2D *)(n2d_queue + n2d_pos * N2D_ELEN);
+      (volatile union SimbricksProtoNetN2D *)
+      (nicif->n2d_queue + nicif->n2d_pos * N2D_ELEN);
 
   /* message not ready */
   if ((msg->dummy.own_type & SIMBRICKS_PROTO_NET_N2D_OWN_MASK) !=
@@ -417,36 +417,38 @@ volatile union SimbricksProtoNetN2D *SimbricksNicIfN2DPoll(
     return NULL;
 
   /* if in sync mode, wait till message is ready */
-  eth_last_rx_time = msg->dummy.timestamp;
-  if (params->sync_eth && eth_last_rx_time > timestamp)
+  nicif->eth_last_rx_time = msg->dummy.timestamp;
+  if (nicif->params.sync_eth && nicif->eth_last_rx_time > timestamp)
     return NULL;
 
   return msg;
 }
 
-void SimbricksNicIfN2DDone(volatile union SimbricksProtoNetN2D *msg) {
+void SimbricksNicIfN2DDone(struct SimbricksNicIf *nicif,
+                           volatile union SimbricksProtoNetN2D *msg) {
   msg->dummy.own_type =
       (msg->dummy.own_type & SIMBRICKS_PROTO_NET_N2D_MSG_MASK) |
       SIMBRICKS_PROTO_NET_N2D_OWN_NET;
 }
 
-void SimbricksNicIfN2DNext(void) {
-  n2d_pos = (n2d_pos + 1) % N2D_ENUM;
+void SimbricksNicIfN2DNext(struct SimbricksNicIf *nicif) {
+  nicif->n2d_pos = (nicif->n2d_pos + 1) % N2D_ENUM;
 }
 
 volatile union SimbricksProtoNetD2N *SimbricksNicIfD2NAlloc(
-    struct SimbricksNicIfParams *params, uint64_t timestamp) {
+    struct SimbricksNicIf *nicif, uint64_t timestamp) {
   volatile union SimbricksProtoNetD2N *msg =
-      (volatile union SimbricksProtoNetD2N *)(d2n_queue + d2n_pos * D2N_ELEN);
+      (volatile union SimbricksProtoNetD2N *)
+      (nicif->d2n_queue + nicif->d2n_pos * D2N_ELEN);
 
   if ((msg->dummy.own_type & SIMBRICKS_PROTO_NET_D2N_OWN_MASK) !=
       SIMBRICKS_PROTO_NET_D2N_OWN_DEV) {
     return NULL;
   }
 
-  msg->dummy.timestamp = timestamp + params->eth_latency;
-  eth_last_tx_time = timestamp;
+  msg->dummy.timestamp = timestamp + nicif->params.eth_latency;
+  nicif->eth_last_tx_time = timestamp;
 
-  d2n_pos = (d2n_pos + 1) % D2N_ENUM;
+  nicif->d2n_pos = (nicif->d2n_pos + 1) % D2N_ENUM;
   return msg;
 }

lib/simbricks/nicif/nicif.h

@@ -25,6 +25,9 @@
 #ifndef SIMBRICKS_NICIF_NICIF_H_
 #define SIMBRICKS_NICIF_NICIF_H_
 
+#include <stddef.h>
+#include <stdint.h>
+
 #include <simbricks/proto/network.h>
 #include <simbricks/proto/pcie.h>
 
@@ -42,31 +45,65 @@ struct SimbricksNicIfParams {
   int sync_mode;
 };
 
-int SimbricksNicIfInit(struct SimbricksNicIfParams *params,
+struct SimbricksNicIf {
+  uint8_t *d2h_queue;
+  size_t d2h_pos;
+  size_t d2h_off; /* offset in shm region */
+
+  uint8_t *h2d_queue;
+  size_t h2d_pos;
+  size_t h2d_off; /* offset in shm region */
+
+  uint8_t *d2n_queue;
+  size_t d2n_pos;
+  size_t d2n_off; /* offset in shm region */
+
+  uint8_t *n2d_queue;
+  size_t n2d_pos;
+  size_t n2d_off; /* offset in shm region */
+
+  uint64_t pci_last_rx_time;
+  uint64_t pci_last_tx_time;
+  uint64_t eth_last_rx_time;
+  uint64_t eth_last_tx_time;
+  uint64_t current_epoch;
+
+  struct SimbricksNicIfParams params;
+
+  int shm_fd;
+  int pci_cfd;
+  int eth_cfd;
+};
+
+int SimbricksNicIfInit(struct SimbricksNicIf *nicif,
+                       struct SimbricksNicIfParams *params,
                        struct SimbricksProtoPcieDevIntro *di);
-void SimbricksNicIfCleanup(void);
+void SimbricksNicIfCleanup(struct SimbricksNicIf *nicif);
 
-int SimbricksNicIfSync(struct SimbricksNicIfParams *params, uint64_t timestamp);
-void SimbricksNicIfAdvanceEpoch(struct SimbricksNicIfParams *params,
+int SimbricksNicIfSync(struct SimbricksNicIf *nicif,
+                       uint64_t timestamp);
+void SimbricksNicIfAdvanceEpoch(struct SimbricksNicIf *nicif,
                                 uint64_t timestamp);
-uint64_t SimbricksNicIfAdvanceTime(struct SimbricksNicIfParams *params,
+uint64_t SimbricksNicIfAdvanceTime(struct SimbricksNicIf *nicif,
                                    uint64_t timestamp);
-uint64_t SimbricksNicIfNextTimestamp(struct SimbricksNicIfParams *params);
+uint64_t SimbricksNicIfNextTimestamp(struct SimbricksNicIf *nicif);
 
 volatile union SimbricksProtoPcieH2D *SimbricksNicIfH2DPoll(
-    struct SimbricksNicIfParams *params, uint64_t timestamp);
-void SimbricksNicIfH2DDone(volatile union SimbricksProtoPcieH2D *msg);
-void SimbricksNicIfH2DNext(void);
+    struct SimbricksNicIf *nicif, uint64_t timestamp);
+void SimbricksNicIfH2DDone(struct SimbricksNicIf *nicif,
+                           volatile union SimbricksProtoPcieH2D *msg);
+void SimbricksNicIfH2DNext(struct SimbricksNicIf *nicif);
 
 volatile union SimbricksProtoPcieD2H *SimbricksNicIfD2HAlloc(
-    struct SimbricksNicIfParams *params, uint64_t timestamp);
+    struct SimbricksNicIf *nicif, uint64_t timestamp);
 
 volatile union SimbricksProtoNetN2D *SimbricksNicIfN2DPoll(
-    struct SimbricksNicIfParams *params, uint64_t timestamp);
-void SimbricksNicIfN2DDone(volatile union SimbricksProtoNetN2D *msg);
-void SimbricksNicIfN2DNext(void);
+    struct SimbricksNicIf *nicif, uint64_t timestamp);
+void SimbricksNicIfN2DDone(struct SimbricksNicIf *nicif,
+                           volatile union SimbricksProtoNetN2D *msg);
+void SimbricksNicIfN2DNext(struct SimbricksNicIf *nicif);
 
 volatile union SimbricksProtoNetD2N *SimbricksNicIfD2NAlloc(
-    struct SimbricksNicIfParams *params, uint64_t timestamp);
+    struct SimbricksNicIf *nicif, uint64_t timestamp);
 
 #endif  // SIMBRICKS_NICIF_NICIF_H_

sims/nic/corundum/corundum_verilator.cc

@@ -54,7 +54,8 @@ static uint64_t eth_latency = 500 * 1000ULL;  // 500ns
 
 static volatile int exiting = 0;
 uint64_t main_time = 0;
-static struct SimbricksNicIfParams nsparams;
+static struct SimbricksNicIf nicif;
+
 #ifdef TRACE_ENABLED
 static VerilatedVcdC *trace;
 #endif
@@ -536,7 +537,7 @@ static void h2d_write(MMIOInterface &mmio,
 
 static void poll_h2d(MMIOInterface &mmio) {
   volatile union SimbricksProtoPcieH2D *msg =
-      SimbricksNicIfH2DPoll(&nsparams, main_time);
+      SimbricksNicIfH2DPoll(&nicif, main_time);
   uint8_t t;
 
   if (msg == NULL)
@@ -572,12 +573,12 @@ static void poll_h2d(MMIOInterface &mmio) {
       std::cerr << "poll_h2d: unsupported type=" << t << std::endl;
   }
 
-  SimbricksNicIfH2DDone(msg);
-  SimbricksNicIfH2DNext();
+  SimbricksNicIfH2DDone(&nicif, msg);
+  SimbricksNicIfH2DNext(&nicif);
 }
 
 static volatile union SimbricksProtoPcieD2H *d2h_alloc(void) {
-  return SimbricksNicIfD2HAlloc(&nsparams, main_time);
+  return SimbricksNicIfD2HAlloc(&nicif, main_time);
 }
 
 class EthernetTx {
@@ -592,7 +593,7 @@ class EthernetTx {
 
   void packet_done() {
     volatile union SimbricksProtoNetD2N *msg =
-        SimbricksNicIfD2NAlloc(&nsparams, main_time);
+        SimbricksNicIfD2NAlloc(&nicif, main_time);
     volatile struct SimbricksProtoNetD2NSend *send;
 
     if (!msg)
@@ -731,7 +732,7 @@ static void n2d_recv(EthernetRx &rx,
 
 static void poll_n2d(EthernetRx &rx) {
   volatile union SimbricksProtoNetN2D *msg =
-      SimbricksNicIfN2DPoll(&nsparams, main_time);
+      SimbricksNicIfN2DPoll(&nicif, main_time);
   uint8_t t;
 
   if (msg == NULL)
@@ -751,8 +752,8 @@ static void poll_n2d(EthernetRx &rx) {
       std::cerr << "poll_n2d: unsupported type=" << t << std::endl;
   }
 
-  SimbricksNicIfN2DDone(msg);
-  SimbricksNicIfN2DNext();
+  SimbricksNicIfN2DDone(&nicif, msg);
+  SimbricksNicIfN2DNext(&nicif);
 }
 
 void pci_msi_issue(uint8_t vec) {
@@ -829,6 +830,7 @@ int main(int argc, char *argv[]) {
   di.pci_revision = 0x00;
   di.pci_msi_nvecs = 32;
 
+  struct SimbricksNicIfParams nsparams;
   nsparams.sync_pci = 1;
   nsparams.sync_eth = 1;
   nsparams.pci_socket_path = argv[1];
@@ -841,7 +843,7 @@ int main(int argc, char *argv[]) {
          sync_mode == SIMBRICKS_PROTO_SYNC_BARRIER);
   nsparams.sync_mode = sync_mode;
 
-  if (SimbricksNicIfInit(&nsparams, &di)) {
+  if (SimbricksNicIfInit(&nicif, &nsparams, &di)) {
     return EXIT_FAILURE;
   }
   std::cout << "sync_pci=" << nsparams.sync_pci
@@ -950,17 +952,17 @@ int main(int argc, char *argv[]) {
   top->rst = 0;
 
   while (!exiting) {
-    while (SimbricksNicIfSync(&nsparams, main_time)) {
+    while (SimbricksNicIfSync(&nicif, main_time)) {
       std::cerr << "warn: SimbricksNicIfSync failed (t=" << main_time << ")"
                 << std::endl;
     }
-    SimbricksNicIfAdvanceEpoch(&nsparams, main_time);
+    SimbricksNicIfAdvanceEpoch(&nicif, main_time);
 
     do {
       poll_h2d(mmio);
       poll_n2d(rx);
     } while ((nsparams.sync_pci || nsparams.sync_eth) &&
-             SimbricksNicIfNextTimestamp(&nsparams) <= main_time && !exiting);
+             SimbricksNicIfNextTimestamp(&nicif) <= main_time && !exiting);
 
     /* falling edge */
     top->clk = !top->clk;