changes for sychronized PCI & ethernet

corundum/corundum_verilator.cpp

@@ -15,10 +15,16 @@ extern "C" {
 #include "dma.h"
 #include "mem.h"
 
+#define CLOCK_PERIOD (10 * 1000 * 1000ULL) // 200KHz
+#define PCI_ASYNCHRONY (500 * 1000 * 1000ULL) // 200us
+#define ETH_ASYNCHRONY (500 * 1000 * 1000ULL) // 200us
+
 struct DMAOp;
 
 static volatile int exiting = 0;
 static uint64_t main_time = 0;
+static uint64_t pci_last_time = 0;
+static uint64_t eth_last_time = 0;
 //static VerilatedVcdC* trace;
 
 
@@ -512,6 +518,10 @@ static void h2d_write(MMIOInterface &mmio,
     }
 }
 
+static void h2d_sync(volatile struct cosim_pcie_proto_h2d_sync *sync)
+{
+    pci_last_time = sync->timestamp;
+}
 
 static void poll_h2d(MMIOInterface &mmio)
 {
@@ -540,6 +550,10 @@ static void poll_h2d(MMIOInterface &mmio)
             h2d_writecomp(&msg->writecomp);
             break;
 
+        case COSIM_PCIE_PROTO_H2D_MSG_SYNC:
+            h2d_sync(&msg->sync);
+            break;
+
         default:
             std::cerr << "poll_h2d: unsupported type=" << t << std::endl;
     }
@@ -729,11 +743,36 @@ static void msi_step(Vinterface &top)
     }
 }
 
+static void sync_pci(MMIOInterface &mmio)
+{
+    uint64_t cur_ts = (main_time / 2) * CLOCK_PERIOD;
+    volatile union cosim_pcie_proto_d2h *msg = nicsim_d2h_alloc();
+    volatile struct cosim_pcie_proto_d2h_sync *sync;
+
+    sync = &msg->sync;
+
+    sync->timestamp = cur_ts + PCI_ASYNCHRONY;
+    // WMB();
+    sync->own_type = COSIM_PCIE_PROTO_D2H_MSG_SYNC |
+        COSIM_PCIE_PROTO_D2H_OWN_HOST;
+
+    while (pci_last_time < cur_ts && !exiting) {
+        /*std::cout << "waiting for pci pci_time=" << pci_last_time <<
+            "  cur=" << cur_ts << std::endl;*/
+        poll_h2d(mmio);
+    }
+}
+
+static void sync_eth(EthernetRx &rx)
+{
+}
+
 int main(int argc, char *argv[])
 {
     Verilated::commandArgs(argc, argv);
     Verilated::traceEverOn(true);
 
+    int sync_pci_en, sync_eth_en;
     struct cosim_pcie_proto_dev_intro di;
     memset(&di, 0, sizeof(di));
 
@@ -747,11 +786,15 @@ int main(int argc, char *argv[])
     di.pci_revision = 0x00;
     di.pci_msi_nvecs = 32;
 
-    if (nicsim_init(&di, "/tmp/cosim-pci", "/tmp/cosim-eth",
-                "/dev/shm/dummy_nic_shm"))
+    sync_pci_en = 1;
+    sync_eth_en = 1;
+    if (nicsim_init(&di, "/tmp/cosim-pci", &sync_pci_en,
+                "/tmp/cosim-eth", &sync_eth_en, "/dev/shm/dummy_nic_shm"))
     {
         return EXIT_FAILURE;
     }
+    std::cout << "sync_pci=" << sync_pci_en << "  sync_eth=" << sync_eth_en <<
+        std::endl;
 
     signal(SIGINT, sigint_handler);
 
@@ -860,6 +903,10 @@ int main(int argc, char *argv[])
     top->rst = 0;
 
     while (!exiting) {
+        if (sync_pci_en)
+            sync_pci(mmio);
+        if (sync_eth_en)
+            sync_eth(rx);
         poll_h2d(mmio);
         poll_n2d(rx);
 

dummy_nic/dummy_nic.c

@@ -110,6 +110,7 @@ void poll_h2d(void)
 
 int main(int argc, char *argv[])
 {
+    int sync = 0;
     struct cosim_pcie_proto_dev_intro di;
     memset(&di, 0, sizeof(di));
 
@@ -126,7 +127,9 @@ int main(int argc, char *argv[])
     di.pci_revision = 0x00;
     di.pci_msi_nvecs = 0x00;
 
-    if (nicsim_init(&di, "/tmp/cosim-pci", NULL, "/dev/shm/dummy_nic_shm")) {
+    if (nicsim_init(&di, "/tmp/cosim-pci", &sync, NULL, NULL,
+                "/dev/shm/dummy_nic_shm"))
+    {
         return EXIT_FAILURE;
     }
 

nicsim_common/include/nicsim.h

@@ -28,7 +28,8 @@
 #include <cosim_eth_proto.h>
 
 int nicsim_init(struct cosim_pcie_proto_dev_intro *di,
-        const char *pci_socket_path, const char *eth_socket_path,
+        const char *pci_socket_path, int *sync_pci,
+        const char *eth_socket_path, int *sync_eth,
         const char *shm_path);
 void nicsim_cleanup(void);
 

nicsim_common/nicsim.c

@@ -66,7 +66,8 @@ static int shm_fd = -1;
 static int pci_cfd = -1;
 static int eth_cfd = -1;
 
-static int accept_pci(struct cosim_pcie_proto_dev_intro *di, int pci_lfd)
+static int accept_pci(struct cosim_pcie_proto_dev_intro *di, int pci_lfd,
+        int *sync_pci)
 {
     if ((pci_cfd = accept(pci_lfd, NULL, NULL)) < 0) {
         return -1;
@@ -82,6 +83,11 @@ static int accept_pci(struct cosim_pcie_proto_dev_intro *di, int pci_lfd)
     di->h2d_elen = H2D_ELEN;
     di->h2d_nentries = H2D_ENUM;
 
+    if (*sync_pci)
+        di->flags |= COSIM_PCIE_PROTO_FLAGS_DI_SYNC;
+    else
+        di->flags &= ~((uint64_t) COSIM_PCIE_PROTO_FLAGS_DI_SYNC);
+
     if (uxsocket_send(pci_cfd, di, sizeof(*di), shm_fd)) {
         return -1;
     }
@@ -89,7 +95,7 @@ static int accept_pci(struct cosim_pcie_proto_dev_intro *di, int pci_lfd)
     return 0;
 }
 
-static int accept_eth(int eth_lfd)
+static int accept_eth(int eth_lfd, int *sync_eth)
 {
     struct cosim_eth_proto_dev_intro di;
 
@@ -101,6 +107,8 @@ static int accept_eth(int eth_lfd)
 
     memset(&di, 0, sizeof(di));
     di.flags = 0;
+    if (*sync_eth)
+        di.flags |= COSIM_ETH_PROTO_FLAGS_DI_SYNC;
 
     di.d2n_offset = d2n_off;
     di.d2n_elen = D2N_ELEN;
@@ -118,7 +126,7 @@ static int accept_eth(int eth_lfd)
 }
 
 static int accept_conns(struct cosim_pcie_proto_dev_intro *di,
-        int pci_lfd, int eth_lfd)
+        int pci_lfd, int *sync_pci, int eth_lfd, int *sync_eth)
 {
     struct pollfd pfds[2];
     int await_pci = pci_lfd != -1;
@@ -140,23 +148,23 @@ static int accept_conns(struct cosim_pcie_proto_dev_intro *di,
             }
 
             if (pfds[0].revents) {
-                if (accept_pci(di, pci_lfd) != 0)
+                if (accept_pci(di, pci_lfd, sync_pci) != 0)
                     return -1;
                 await_pci = 0;
             }
             if (pfds[1].revents) {
-                if (accept_eth(eth_lfd) != 0)
+                if (accept_eth(eth_lfd, sync_eth) != 0)
                     return -1;
                 await_eth = 0;
             }
         } else if (await_pci) {
             /* waiting just on pci */
-            if (accept_pci(di, pci_lfd) != 0)
+            if (accept_pci(di, pci_lfd, sync_pci) != 0)
                 return -1;
             await_pci = 0;
         } else {
             /* waiting just on ethernet */
-            if (accept_eth(eth_lfd) != 0)
+            if (accept_eth(eth_lfd, sync_eth) != 0)
                 return -1;
             await_eth = 0;
         }
@@ -166,7 +174,8 @@ static int accept_conns(struct cosim_pcie_proto_dev_intro *di,
 }
 
 int nicsim_init(struct cosim_pcie_proto_dev_intro *di,
-        const char *pci_socket_path, const char *eth_socket_path,
+        const char *pci_socket_path, int *sync_pci,
+        const char *eth_socket_path, int *sync_eth,
         const char *shm_path)
 {
     int pci_lfd = -1, eth_lfd = -1;
@@ -202,7 +211,7 @@ int nicsim_init(struct cosim_pcie_proto_dev_intro *di,
     }
 
     /* accept connection fds */
-    if (accept_conns(di, pci_lfd, eth_lfd) != 0) {
+    if (accept_conns(di, pci_lfd, sync_pci, eth_lfd, sync_eth) != 0) {
         return -1;
     }
 
@@ -212,6 +221,8 @@ int nicsim_init(struct cosim_pcie_proto_dev_intro *di,
         if (recv(pci_cfd, &hi, sizeof(hi), 0) != sizeof(hi)) {
             return -1;
         }
+        if ((hi.flags & COSIM_PCIE_PROTO_FLAGS_HI_SYNC) == 0)
+            *sync_pci = 0;
         printf("pci host info received\n");
     }
     if (eth_socket_path != NULL) {
@@ -219,6 +230,8 @@ int nicsim_init(struct cosim_pcie_proto_dev_intro *di,
         if (recv(eth_cfd, &ni, sizeof(ni), 0) != sizeof(ni)) {
             return -1;
         }
+        if ((ni.flags & COSIM_ETH_PROTO_FLAGS_NI_SYNC) == 0)
+            *sync_eth = 0;
         printf("eth net info received\n");
     }
 

proto/cosim_eth_proto.h

@@ -33,7 +33,7 @@ struct cosim_eth_proto_dev_intro {
 } __attribute__((packed));
 
 
-#define COSIM_ETH_PROTO_FLAGS_IN_SYNC (1 << 0)
+#define COSIM_ETH_PROTO_FLAGS_NI_SYNC (1 << 0)
 
 /** welcome message sent by network to device */
 struct cosim_eth_proto_net_intro {