move common nicsim code to library

.gitignore

 *.o
 _vimrc_local.vim
 dummy_nic/dummy_nic
+nicsim_common/libnicsim_common.a
 corundum/obj_dir
 corundum/corundum_verilator

dummy_nic/Makefile

+CPPFLAGS += -I../nicsim_common/include -I../proto
 CFLAGS += -Wall -Wextra -Wno-unused-parameter -O3
 
-dummy_nic: dummy_nic.o poll.o utils.o
+dummy_nic: dummy_nic.o ../nicsim_common/libnicsim_common.a
 
 all: dummy_nic
 

dummy_nic/dummy_nic.c

@@ -27,53 +27,91 @@
 #include <sys/socket.h>
 #include <unistd.h>
 
-#include "dummy_nic.h"
+#include <nicsim.h>
 
-uint8_t *d2h_queue;
-size_t d2h_pos;
+static volatile union cosim_pcie_proto_d2h *d2h_alloc(void)
+{
+    volatile union cosim_pcie_proto_d2h *msg = nicsim_d2h_alloc();
+    if (msg == NULL) {
+        fprintf(stderr, "d2h_alloc: no entry available\n");
+        abort();
+    }
+    return msg;
+}
 
-uint8_t *h2d_queue;
-size_t h2d_pos;
+static void h2d_read(volatile struct cosim_pcie_proto_h2d_read *read)
+{
+    volatile union cosim_pcie_proto_d2h *msg;
+    volatile struct cosim_pcie_proto_d2h_readcomp *rc;
+    uint64_t val;
 
+    msg = d2h_alloc();
+    rc = &msg->readcomp;
 
-int main(int argc, char *argv[])
+    val = read->offset + 42;
+    printf("read(bar=%u, off=%lu, len=%u) = %lu\n", read->bar, read->offset,
+            read->len, val);
+
+    memcpy((void *) rc->data, &val, read->len);
+    rc->req_id = read->req_id;
+
+    //WMB();
+    rc->own_type = COSIM_PCIE_PROTO_D2H_MSG_READCOMP |
+        COSIM_PCIE_PROTO_D2H_OWN_HOST;
+}
+
+static void h2d_write(volatile struct cosim_pcie_proto_h2d_write *write)
 {
-    int shm_fd, pci_lfd, pci_cfd;
-    size_t d2h_off, h2d_off;
-    void *shmptr;
+    volatile union cosim_pcie_proto_d2h *msg;
+    volatile struct cosim_pcie_proto_d2h_writecomp *wc;
+    uint64_t val;
 
-    if ((shm_fd = shm_create("/dev/shm/dummy_nic_shm", 32 * 1024 * 1024, &shmptr)) < 0) {
-        return EXIT_FAILURE;
-    }
+    msg = d2h_alloc();
+    wc = &msg->writecomp;
 
-    if ((pci_lfd = uxsocket_init("/tmp/cosim-pci")) < 0) {
-        return EXIT_FAILURE;
-    }
+    val = 0;
+    memcpy(&val, (void *) write->data, write->len);
 
-    if ((pci_cfd = accept(pci_lfd, NULL, NULL)) < 0) { 
-        perror("accept pci_lfd failed");
-        return EXIT_FAILURE;
-    }
-    printf("connection accepted\n");
+    printf("write(bar=%u, off=%lu, len=%u, val=%lu)\n", write->bar,
+            write->offset, write->len, val);
 
-    d2h_off = 0;
-    h2d_off = (uint64_t) D2H_ELEN * D2H_ENUM;
+    wc->req_id = write->req_id;
 
-    d2h_queue = (uint8_t *) shmptr + d2h_off;
-    h2d_queue = (uint8_t *) shmptr + h2d_off;
+    //WMB();
+    wc->own_type = COSIM_PCIE_PROTO_D2H_MSG_WRITECOMP |
+        COSIM_PCIE_PROTO_D2H_OWN_HOST;
+}
 
-    d2h_pos = h2d_pos = 0;
+void poll_h2d(void)
+{
+    volatile union cosim_pcie_proto_h2d *msg = nicif_h2d_poll();
+    uint8_t type;
 
-    struct cosim_pcie_proto_dev_intro di;
-    memset(&di, 0, sizeof(di));
+    if (msg == NULL)
+        return;
+
+    type = msg->dummy.own_type & COSIM_PCIE_PROTO_H2D_MSG_MASK;
+    switch (type) {
+        case COSIM_PCIE_PROTO_H2D_MSG_READ:
+            h2d_read(&msg->read);
+            break;
+
+        case COSIM_PCIE_PROTO_H2D_MSG_WRITE:
+            h2d_write(&msg->write);
+            break;
 
-    di.d2h_offset = d2h_off;
-    di.d2h_elen = D2H_ELEN;
-    di.d2h_nentries = D2H_ENUM;
+        default:
+            fprintf(stderr, "poll_h2d: unsupported type=%u\n", type);
+    }
+
+    nicif_h2d_done(msg);
+    nicif_h2d_next();
+}
 
-    di.h2d_offset = h2d_off;
-    di.h2d_elen = H2D_ELEN;
-    di.h2d_nentries = H2D_ENUM;
+int main(int argc, char *argv[])
+{
+    struct cosim_pcie_proto_dev_intro di;
+    memset(&di, 0, sizeof(di));
 
     di.bars[0].len = 0x1000;
     di.bars[0].flags = COSIM_PCIE_PROTO_BAR_64;
@@ -87,23 +125,14 @@ int main(int argc, char *argv[])
     di.pci_subclass = 0x00;
     di.pci_revision = 0x00;
 
-    if (uxsocket_send(pci_cfd, &di, sizeof(di), shm_fd)) {
-        return EXIT_FAILURE;
-    }
-    printf("connection sent\n");
-
-
-    struct cosim_pcie_proto_host_intro hi;
-    if (recv(pci_cfd, &hi, sizeof(hi), 0) != sizeof(hi)) {
-
+    if (nicsim_init(&di, "/tmp/cosim-pci", "/dev/shm/dummy_nic_shm")) {
         return EXIT_FAILURE;
     }
-    printf("host info received\n");
 
     while (1) {
         poll_h2d();
     }
-    close(pci_lfd);
 
+    nicsim_cleanup();
     return 0;
 }

nicsim_common/Makefile

+CPPFLAGS += -I include/ -I../proto/
+CFLAGS += -Wall -Wextra -Wno-unused-parameter -O3
+
+all: libnicsim_common.a
+
+libnicsim_common.a: nicsim.o utils.o
+	$(AR) rcs $@ $^
+
+clean:
+	rm -rf libnicsim_common.a nicsim.o utils.o

nicsim_common/include/nicsim.h

+/*
+ * Copyright 2020 Max Planck Institute for Software Systems
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef COSIM_NICSIM_H_
+#define COSIM_NICSIM_H_
+
+#include <cosim_pcie_proto.h>
+
+int nicsim_init(struct cosim_pcie_proto_dev_intro *di,
+        const char *uxsocket_path, const char *shm_path);
+void nicsim_cleanup(void);
+
+volatile union cosim_pcie_proto_h2d *nicif_h2d_poll(void);
+void nicif_h2d_done(volatile union cosim_pcie_proto_h2d *msg);
+void nicif_h2d_next(void);
+
+volatile union cosim_pcie_proto_d2h *nicsim_d2h_alloc(void);
+
+#endif /* ndef COSIM_NICSIM_H_ */

dummy_nic/dummy_nic.h → nicsim_common/internal.h

@@ -24,24 +24,6 @@
 #include <stddef.h>
 #include <stdio.h>
 
-#include "../proto/cosim_pcie_proto.h"
-
-#define D2H_ELEN (4096 + 64)
-#define D2H_ENUM 1024
-
-#define H2D_ELEN (4096 + 64)
-#define H2D_ENUM 1024
-
-extern uint8_t *d2h_queue;
-extern size_t d2h_pos;
-
-extern uint8_t *h2d_queue;
-extern size_t h2d_pos;
-
-/* poll.c */
-void poll_h2d(void);
-
-/* utils.c */
 int uxsocket_init(const char *path);
 int uxsocket_send(int connfd, void *data, size_t len, int fd);
 int shm_create(const char *path, size_t size, void **addr);

dummy_nic/poll.c → nicsim_common/nicsim.c

@@ -24,97 +24,123 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
+#include <sys/socket.h>
+#include <unistd.h>
 
-#include "dummy_nic.h"
+#include <nicsim.h>
 
-static volatile union cosim_pcie_proto_d2h *d2h_alloc(void)
+#include "internal.h"
+
+#define D2H_ELEN (4096 + 64)
+#define D2H_ENUM 1024
+
+#define H2D_ELEN (4096 + 64)
+#define H2D_ENUM 1024
+
+
+static uint8_t *d2h_queue;
+static size_t d2h_pos;
+
+static uint8_t *h2d_queue;
+static size_t h2d_pos;
+
+static int pci_cfd = -1;
+
+int nicsim_init(struct cosim_pcie_proto_dev_intro *di,
+        const char *uxsocket_path, const char *shm_path)
 {
-    volatile union cosim_pcie_proto_d2h *msg =
-        (volatile union cosim_pcie_proto_d2h *)
-        (d2h_queue + d2h_pos * D2H_ELEN);
+    int shm_fd, pci_lfd;
+    size_t d2h_off, h2d_off;
+    void *shmptr;
 
-    if ((msg->dummy.own_type & COSIM_PCIE_PROTO_D2H_OWN_MASK) !=
-            COSIM_PCIE_PROTO_D2H_OWN_DEV)
-    {
-        fprintf(stderr, "d2h_alloc: no entry available\n");
-        abort();
+    if ((shm_fd = shm_create(shm_path, 32 * 1024 * 1024, &shmptr)) < 0) {
+        return -1;
     }
 
-    d2h_pos = (d2h_pos + 1) % D2H_ENUM;
-    return msg;
-}
+    if ((pci_lfd = uxsocket_init(uxsocket_path)) < 0) {
+        return -1;
+    }
 
-static void h2d_read(volatile struct cosim_pcie_proto_h2d_read *read)
-{
-    volatile union cosim_pcie_proto_d2h *msg;
-    volatile struct cosim_pcie_proto_d2h_readcomp *rc;
-    uint64_t val;
+    if ((pci_cfd = accept(pci_lfd, NULL, NULL)) < 0) { 
+        return -1;
+    }
+    close(pci_lfd);
+    printf("connection accepted\n");
 
-    msg = d2h_alloc();
-    rc = &msg->readcomp;
+    d2h_off = 0;
+    h2d_off = (uint64_t) D2H_ELEN * D2H_ENUM;
 
-    val = read->offset + 42;
-    printf("read(bar=%u, off=%lu, len=%u) = %lu\n", read->bar, read->offset,
-            read->len, val);
+    d2h_queue = (uint8_t *) shmptr + d2h_off;
+    h2d_queue = (uint8_t *) shmptr + h2d_off;
 
-    memcpy((void *) rc->data, &val, read->len);
-    rc->req_id = read->req_id;
+    d2h_pos = h2d_pos = 0;
 
-    //WMB();
-    rc->own_type = COSIM_PCIE_PROTO_D2H_MSG_READCOMP |
-        COSIM_PCIE_PROTO_D2H_OWN_HOST;
-}
+    di->d2h_offset = d2h_off;
+    di->d2h_elen = D2H_ELEN;
+    di->d2h_nentries = D2H_ENUM;
 
-static void h2d_write(volatile struct cosim_pcie_proto_h2d_write *write)
-{
-    volatile union cosim_pcie_proto_d2h *msg;
-    volatile struct cosim_pcie_proto_d2h_writecomp *wc;
-    uint64_t val;
+    di->h2d_offset = h2d_off;
+    di->h2d_elen = H2D_ELEN;
+    di->h2d_nentries = H2D_ENUM;
 
-    msg = d2h_alloc();
-    wc = &msg->writecomp;
+    if (uxsocket_send(pci_cfd, di, sizeof(*di), shm_fd)) {
+        return -1;
+    }
+    printf("connection sent\n");
 
-    val = 0;
-    memcpy(&val, (void *) write->data, write->len);
 
-    printf("write(bar=%u, off=%lu, len=%u, val=%lu)\n", write->bar,
-            write->offset, write->len, val);
+    struct cosim_pcie_proto_host_intro hi;
+    if (recv(pci_cfd, &hi, sizeof(hi), 0) != sizeof(hi)) {
+        return -1;
+    }
+    printf("host info received\n");
 
-    wc->req_id = write->req_id;
+    return 0;
+}
 
-    //WMB();
-    wc->own_type = COSIM_PCIE_PROTO_D2H_MSG_WRITECOMP |
-        COSIM_PCIE_PROTO_D2H_OWN_HOST;
+void nicsim_cleanup(void)
+{
+    close(pci_cfd);
 }
 
-void poll_h2d(void)
+volatile union cosim_pcie_proto_h2d *nicif_h2d_poll(void)
 {
     volatile union cosim_pcie_proto_h2d *msg =
         (volatile union cosim_pcie_proto_h2d *)
         (h2d_queue + h2d_pos * H2D_ELEN);
-    uint8_t type;
 
     /* message not ready */
     if ((msg->dummy.own_type & COSIM_PCIE_PROTO_H2D_OWN_MASK) !=
             COSIM_PCIE_PROTO_H2D_OWN_DEV)
-        return;
+        return NULL;
 
-    type = msg->dummy.own_type & COSIM_PCIE_PROTO_H2D_MSG_MASK;
-    switch (type) {
-        case COSIM_PCIE_PROTO_H2D_MSG_READ:
-            h2d_read(&msg->read);
-            break;
+    return msg;
+}
 
-        case COSIM_PCIE_PROTO_H2D_MSG_WRITE:
-            h2d_write(&msg->write);
-            break;
+void nicif_h2d_done(volatile union cosim_pcie_proto_h2d *msg)
+{
+    msg->dummy.own_type = (msg->dummy.own_type & COSIM_PCIE_PROTO_H2D_MSG_MASK)
+        | COSIM_PCIE_PROTO_H2D_OWN_HOST;
+}
 
-        default:
-            fprintf(stderr, "poll_h2d: unsupported type=%u\n", type);
-    }
+void nicif_h2d_next(void)
+{
+    h2d_pos = (h2d_pos + 1) % H2D_ENUM;
+}
 
-    //WMB();
-    msg->dummy.own_type = type | COSIM_PCIE_PROTO_H2D_OWN_HOST;
+volatile union cosim_pcie_proto_d2h *nicsim_d2h_alloc(void)
+{
+    volatile union cosim_pcie_proto_d2h *msg =
+        (volatile union cosim_pcie_proto_d2h *)
+        (d2h_queue + d2h_pos * D2H_ELEN);
 
-    h2d_pos = (h2d_pos + 1) % H2D_ENUM;
+    if ((msg->dummy.own_type & COSIM_PCIE_PROTO_D2H_OWN_MASK) !=
+            COSIM_PCIE_PROTO_D2H_OWN_DEV)
+    {
+        return NULL;
+    }
+
+    d2h_pos = (d2h_pos + 1) % D2H_ENUM;
+    return msg;
 }
+

dummy_nic/utils.c → nicsim_common/utils.c

@@ -31,7 +31,7 @@
 #include <sys/mman.h>
 #include <unistd.h>
 
-#include "dummy_nic.h"
+#include "internal.h"
 
 int uxsocket_init(const char *path)
 {