corundum-bm: checkpoint

corundum_bm/.gitignore

+corundum_bm
+tester

corundum_bm/Makefile

+CPPFLAGS += -I../nicsim_common/include -I../proto
+CFLAGS += -Wall -Wextra -Wno-unused-parameter -O3
+
+all: corundum_bm tester
+
+corundum_bm: corundum_bm.o ../nicsim_common/libnicsim_common.a
+
+tester: tester.o ../nicsim_common/libnicsim_common.a
+
+clean:
+	rm -f *.o corundum_bm tester

corundum_bm/corundum_bm.c

+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <unistd.h>
+#include <signal.h>
+
+#include <corundum_bm.h>
+#include <nicsim.h>
+
+static volatile int exiting = 0;
+static struct CorundumRegs regs;
+
+static void sigint_handler(int dummy)
+{
+    exiting = 1;
+}
+
+static volatile union cosim_pcie_proto_d2h *d2h_alloc(void)
+{
+    volatile union cosim_pcie_proto_d2h *msg = nicsim_d2h_alloc();
+    if (msg == NULL) {
+        fprintf(stderr, "d2h_alloc: no entry available\n");
+        abort();
+    }
+    return msg;
+}
+
+static uint64_t csr_read(uint64_t off)
+{
+    switch (off) {
+        case   0x00: return 32; /* firmware id */
+        case   0x04: return 1; /* firmware version */
+        case   0x08: return 0x43215678; /* board id */
+        case   0x0c: return 0x1; /* board version */
+        case   0x10: return 1; /* phc count */
+        case   0x14: return 0x200; /* phc offset */
+        case   0x18: return 0x80; /* phc stride */
+        case   0x20: return 1; /* if_count */
+        case   0x24: return 0x80000; /* if stride */
+        case   0x2c: return 0x80000; /* if csr offset */
+        case  0x200: return 0x1; /* phc features */
+        default:
+            fprintf(stderr, "csr_read(%lu) unimplemented\n", off);
+            return 0;
+    }
+}
+
+static void csr_write(uint64_t off, uint64_t val)
+{
+}
+
+static void read_complete(uint64_t req_id, void *val, uint16_t len)
+{
+    volatile union cosim_pcie_proto_d2h *msg;
+    volatile struct cosim_pcie_proto_d2h_readcomp *rc;
+
+    msg = d2h_alloc();
+    rc = &msg->readcomp;
+
+    memcpy((void *)rc->data, val, len);
+    rc->req_id = req_id;
+
+    //WMB();
+    rc->own_type = COSIM_PCIE_PROTO_D2H_MSG_READCOMP |
+        COSIM_PCIE_PROTO_D2H_OWN_HOST;
+}
+
+static void h2d_read(volatile struct cosim_pcie_proto_h2d_read *read)
+{
+    printf("read(bar=0x%x, off=0x%lx, len=%u)\n", read->bar, read->offset, read->len);
+    if (read->offset < 0x80000) {
+        uint64_t val = csr_read(read->offset);
+        read_complete(read->req_id, &val, read->len);
+    } else {
+        switch (read->offset - 0x80000) {
+        case REG_A:
+            read_complete(read->req_id, &regs.reg_a, read->len);
+            break;
+        case REG_B:
+            read_complete(read->req_id, &regs.reg_b, read->len);
+            break;
+        case REG_C:
+            read_complete(read->req_id, &regs.reg_c, read->len);
+            break;
+        case REG_D:
+            read_complete(read->req_id, &regs.reg_d, read->len);
+            break;
+        case REG_E:
+            read_complete(read->req_id, &regs.reg_e, read->len);
+            break;
+        case REG_F:
+            read_complete(read->req_id, &regs.reg_f, read->len);
+            break;
+        case REG_G:
+            read_complete(read->req_id, &regs.reg_g, read->len);
+            break;
+        case REG_H:
+            read_complete(read->req_id, &regs.reg_h, read->len);
+            break;
+        case REG_I:
+            read_complete(read->req_id, &regs.reg_i, read->len);
+            break;
+        case REG_J:
+            read_complete(read->req_id, &regs.reg_j, read->len);
+            break;
+        case REG_K:
+            read_complete(read->req_id, &regs.reg_k, read->len);
+            break;
+        case REG_L:
+            read_complete(read->req_id, &regs.reg_l, read->len);
+            break;
+        case REG_M:
+            read_complete(read->req_id, &regs.reg_m, read->len);
+            break;
+        case REG_N:
+            read_complete(read->req_id, &regs.reg_n, read->len);
+            break;
+        case REG_O:
+            read_complete(read->req_id, &regs.reg_o, read->len);
+            break;
+        default:
+            fprintf(stderr, "unimplemented read at off=0x%lx len=%u\n", read->offset, read->len);
+            uint64_t val = 0;
+            read_complete(read->req_id, &val, read->len);
+            break;
+        }
+    }
+}
+
+static void h2d_write(volatile struct cosim_pcie_proto_h2d_write *write)
+{
+    uint64_t val = 0;
+    memcpy(&val, (void *)write->data, write->len);
+
+    if (write->offset < 0x80000) {
+        volatile union cosim_pcie_proto_d2h *msg;
+        volatile struct cosim_pcie_proto_d2h_writecomp *wc;
+
+        msg = d2h_alloc();
+        wc = &msg->writecomp;
+
+        printf("write(bar=0x%x, off=0x%lx, len=%u)\n", write->bar, write->offset, write->len);
+
+        csr_write(write->offset, val);
+        wc->req_id = write->req_id;
+
+        //WMB();
+        wc->own_type = COSIM_PCIE_PROTO_D2H_MSG_WRITECOMP |
+            COSIM_PCIE_PROTO_D2H_OWN_HOST;
+    } else {
+        fprintf(stderr, "unimplemented write at off=0x%lx\n", write->offset);
+    }
+}
+
+static void h2d_readcomp(volatile struct cosim_pcie_proto_h2d_readcomp *rc)
+{
+    printf("read complete(req_id=%lu)\n", rc->req_id);
+}
+
+static void h2d_writecomp(volatile struct cosim_pcie_proto_h2d_writecomp *wc)
+{
+    printf("write complete(req_id=%lu\n", wc->req_id);
+}
+
+static void n2d_recv(volatile struct cosim_eth_proto_n2d_recv *recv)
+{
+    printf("RX recv(port=%u, len=%u)\n", recv->port, recv->len);
+}
+
+static void poll_h2d(void)
+{
+    volatile union cosim_pcie_proto_h2d *msg = nicif_h2d_poll();
+    uint8_t type;
+
+    if (msg == NULL)
+        return;
+
+    type = msg->dummy.own_type & COSIM_PCIE_PROTO_H2D_MSG_MASK;
+    switch (type) {
+        case COSIM_PCIE_PROTO_H2D_MSG_READ:
+            h2d_read(&msg->read);
+            break;
+
+        case COSIM_PCIE_PROTO_H2D_MSG_WRITE:
+            h2d_write(&msg->write);
+            break;
+
+        case COSIM_PCIE_PROTO_H2D_MSG_READCOMP:
+            h2d_readcomp(&msg->readcomp);
+            break;
+
+        case COSIM_PCIE_PROTO_H2D_MSG_WRITECOMP:
+            h2d_writecomp(&msg->writecomp);
+            break;
+
+        default:
+            fprintf(stderr, "poll_h2d: unsupported type=%u\n", type);
+    }
+
+    nicif_h2d_done(msg);
+    nicif_h2d_next();
+}
+
+static void poll_n2d(void)
+{
+    volatile union cosim_eth_proto_n2d *msg = nicif_n2d_poll();
+    uint8_t t;
+
+    if (msg == NULL)
+        return;
+
+    t = msg->dummy.own_type & COSIM_ETH_PROTO_N2D_MSG_MASK;
+    switch (t) {
+        case COSIM_ETH_PROTO_N2D_MSG_RECV:
+            n2d_recv(&msg->recv);
+            break;
+
+        default:
+            fprintf(stderr, "poll_n2d: unsupported type=%u", t);
+    }
+
+    nicif_n2d_done(msg);
+    nicif_n2d_next();
+}
+
+int main(int argc, char *argv[])
+{
+    struct cosim_pcie_proto_dev_intro di;
+    memset(&di, 0, sizeof(di));
+
+    di.bars[0].len = 1 << 24;
+    di.bars[0].flags = COSIM_PCIE_PROTO_BAR_64;
+
+    di.pci_vendor_id = 0x5543;
+    di.pci_device_id = 0x1001;
+    di.pci_class = 0x02;
+    di.pci_subclass = 0x00;
+    di.pci_revision = 0x00;
+    di.pci_msi_nvecs = 32;
+
+    int sync_pci_en = 0, sync_eth_en = 0;
+    if (nicsim_init(&di, "/tmp/cosim-pci", &sync_pci_en,
+                NULL, &sync_eth_en,
+                "/dev/shm/dummy_nic_shm")) {
+        return EXIT_FAILURE;
+    }
+
+    signal(SIGINT, sigint_handler);
+
+    memset(&regs, 0, sizeof(regs));
+
+    while (!exiting) {
+        poll_h2d();
+        poll_n2d();
+    }
+
+    nicsim_cleanup();
+    return 0;
+}

corundum_bm/corundum_bm.h

+#pragma once
+
+#include <stdint.h>
+
+#define REG_A 0x0
+#define REG_B 0x4
+#define REG_C 0x10
+#define REG_D 0x14
+#define REG_E 0x20
+#define REG_F 0x24
+#define REG_G 0x28
+#define REG_H 0x2c
+#define REG_I 0x30
+#define REG_J 0x34
+#define REG_K 0x38
+#define REG_L 0x3c
+#define REG_M 0x40
+#define REG_N 0x44
+#define REG_O 0x48
+
+struct CorundumRegs {
+    uint32_t reg_a;
+    uint32_t reg_b;
+    uint32_t reg_c;
+    uint32_t reg_d;
+    uint32_t reg_e;
+    uint32_t reg_f;
+    uint32_t reg_g;
+    uint32_t reg_h;
+    uint32_t reg_i;
+    uint32_t reg_j;
+    uint32_t reg_k;
+    uint32_t reg_l;
+    uint32_t reg_m;
+    uint32_t reg_n;
+    uint32_t reg_o;
+};

corundum_bm/tester.c

+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include <signal.h>
+#include <fcntl.h>
+#include <assert.h>
+
+#include <nicsim.h>
+#include <cosim_pcie_proto.h>
+
+static uint8_t *d2h_queue;
+static size_t d2h_pos;
+static size_t d2h_elen;
+static size_t d2h_enum;
+
+static uint8_t *h2d_queue;
+static size_t h2d_pos;
+static size_t h2d_elen;
+static size_t h2d_enum;
+
+static void sigint_handler(int dummy)
+{
+    exit(1);
+}
+
+static int uxsocket_init()
+{
+    int cfd;
+
+    if ((cfd = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) {
+        return -1;
+    }
+
+    struct sockaddr_un saun;
+    memset(&saun, 0, sizeof(saun));
+    saun.sun_family = AF_UNIX;
+    memcpy(saun.sun_path, "/tmp/cosim-pci", strlen("/tmp/cosim-pci"));
+
+    if (connect(cfd, (struct sockaddr *)&saun, sizeof(saun)) == -1) {
+        close(cfd);
+        return -1;
+    }
+
+    return cfd;
+}
+
+static int queue_create(const struct cosim_pcie_proto_dev_intro di)
+{
+    int fd = -1;
+    if ((fd = open("/dev/shm/dummy_nic_shm", O_RDWR)) == -1) {
+        perror("Failed to open shm file");
+        goto error;
+    }
+
+    void *addr;
+    if ((addr = mmap(NULL, 32 * 1024 * 1024, PROT_READ | PROT_WRITE,
+                     MAP_SHARED | MAP_POPULATE, fd, 0)) == (void *)-1) {
+        perror("mmap failed");
+        goto error;
+    }
+
+    d2h_queue = (uint8_t *)addr + di.d2h_offset;
+    d2h_pos = 0;
+    d2h_elen = di.d2h_elen;
+    d2h_enum = di.d2h_nentries;
+
+    h2d_queue = (uint8_t *)addr + di.h2d_offset;
+    h2d_pos = 0;
+    h2d_elen = di.h2d_elen;
+    h2d_enum = di.h2d_nentries;
+
+    return 0;
+
+error:
+    if (fd > 0) {
+        close(fd);
+    }
+    return -1;
+}
+
+volatile union cosim_pcie_proto_h2d *h2d_alloc()
+{
+    volatile union cosim_pcie_proto_h2d *msg =
+        (volatile union cosim_pcie_proto_h2d *)
+        (h2d_queue + h2d_pos * h2d_elen);
+
+    if ((msg->dummy.own_type & COSIM_PCIE_PROTO_H2D_OWN_MASK) !=
+            COSIM_PCIE_PROTO_H2D_OWN_HOST) {
+        fprintf(stderr, "cosim: failed to allocate h2d message\n");
+        exit(1);
+    }
+
+    h2d_pos = (h2d_pos + 1) % h2d_enum;
+    return msg;
+}
+
+volatile union cosim_pcie_proto_d2h *d2h_poll()
+{
+    volatile union cosim_pcie_proto_d2h *msg;
+
+    msg = (volatile union cosim_pcie_proto_d2h *)
+        (d2h_queue + d2h_pos * d2h_elen);
+    if ((msg->dummy.own_type & COSIM_PCIE_PROTO_D2H_OWN_MASK) ==
+            COSIM_PCIE_PROTO_D2H_OWN_DEV) {
+        return NULL;
+    }
+    return msg;
+}
+
+void d2h_done(volatile union cosim_pcie_proto_d2h *msg)
+{
+    msg->dummy.own_type = (msg->dummy.own_type & COSIM_PCIE_PROTO_D2H_MSG_MASK) |
+        COSIM_PCIE_PROTO_D2H_OWN_DEV;
+    d2h_pos = (d2h_pos + 1) % d2h_enum;
+}
+
+static void dev_read(uint64_t offset, uint16_t len)
+{
+    volatile union cosim_pcie_proto_h2d *h2d_msg = h2d_alloc();
+    volatile struct cosim_pcie_proto_h2d_read *read = &h2d_msg->read;
+    read->req_id = 0xF;
+    read->offset = offset;
+    read->len = len;
+    read->bar = 0;
+    read->own_type = COSIM_PCIE_PROTO_H2D_MSG_READ | COSIM_PCIE_PROTO_H2D_OWN_DEV;
+
+    volatile union cosim_pcie_proto_d2h *d2h_msg = NULL;
+    while (d2h_msg == NULL) {
+        d2h_msg = d2h_poll();
+    }
+    volatile struct cosim_pcie_proto_d2h_readcomp *rc;
+    rc = &d2h_msg->readcomp;
+    assert(rc->req_id == 0xF);
+    printf("received readcomp with data ");
+    for (int i = 0; i < read->len; i++) {
+        printf("%x ", ((const uint8_t *)rc->data)[i]);
+    }
+    printf("\n");
+
+    d2h_done(d2h_msg);
+}
+
+int main(int argc, char *argv[])
+{
+    signal(SIGINT, sigint_handler);
+
+    int cfd;
+    if ((cfd = uxsocket_init()) < 0) {
+        fprintf(stderr, "Failed to open unix socket\n");
+        return -1;
+    }
+
+    struct cosim_pcie_proto_dev_intro di;
+    if (recv(cfd, &di, sizeof(di), 0) != sizeof(di)) {
+        perror("Failed to receive dev_intro");
+        close(cfd);
+        return -1;
+    }
+
+    if (queue_create(di) != 0) {
+        fprintf(stderr, "Failed to create shm queues\n");
+        close(cfd);
+        return -1;
+    }
+
+    struct cosim_pcie_proto_host_intro hi;
+    hi.flags = COSIM_PCIE_PROTO_FLAGS_HI_SYNC;
+    if (send(cfd, &hi, sizeof(hi), 0) != sizeof(hi)) {
+        perror("Failed to send host_intro");
+        close(cfd);
+        return -1;
+    }
+
+    while (1) {
+        int op_type;
+        uint64_t offset;
+        uint16_t len;
+        printf("op type (0-read): ");
+        scanf("%d", &op_type);
+        printf("offset: ");
+        scanf("%lx", &offset);
+        printf("len: ");
+        scanf("%hu", &len);
+        switch (op_type) {
+        case 0:
+            dev_read(offset, len);
+            break;
+        default:
+            fprintf(stderr, "Unimplemented type %u\n", op_type);
+        }
+    }
+
+    close(cfd);
+    return 0;
+}