Merge branch 'master' into corundum-bm

.gitignore

 *.o
+*.a
 _vimrc_local.vim
 dummy_nic/dummy_nic
-nicsim_common/libnicsim_common.a
 corundum/obj_dir
 corundum/corundum_verilator
+net_wire/net_wire
+net_tap/net_tap

corundum/Makefile

@@ -10,7 +10,7 @@ SRCS = corundum_verilator.cpp dma.cpp mem.cpp
 all: corundum_verilator
 
 obj_dir/Vinterface.cpp: rtl/interface.v
-	$(VERILATOR) $(VFLAGS) --cc -O3  \
+	$(VERILATOR) $(VFLAGS) --cc -O3 \
 	    -CFLAGS "-I$(PWD)/../nicsim_common/include -I$(PWD)/../proto -O3 -g -Wall" \
 	    -y rtl \
 	    -y lib/axi/rtl \
@@ -39,7 +39,9 @@ corundum_verilator: obj_dir/Vinterface
 #	    obj_dir/Vinterface__Trace__Slow.o
 #	$(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $^ $(LDLIBS)
 
+lightclean:
+	rm -f obj_dir/Vinterface corundum_verilator obj_dir/mem.o obj_dir/dma.o
 clean:
 	rm -rf obj_dir corundum_verilator *.o
 
-.PHONY: all clean
+.PHONY: all clean lightclean

corundum/coord.h

+#ifndef COORD_H_
+#define COORD_H_
+
+#include <deque>
+#include <map>
+#include <iostream>
+
+#include "debug.h"
+
+class DMAOp;
+struct MMIOOp;
+
+void pci_dma_issue(DMAOp *op);
+void pci_msi_issue(uint8_t vec);
+void pci_rwcomp_issue(MMIOOp *op);
+
+class PCICoordinator {
+    protected:
+        struct PCIOp {
+            union {
+                DMAOp *dma_op;
+                MMIOOp *mmio_op;
+                uint32_t msi_vec;
+            };
+            enum {
+                OP_DMA,
+                OP_MSI,
+                OP_RWCOMP,
+            } type;
+            bool ready;
+        };
+
+        std::deque<PCIOp *> queue;
+        std::map<DMAOp *, PCIOp *> dmamap;
+
+        void process()
+        {
+            PCIOp *op;
+            while (!queue.empty()) {
+                op = queue.front();
+                if (!op->ready)
+                    break;
+
+                queue.pop_front();
+                if (op->type == PCIOp::OP_MSI) {
+#ifdef COORD_DEBUG
+                    std::cout << "issuing msi " << op->msi_vec << std::endl;
+#endif
+                    pci_msi_issue(op->msi_vec);
+                } else if (op->type == PCIOp::OP_DMA) {
+#ifdef COORD_DEBUG
+                    std::cout << "issuing dma " << op->dma_op << std::endl;
+#endif
+                    pci_dma_issue(op->dma_op);
+                    dmamap.erase(op->dma_op);
+                } else if (op->type == PCIOp::OP_RWCOMP) {
+#ifdef COORD_DEBUG
+                    std::cout << "issuing mmio " << op->mmio_op << std::endl;
+#endif
+                    pci_rwcomp_issue(op->mmio_op);
+                } else {
+                    throw "unknown type";
+                }
+
+                delete op;
+            }
+        }
+
+    public:
+        void dma_register(DMAOp *dma_op, bool ready)
+        {
+#ifdef COORD_DEBUG
+            std::cout << "registering dma op " << dma_op << "  " << ready << std::endl;
+#endif
+            PCIOp *op = new PCIOp;
+            op->dma_op = dma_op;
+            op->type = PCIOp::OP_DMA;
+            op->ready = ready;
+
+            queue.push_back(op);
+            dmamap[dma_op] = op;
+
+            process();
+        }
+
+        void dma_mark_ready(DMAOp *op)
+        {
+#ifdef COORD_DEBUG
+            std::cout << "readying dma op " << op <<  std::endl;
+#endif
+            dmamap[op]->ready = true;
+
+            process();
+        }
+
+        void msi_enqueue(uint32_t vec)
+        {
+#ifdef COORD_DEBUG
+            std::cout << "enqueuing MSI " << vec <<  std::endl;
+#endif
+            PCIOp *op = new PCIOp;
+            op->msi_vec = vec;
+            op->type = PCIOp::OP_MSI;
+            op->ready = true;
+            queue.push_back(op);
+
+            process();
+        }
+
+        void mmio_comp_enqueue(MMIOOp *mmio_op)
+        {
+#ifdef COORD_DEBUG
+            std::cout << "enqueuing MMIO comp " << mmio_op <<  std::endl;
+#endif
+            PCIOp *op = new PCIOp;
+            op->mmio_op = mmio_op;
+            op->type = PCIOp::OP_RWCOMP;
+            op->ready = true;
+            queue.push_back(op);
+
+            process();
+
+        }
+};
+
+
+
+#endif

corundum/corundum_verilator.cpp

@@ -9,9 +9,13 @@ extern "C" {
 
 #include "Vinterface.h"
 #include "verilated.h"
+#ifdef TRACE_ENABLED
 #include "verilated_vcd_c.h"
+#endif
 
+#include "debug.h"
 #include "corundum.h"
+#include "coord.h"
 #include "dma.h"
 #include "mem.h"
 
@@ -25,7 +29,9 @@ static volatile int exiting = 0;
 static uint64_t main_time = 0;
 static uint64_t pci_last_time = 0;
 static uint64_t eth_last_time = 0;
-//static VerilatedVcdC* trace;
+#ifdef TRACE_ENABLED
+static VerilatedVcdC* trace;
+#endif
 
 
 
@@ -169,15 +175,16 @@ static void report_outputs(Vinterface *top)
     report_output("msi_irq", top->msi_irq);
 }
 
+struct MMIOOp {
+    uint64_t id;
+    uint64_t addr;
+    uint64_t value;
+    size_t len;
+    bool isWrite;
+};
+
 class MMIOInterface {
     protected:
-        struct Op {
-            uint64_t id;
-            uint64_t addr;
-            uint64_t value;
-            size_t len;
-            bool isWrite;
-        };
 
         enum OpState {
             AddrIssued,
@@ -186,13 +193,14 @@ class MMIOInterface {
         };
 
         Vinterface ⊤
-        std::deque<Op *> queue;
-        Op *rCur, *wCur;
+        PCICoordinator &coord;
+        std::deque<MMIOOp *> queue;
+        MMIOOp *rCur, *wCur;
         enum OpState rState, wState;
 
     public:
-        MMIOInterface(Vinterface &top_)
-            : top(top_), rCur(0), wCur(0)
+        MMIOInterface(Vinterface &top_,  PCICoordinator &coord_)
+            : top(top_), coord(coord_), rCur(0), wCur(0)
         {
         }
 
@@ -210,8 +218,11 @@ class MMIOInterface {
                     /* read data received */
                     top.s_axil_rready = 0;
                     rCur->value = top.s_axil_rdata;
-                    completeRead(*rCur);
-                    delete rCur;
+                    coord.mmio_comp_enqueue(rCur);
+#ifdef MMIO_DEBUG
+                    std::cout << "MMIO: completed AXI read op=" << rCur << " val=" <<
+                        rCur->value << std::endl;
+#endif
                     rCur = 0;
                 }
             } else if (wCur) {
@@ -231,14 +242,18 @@ class MMIOInterface {
                 if (wState == AddrDone && top.s_axil_bvalid) {
                     /* write complete */
                     top.s_axil_bready = 0;
-                    completeWrite(*wCur, top.s_axil_bresp);
-                    delete wCur;
+                    // TODO: check top.s_axil_bresp
+#ifdef MMIO_DEBUG
+                    std::cout << "MMIO: completed AXI write op=" << wCur <<
+                        std::endl;
+#endif
+                    coord.mmio_comp_enqueue(wCur);
                     wCur = 0;
                 }
             } else if (/*!top.clk &&*/ !queue.empty()) {
                 /* issue new operation */
 
-                Op *op = queue.front();
+                MMIOOp *op = queue.front();
                 queue.pop_front();
                 if (!op->isWrite) {
                     /* issue new read */
@@ -270,7 +285,11 @@ class MMIOInterface {
 
         void issueRead(uint64_t id, uint64_t addr, size_t len)
         {
-            Op *op = new Op;
+            MMIOOp *op = new MMIOOp;
+#ifdef MMIO_DEBUG
+            std::cout << "MMIO: read id=" << id << " addr=" << std::hex << addr
+                << " len=" << len << " op=" << op << std::endl;
+#endif
             op->id = id;
             op->addr = addr;
             op->len = len;
@@ -278,29 +297,14 @@ class MMIOInterface {
             queue.push_back(op);
         }
 
-        void completeRead(Op &op)
-        {
-            volatile union cosim_pcie_proto_d2h *msg = nicsim_d2h_alloc();
-            volatile struct cosim_pcie_proto_d2h_readcomp *rc;
-
-            if (!msg)
-                throw "completion alloc failed";
-
-            rc = &msg->readcomp;
-            memcpy((void *) rc->data, &op.value, op.len);
-            rc->req_id = op.id;
-
-            //WMB();
-            rc->own_type = COSIM_PCIE_PROTO_D2H_MSG_READCOMP |
-                COSIM_PCIE_PROTO_D2H_OWN_HOST;
-
-            //std::cout << "read complete addr=" << op.addr << " val=" << op.value << std::endl;
-        }
-
 
         void issueWrite(uint64_t id, uint64_t addr, size_t len, uint64_t val)
         {
-            Op *op = new Op;
+            MMIOOp *op = new MMIOOp;
+#ifdef MMIO_DEBUG
+            std::cout << "MMIO: write id=" << id << " addr=" << std::hex << addr
+                << " len=" << len << " val=" << val << " op=" << op << std::endl;
+#endif
             op->id = id;
             op->addr = addr;
             op->len = len;
@@ -309,25 +313,36 @@ class MMIOInterface {
             queue.push_back(op);
         }
 
-        void completeWrite(Op &op, uint8_t status)
-        {
-            volatile union cosim_pcie_proto_d2h *msg = nicsim_d2h_alloc();
-            volatile struct cosim_pcie_proto_d2h_writecomp *rc;
+};
 
-            if (!msg)
-                throw "completion alloc failed";
+void pci_rwcomp_issue(MMIOOp *op)
+{
+    volatile union cosim_pcie_proto_d2h *msg = nicsim_d2h_alloc();
+    volatile struct cosim_pcie_proto_d2h_readcomp *rc;
+    volatile struct cosim_pcie_proto_d2h_writecomp *wc;
 
-            rc = &msg->writecomp;
-            rc->req_id = op.id;
+    if (!msg)
+        throw "completion alloc failed";
 
-            //WMB();
-            rc->own_type = COSIM_PCIE_PROTO_D2H_MSG_WRITECOMP |
-                COSIM_PCIE_PROTO_D2H_OWN_HOST;
+    if (op->isWrite) {
+        wc = &msg->writecomp;
+        wc->req_id = op->id;
 
-            //std::cout << "write complete addr=" << op.addr << " val=" << op.value << std::endl;
-        }
+        //WMB();
+        wc->own_type = COSIM_PCIE_PROTO_D2H_MSG_WRITECOMP |
+            COSIM_PCIE_PROTO_D2H_OWN_HOST;
+    } else {
+        rc = &msg->readcomp;
+        memcpy((void *) rc->data, &op->value, op->len);
+        rc->req_id = op->id;
 
-};
+        //WMB();
+        rc->own_type = COSIM_PCIE_PROTO_D2H_MSG_READCOMP |
+            COSIM_PCIE_PROTO_D2H_OWN_HOST;
+    }
+
+    delete op;
+}
 
 #if 0
 class MemAccessor {
@@ -408,6 +423,7 @@ void pci_dma_issue(DMAOp *op)
     pci_dma_pending.insert(op);
 }
 
+
 static void h2d_readcomp(volatile struct cosim_pcie_proto_h2d_readcomp *rc)
 {
     DMAOp *op = (DMAOp *) (uintptr_t) rc->req_id;
@@ -591,11 +607,13 @@ class EthernetTx {
             send->own_type = COSIM_ETH_PROTO_D2N_MSG_SEND |
                 COSIM_ETH_PROTO_D2N_OWN_NET;
 
+#ifdef ETH_DEBUG
             std::cerr << "EthernetTx: packet len=" << std::hex << packet_len << " ";
             for (size_t i = 0; i < packet_len; i++) {
                 std::cerr << (unsigned) packet_buf[i] << " ";
             }
             std::cerr << std::endl;
+#endif
         }
 
         void step()
@@ -622,59 +640,83 @@ class EthernetTx {
 class EthernetRx {
     protected:
         Vinterface &top;
-        uint8_t packet_buf[2048];
-        size_t packet_len;
+
+        static const size_t FIFO_SIZE = 32;
+        uint8_t fifo_bufs[FIFO_SIZE][2048];
+        size_t fifo_lens[FIFO_SIZE];
+        size_t fifo_pos_rd;
+        size_t fifo_pos_wr;
+
+
         size_t packet_off;
 
     public:
         EthernetRx(Vinterface &top_)
-            : top(top_), packet_len(0), packet_off(0)
+            : top(top_), fifo_pos_rd(0), fifo_pos_wr(0), packet_off(0)
         {
+            for (size_t i = 0; i < FIFO_SIZE; i++)
+                fifo_lens[i] = 0;
         }
 
         void packet_received(const void *data, size_t len)
         {
-            if (packet_len != 0) {
+            if (fifo_lens[fifo_pos_wr] != 0) {
                 std::cerr << "EthernetRx: dropping packet" << std::endl;
                 return;
             }
 
-            packet_off = 0;
-            packet_len = len;
-            memcpy(packet_buf, data, len);
+            memcpy(fifo_bufs[fifo_pos_wr], data, len);
+            fifo_lens[fifo_pos_wr] = len;
 
-            std::cerr << "EthernetRx: packet len=" << std::hex << packet_len << " ";
-            for (size_t i = 0; i < packet_len; i++) {
-                std::cerr << (unsigned) packet_buf[i] << " ";
+#ifdef ETH_DEBUG
+            std::cout << "rx into " << fifo_pos_wr << std::endl;
+            std::cerr << "EthernetRx: packet len=" << std::hex << len << " ";
+            for (size_t i = 0; i < len; i++) {
+                std::cerr << (unsigned) fifo_bufs[fifo_pos_wr][i] << " ";
             }
             std::cerr << std::endl;
+#endif
+
+            fifo_pos_wr = (fifo_pos_wr + 1) % FIFO_SIZE;
         }
 
         void step()
         {
-            if (packet_len != 0) {
+            if (fifo_lens[fifo_pos_rd] != 0) {
                 // we have data to send
                 if (packet_off != 0 && !top.rx_axis_tready) {
                     // no ready signal, can't advance
-                } else if (packet_off == packet_len) {
+                    std::cerr << "eth rx: no ready" << std::endl;
+                } else if (packet_off == fifo_lens[fifo_pos_rd]) {
                     // done with packet
+#ifdef ETH_DEBUG
                     std::cerr << "EthernetRx: finished packet" << std::endl;
+#endif
                     top.rx_axis_tvalid = 0;
                     top.rx_axis_tlast = 0;
-                    packet_off = packet_len = 0;
+
+                    packet_off = 0;
+                    fifo_lens[fifo_pos_rd] = 0;
+                    fifo_pos_rd = (fifo_pos_rd + 1) % FIFO_SIZE;
                 } else {
                     // put out more packet data
+#ifdef ETH_DEBUG
                     std::cerr << "EthernetRx: push flit " << packet_off << std::endl;
+                    if (packet_off == 0)
+                        std::cout << "rx from " << fifo_pos_rd << std::endl;
+#endif
                     top.rx_axis_tkeep = 0;
                     top.rx_axis_tdata = 0;
-                    for (size_t i = 0; i < 8 && packet_off < packet_len; i++) {
+                    size_t i;
+                    for (i = 0; i < 8 && packet_off < fifo_lens[fifo_pos_rd]; i++) {
                         top.rx_axis_tdata |=
-                            ((uint64_t) packet_buf[packet_off]) << (i * 8);
+                            ((uint64_t) fifo_bufs[fifo_pos_rd][packet_off]) <<
+                            (i * 8);
                         top.rx_axis_tkeep |= (1 << i);
                         packet_off++;
                     }
                     top.rx_axis_tvalid = 1;
-                    top.rx_axis_tlast = (packet_off == packet_len);
+                    top.rx_axis_tlast = (packet_off == fifo_lens[fifo_pos_rd]);
                 }
                 //trace->dump(main_time);
             } else {
@@ -714,12 +756,89 @@ static void poll_n2d(EthernetRx &rx)
     nicif_n2d_next();
 }
 
-static void msi_issue(uint8_t vec)
+#if 0
+class PCICoordinator {
+    protected:
+        struct PCIOp {
+            union {
+                DMAOp *dma_op;
+                uint32_t msi_vec;
+            };
+            bool isDma;
+            bool ready;
+        };
+
+        Vinterface &top;
+        std::deque<PCIOp *> queue;
+        std::map<DMAOp *, PCIOp *> dmamap;
+
+        void process()
+        {
+            PCIOp *op;
+            while (queue.empty()) {
+                op = queue.front();
+                if (!op->ready)
+                    break;
+
+                queue.pop_front();
+                if (!op->isDma) {
+                    pci_msi_issue(op->msi_vec);
+                    delete op;
+                } else {
+                    pci_dma_issue(op->dma_op);
+                    dmamap.erase(op->dma_op);
+                    delete op;
+                }
+            }
+        }
+
+    public:
+        PCICoordinator(Vinterface &top_)
+            : top(top_)
+        {
+        }
+
+        void dma_register(DMAOp *dma_op, bool ready)
+        {
+            PCIOp *op = new PCIOp;
+            op->dma_op = vec;
+            op->isDma = true;
+            op->ready = ready;
+
+            queue.push_back(op);
+            dmamap[op] = dma_op;
+
+            process();
+        }
+
+        void dma_mark_ready(DMAOp *op)
+        {
+            dmamap[op]->ready = true;
+
+            process();
+        }
+
+        void msi_enqueue(uint32_t vec)
+        {
+            PCIOp *op = new PCIOp;
+            op->msi_vec = vec;
+            op->isDma = false;
+            op->ready = true;
+            queue.push_back(op);
+
+            process();
+        }
+};
+#endif
+
+void pci_msi_issue(uint8_t vec)
 {
     volatile union cosim_pcie_proto_d2h *msg = nicsim_d2h_alloc();
     volatile struct cosim_pcie_proto_d2h_interrupt *intr;
 
+#ifdef MSI_DEBUG
     std::cerr << "MSI interrupt vec=" << (int) vec << std::endl;
+#endif
 
     intr = &msg->interrupt;
     intr->vector = vec;
@@ -730,16 +849,19 @@ static void msi_issue(uint8_t vec)
         COSIM_PCIE_PROTO_D2H_OWN_HOST;
 }
 
-static void msi_step(Vinterface &top)
+
+static void msi_step(Vinterface &top, PCICoordinator &coord)
 {
     if (!top.msi_irq)
         return;
 
+#ifdef MSI_DEBUG
+    std::cerr << "msi_step: MSI interrupt raw vec=" << (int) top.msi_irq << std::endl;
+#endif
     for (size_t i = 0; i < 32; i++) {
         if (!((1ULL << i) & top.msi_irq))
             continue;
-
-        msi_issue(i);
+        coord.msi_enqueue(i);
     }
 }
 
@@ -769,8 +891,17 @@ static void sync_eth(EthernetRx &rx)
 
 int main(int argc, char *argv[])
 {
-    Verilated::commandArgs(argc, argv);
+    char *vargs[2] = { argv[0], NULL };
+    Verilated::commandArgs(1, vargs);
+#ifdef TRACE_ENABLED
     Verilated::traceEverOn(true);
+#endif
+
+    if (argc != 4) {
+        fprintf(stderr, "Usage: corundum_verilator PCI-SOCKET ETH-SOCKET "
+                "SHM\n");
+        return EXIT_FAILURE;
+    }
 
     int sync_pci_en, sync_eth_en;
     struct cosim_pcie_proto_dev_intro di;
@@ -788,8 +919,8 @@ int main(int argc, char *argv[])
 
     sync_pci_en = 1;
     sync_eth_en = 1;
-    if (nicsim_init(&di, "/tmp/cosim-pci", &sync_pci_en,
-                "/tmp/cosim-eth", &sync_eth_en, "/dev/shm/dummy_nic_shm"))
+    if (nicsim_init(&di, argv[1], &sync_pci_en, argv[2], &sync_eth_en,
+                argv[3]))
     {
         return EXIT_FAILURE;
     }
@@ -800,9 +931,11 @@ int main(int argc, char *argv[])
 
 
     Vinterface *top = new Vinterface;
-    /*trace = new VerilatedVcdC;
+#ifdef TRACE_ENABLED
+    trace = new VerilatedVcdC;
     top->trace(trace, 99);
-    trace->open("debug.vcd");*/
+    trace->open("debug.vcd");
+#endif
 
     MemWritePort p_mem_write_ctrl_dma(
             top->ctrl_dma_ram_wr_cmd_sel,
@@ -876,18 +1009,18 @@ int main(int argc, char *argv[])
             top->s_axis_data_dma_write_desc_status_tag,
             top->s_axis_data_dma_write_desc_status_valid);
 
-
-    MMIOInterface mmio(*top);
+    PCICoordinator pci_coord;
+    MMIOInterface mmio(*top, pci_coord);
 
     MemWriter mem_control_writer(p_mem_write_ctrl_dma);
     MemReader mem_control_reader(p_mem_read_ctrl_dma);
     MemWriter mem_data_writer(p_mem_write_data_dma);
     MemReader mem_data_reader(p_mem_read_data_dma);
 
-    DMAReader dma_read_ctrl("read ctrl", p_dma_read_ctrl, mem_control_writer);
-    DMAWriter dma_write_ctrl("write ctrl", p_dma_write_ctrl, mem_control_reader);
-    DMAReader dma_read_data("read data", p_dma_read_data, mem_data_writer);
-    DMAWriter dma_write_data("write data", p_dma_write_data, mem_data_reader);
+    DMAReader dma_read_ctrl("read ctrl", p_dma_read_ctrl, mem_control_writer, pci_coord);
+    DMAWriter dma_write_ctrl("write ctrl", p_dma_write_ctrl, mem_control_reader, pci_coord);
+    DMAReader dma_read_data("read data", p_dma_read_data, mem_data_writer, pci_coord);
+    DMAWriter dma_write_data("write data", p_dma_write_data, mem_data_reader, pci_coord);
 
     EthernetTx tx(*top);
     EthernetRx rx(*top);
@@ -930,7 +1063,7 @@ int main(int argc, char *argv[])
         tx.step();
         rx.step();
 
-        msi_step(*top);
+        msi_step(*top, pci_coord);
 
         /* raising edge */
         top->clk = !top->clk;
@@ -944,7 +1077,10 @@ int main(int argc, char *argv[])
     }
     report_outputs(top);
 
-    //trace->close();
+#ifdef TRACE_ENABLED
+    trace->dump(main_time + 1);
+    trace->close();
+#endif
     top->final();
     delete top;
     return 0;

corundum/debug.h

+//#define COORD_DEBUG 1
+//#define ETH_DEBUG 1
+//#define MSI_DEBUG 1
+//#define DMA_DEBUG 1
+//#define MEM_DEBUG 1
+//#define MMIO_DEBUG 1

corundum/dma.cpp

 #include <iostream>
 
+#include "debug.h"
 #include "corundum.h"
 #include "dma.h"
 #include "mem.h"
 
+
 void DMAReader::step()
 {
     p.dma_ready = 1;
@@ -17,11 +19,14 @@ void DMAReader::step()
         op->tag = p.dma_tag;
         op->write = false;
         pending.insert(op);
-        /*std::cout << "dma[" << label << "] op " << op->dma_addr << " -> " <<
+
+#ifdef DMA_DEBUG
+        std::cout << "dma[" << label << "] op " << std::hex << op->dma_addr << " -> " <<
             op->ram_sel << ":" << op->ram_addr <<
-            "   len=" << op->len << "   tag=" << (int) op->tag << std::endl;*/
+            "   len=" << op->len << "   tag=" << (int) op->tag << std::endl;
+#endif
 
-        pci_dma_issue(op);
+        coord.dma_register(op, true);
     }
 
     p.dma_status_valid = 0;
@@ -64,10 +69,14 @@ void DMAWriter::step()
         op->tag = p.dma_tag;
         op->write = true;
         pending.insert(op);
-        std::cout << "dma write [" << label << "] op " << op->dma_addr << " -> " <<
+
+#ifdef DMA_DEBUG
+        std::cout << "dma write [" << label << "] op " << std::hex << op->dma_addr << " -> " <<
             op->ram_sel << ":" << op->ram_addr <<
             "   len=" << op->len << "   tag=" << (int) op->tag << std::endl;
+#endif
 
+        coord.dma_register(op, false);
         mr.op_issue(op);
     }
 
@@ -76,23 +85,33 @@ void DMAWriter::step()
         DMAOp *op = completed.front();
         completed.pop_front();
 
+#ifdef DMA_DEBUG
         std::cout << "dma write [" << label << "] status complete " << op->dma_addr << std::endl;
+#endif
 
         p.dma_status_valid = 1;
         p.dma_status_tag = op->tag;
         pending.erase(op);
+        //coord.msi_enqueue(0);
         delete op;
     }
 }
 
 void DMAWriter::pci_op_complete(DMAOp *op)
 {
+#ifdef DMA_DEBUG
     std::cout << "dma write [" << label << "] pci complete " << op->dma_addr << std::endl;
+#endif
     completed.push_back(op);
 }
 
 void DMAWriter::mem_op_complete(DMAOp *op)
 {
-    std::cout << "dma write [" << label << "] mem complete " << op->dma_addr << std::endl;
-    pci_dma_issue(op);
+#ifdef DMA_DEBUG
+    std::cout << "dma write [" << label << "] mem complete " << op->dma_addr << ": ";
+    for (size_t i = 0; i < op->len; i++)
+        std::cout << (unsigned) op->data[i] << " ";
+    std::cout << std::endl;
+#endif
+    coord.dma_mark_ready(op);
 }

corundum/dma.h

@@ -7,6 +7,9 @@
 #include "Vinterface.h"
 #include "verilated.h"
 
+#include "debug.h"
+#include "coord.h"
+
 #define MAX_DMA_LEN 2048
 
 class DMAEngine;
@@ -56,9 +59,10 @@ struct DMAOp {
 class DMAEngine {
     protected:
         DMAPorts &p;
+        PCICoordinator &coord;
 
-        DMAEngine(DMAPorts &p_)
-            : p(p_) { }
+        DMAEngine(DMAPorts &p_, PCICoordinator &coord_)
+            : p(p_), coord(coord_) { }
 
     public:
         virtual void pci_op_complete(DMAOp *op) = 0;
@@ -73,8 +77,9 @@ class DMAReader : public DMAEngine {
         MemWriter &mw;
 
     public:
-        DMAReader(const char *label_, DMAPorts &p_, MemWriter &mw_)
-            : DMAEngine(p_), label(label_), mw(mw_)
+        DMAReader(const char *label_, DMAPorts &p_, MemWriter &mw_,
+                PCICoordinator &coord_)
+            : DMAEngine(p_, coord_), label(label_), mw(mw_)
         {
         }
 
@@ -91,8 +96,9 @@ class DMAWriter : public DMAEngine {
         MemReader &mr;
 
     public:
-        DMAWriter(const char *label_, DMAPorts &p_, MemReader &mr_)
-            : DMAEngine(p_), label(label_), mr(mr_)
+        DMAWriter(const char *label_, DMAPorts &p_, MemReader &mr_,
+                PCICoordinator &coord_)
+            : DMAEngine(p_, coord_), label(label_), mr(mr_)
         {
         }
 

corundum/mem.cpp

 #include <iostream>
 
+#include "debug.h"
 #include "mem.h"
 #include "dma.h"
 
@@ -17,8 +18,12 @@
 
 void MemWriter::step()
 {
-    if (cur && p.mem_ready) {
-        //std::cerr << "completed write to: " << cur->ram_addr << std::endl;
+    if (cur && p.mem_ready &&
+            ((p.mem_ready & p.mem_valid) == p.mem_valid))
+    {
+#ifdef MEM_DEBUG
+        std::cerr << "completed write to: " << cur->ram_addr << std::endl;
+#endif
         p.mem_valid = 0;
         p.mem_be[0] = p.mem_be[1] = p.mem_be[2] = p.mem_be[3] = 0;
 
@@ -34,21 +39,20 @@ void MemWriter::step()
     } else if (!cur && !pending.empty()) {
         cur = pending.front();
 
-        //std::cerr << "issuing write to " << cur->ram_addr << std::endl;
+#ifdef MEM_DEBUG
+        std::cerr << "issuing write to " << cur->ram_addr << std::endl;
+#endif
 
         size_t data_byte_width = DATA_WIDTH / 8;
         size_t data_offset = (cur->ram_addr + cur_off) % data_byte_width;
 
-        /*if (cur->len > data_byte_width - data_offset) {
-            std::cerr << "MemWriter::step: cannot be written in one cycle TODO" << std::endl;
-            throw "unsupported";
-        }*/
-
         /* first reset everything */
         p.mem_sel = 0;
         p.mem_addr[0] = p.mem_addr[1] = p.mem_addr[2] = 0;
         p.mem_be[0] = p.mem_be[1] = p.mem_be[2] = p.mem_be[3] = 0;
         p.mem_valid = 0;
+        for (size_t i = 0; i < data_byte_width / 4; i++)
+            p.mem_data[i] = 0;
 
 
         /* put data bytes in the right places */
@@ -57,10 +61,7 @@ void MemWriter::step()
                 data_byte_width - data_offset : cur->len - cur_off);
         for (size_t i = 0; i < cur_len; i++, off++) {
             size_t byte_off = off % 4;
-            // first clear data byte
-            p.mem_data[off / 4] &= ~(0xffu << (byte_off * 8));
-            // then set data byte
-            p.mem_data[off / 4] |= (((uint32_t) cur->data[i]) << (byte_off * 8));
+            p.mem_data[off / 4] |= (((uint32_t) cur->data[cur_off + i]) << (byte_off * 8));
             p.mem_be[off / 32] |= (1 << (off % 32));
             p.mem_valid |= (1 << (off / (SEG_WIDTH / 8)));
         }
@@ -84,7 +85,9 @@ void MemWriter::step()
 
 void MemWriter::op_issue(DMAOp *op)
 {
-    //std::cerr << "enqueued write to " << op->ram_addr << std::endl;
+#ifdef MEM_DEBUG
+    std::cerr << "enqueued write to " << op->ram_addr << std::endl;
+#endif
     pending.push_back(op);
 }
 
@@ -95,16 +98,26 @@ void MemReader::step()
 {
     size_t data_byte_width = DATA_WIDTH / 8;
 
-    if (cur && p.mem_resvalid) {
+    if (cur && p.mem_resvalid &&
+            ((p.mem_resvalid & p.mem_valid) == p.mem_valid)) {
+#ifdef MEM_DEBUG
         std::cerr << "completed read from: " << std::hex << cur->ram_addr << std::endl;
+        std::cerr << "  reval = " << (unsigned) p.mem_resvalid << std::endl;
+#endif
         p.mem_valid = 0;
-        /*for (size_t i = 0; i < 32; i++)
-            std::cerr << "    val = " << p.mem_data[i] << std::endl;*/
-        size_t off = cur->ram_addr % data_byte_width;
-        for (size_t i = 0; i < cur->len; i++, off++) {
+#ifdef MEM_DEBUG
+        for (size_t i = 0; i < 32; i++)
+            std::cerr << "    val = " << p.mem_data[i] << std::endl;
+#endif
+
+        size_t off = (cur->ram_addr + cur_off) % data_byte_width;
+        size_t cur_len = (cur->len - cur_off > data_byte_width - off ?
+                data_byte_width - off : cur->len - cur_off);
+        for (size_t i = 0; i < cur_len; i++, off++) {
             size_t byte_off = (off % 4);
-            cur->data[i] = (p.mem_data[off / 4] >> (byte_off * 8)) & 0xff;
+            cur->data[cur_off + i] = (p.mem_data[off / 4] >> (byte_off * 8)) & 0xff;
         }
+        cur_off += cur_len;
 
         if (cur_off == cur->len) {
             /* operation is done */
@@ -118,16 +131,12 @@ void MemReader::step()
         cur = 0;
     } else if (!cur && !pending.empty()) {
         cur = pending.front();
-
-        std::cerr << "issuing read from " << std::hex << cur->ram_addr << std::endl;
-
         size_t data_offset = (cur->ram_addr + cur_off) % data_byte_width;
-        std::cerr << "    off=" << data_offset << std::endl;
 
-        /*if (cur->len > data_byte_width - data_offset) {
-            std::cerr << "MemReader::step: cannot be written in one cycle TODO" << std::endl;
-            throw "unsupported";
-        }*/
+#ifdef MEM_DEBUG
+        std::cerr << "issuing op=" << cur << " read from " << std::hex << cur->ram_addr << std::endl;
+        std::cerr << "    off=" << data_offset << std::endl;
+#endif
 
         /* first reset everything */
         p.mem_sel = 0;
@@ -140,10 +149,10 @@ void MemReader::step()
         size_t cur_len = (cur->len - cur_off > data_byte_width - data_offset ?
                 data_byte_width - data_offset : cur->len - cur_off);
         for (size_t i = 0; i < cur_len; i++, off++) {
-            size_t byte_off = off % 4;
             p.mem_valid |= (1 << (off / (SEG_WIDTH / 8)));
         }
-        p.mem_resready = p.mem_valid;
+        //p.mem_resready = p.mem_valid;
+        p.mem_resready = 0xff;
 
         uint64_t seg_addr = (cur->ram_addr + cur_off) / data_byte_width;
         size_t seg_addr_bits = 12;
@@ -158,16 +167,19 @@ void MemReader::step()
             }
         }
 
-        /*for (size_t i = 0; i < 3; i++)
+#ifdef MEM_DEBUG
+        for (size_t i = 0; i < 3; i++)
             std::cerr << "    addr = " << p.mem_addr[i] << std::endl;
-        std::cerr << "    mem_valid = " << (unsigned) p.mem_valid << std::endl;*/
+        std::cerr << "    mem_valid = " << (unsigned) p.mem_valid << std::endl;
+#endif
 
-        cur_off += cur_len;
     }
 }
 
 void MemReader::op_issue(DMAOp *op)
 {
+#ifdef MEM_DEBUG
     std::cerr << "enqueued read from " << op->ram_addr << std::endl;
+#endif
     pending.push_back(op);
 }

corundum/rtl/interface.v

@@ -103,7 +103,7 @@ module interface #
     // Log desc block size field width
     parameter LOG_BLOCK_SIZE_WIDTH = 2,  /* this one is not explicitly set */
     // Enable PTP timestamping
-    parameter PTP_TS_ENABLE = 1,
+    parameter PTP_TS_ENABLE = 0,
     // PTP timestamp width
     parameter PTP_TS_WIDTH = 96,
     // Enable TX checksum offload

net_tap/Makefile

-CPPFLAGS += -I../proto
+CPPFLAGS += -I../proto -I../netsim_common/include
 CFLAGS += -Wall -Wextra -Wno-unused-parameter -O3
 LDLIBS += -lpthread
 
-net_tap: net_tap.o
+net_tap: net_tap.o ../netsim_common/libnetsim_common.a
 
 all: net_tap
 

net_tap/net_tap.c

@@ -28,109 +28,17 @@
 #include <string.h>
 #include <sys/ioctl.h>
 #include <sys/mman.h>
-#include <sys/un.h>
-#include <sys/socket.h>
-#include <sys/stat.h>
 #include <unistd.h>
 #include <linux/if.h>
 #include <linux/if_tun.h>
 
-#include <cosim_eth_proto.h>
+#include <netsim.h>
 
-static uint8_t *d2n_queue;
-static size_t d2n_pos;
-static size_t d2n_elen;
-static size_t d2n_enum;
-
-static uint8_t *n2d_queue;
-static size_t n2d_pos;
-static size_t n2d_elen;
-static size_t n2d_enum;
+//#define DEBUG_PKTMETA
 
+static struct netsim_interface nsif;
 static int tap_fd;
 
-int uxsocket_connect(const char *path)
-{
-    int fd;
-    struct sockaddr_un saun;
-
-    /* prepare and connect socket */
-    memset(&saun, 0, sizeof(saun));
-    saun.sun_family = AF_UNIX;
-    strcpy(saun.sun_path, path);
-
-    if ((fd = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) {
-        perror("socket failed");
-        return -1;
-    }
-
-    if (connect(fd, (struct sockaddr *) &saun, sizeof(saun)) != 0) {
-        perror("connect failed");
-        return -1;
-    }
-
-    return fd;
-}
-
-int uxsocket_recv(int fd, void *data, size_t len, int *pfd)
-{
-    int *ppfd;
-    ssize_t ret;
-    struct cmsghdr *cmsg;
-    union {
-        char buf[CMSG_SPACE(sizeof(int))];
-        struct cmsghdr align;
-    } u;
-    struct iovec iov = {
-        .iov_base = data,
-        .iov_len = len,
-    };
-    struct msghdr msg = {
-        .msg_name = NULL,
-        .msg_namelen = 0,
-        .msg_iov = &iov,
-        .msg_iovlen = 1,
-        .msg_control = u.buf,
-        .msg_controllen = sizeof(u.buf),
-        .msg_flags = 0,
-    };
-
-    if ((ret = recvmsg(fd, &msg, 0)) != (ssize_t) len) {
-        perror("recvmsg failed");
-        return -1;
-    }
-
-    cmsg = CMSG_FIRSTHDR(&msg);
-    ppfd = (int *) CMSG_DATA(cmsg);
-    if (msg.msg_controllen <= 0 || cmsg->cmsg_len != CMSG_LEN(sizeof(int))) {
-        fprintf(stderr, "accessing ancillary data failed\n");
-        return -1;
-    }
-
-    *pfd = *ppfd;
-    return 0;
-}
-
-void *shm_map(int shm_fd)
-{
-    void *p;
-    struct stat statbuf;
-
-    if (fstat(shm_fd, &statbuf) != 0) {
-        perror("shm_map: fstat failed");
-        return NULL;
-    }
-
-    p = mmap(NULL, statbuf.st_size, PROT_READ | PROT_WRITE, MAP_SHARED, shm_fd,
-            0);
-    if (p == MAP_FAILED) {
-        perror("shm_map: mmap failed");
-        return NULL;
-    }
-
-    return p;
-}
-
 static int tap_open(const char *name)
 {
     struct ifreq ifr;
@@ -156,7 +64,9 @@ static int tap_open(const char *name)
 
 static void d2n_send(volatile struct cosim_eth_proto_d2n_send *s)
 {
+#ifdef DEBUG_PKTMETA
     printf("sent packet: len=%u\n", s->len);
+#endif
 
     if (write(tap_fd, (void *) s->data, s->len) != (ssize_t) s->len) {
         perror("d2n_send: send failed");
@@ -165,14 +75,11 @@ static void d2n_send(volatile struct cosim_eth_proto_d2n_send *s)
 
 static void poll_d2n(void)
 {
+    volatile union cosim_eth_proto_d2n *msg = netsim_d2n_poll(&nsif);
     uint8_t type;
-    volatile union cosim_eth_proto_d2n *msg =
-        (volatile union cosim_eth_proto_d2n *)
-        (d2n_queue + d2n_pos * d2n_elen);
 
     /* message not ready */
-    if ((msg->dummy.own_type & COSIM_ETH_PROTO_D2N_OWN_MASK) !=
-            COSIM_ETH_PROTO_D2N_OWN_NET)
+    if (msg == NULL)
         return;
 
     type = msg->dummy.own_type & COSIM_ETH_PROTO_D2N_MSG_MASK;
@@ -185,9 +92,7 @@ static void poll_d2n(void)
             fprintf(stderr, "poll_d2n: unsupported type=%u\n", type);
     }
 
-    msg->dummy.own_type = (msg->dummy.own_type & COSIM_ETH_PROTO_D2N_MSG_MASK)
-        | COSIM_ETH_PROTO_D2N_OWN_DEV;
-    d2n_pos = (d2n_pos + 1) % d2n_enum;
+    netsim_d2n_done(&nsif, msg);
 }
 
 static void *rx_handler(void *arg)
@@ -197,38 +102,32 @@ static void *rx_handler(void *arg)
     ssize_t len;
 
     while (1) {
-        msg = (volatile union cosim_eth_proto_n2d *)
-                (n2d_queue + n2d_pos * n2d_elen);
-
-        if ((msg->dummy.own_type & COSIM_ETH_PROTO_N2D_OWN_MASK) !=
-                COSIM_ETH_PROTO_N2D_OWN_NET)
-        {
+        msg = netsim_n2d_alloc(&nsif);
+        if (msg == NULL) {
             fprintf(stderr, "coudl not allocate message for rx\n");
             abort();
         }
         rx = &msg->recv;
 
-        len = read(tap_fd, (void *) rx->data, n2d_elen - sizeof(*msg));
+        len = read(tap_fd, (void *) rx->data, nsif.n2d_elen - sizeof(*msg));
         if (len <= 0) {
             perror("rx handler: read failed");
         }
         rx->len = len;
         rx->port = 0;
+#ifdef DEBUG_PKTMETA
         printf("received packet: len=%u\n", rx->len);
+#endif
 
         // WMB();
         rx->own_type = COSIM_ETH_PROTO_N2D_MSG_RECV |
             COSIM_ETH_PROTO_N2D_OWN_DEV;
-        n2d_pos = (n2d_pos + 1) % n2d_enum;
     }
 }
 
 int main(int argc, char *argv[])
 {
-    struct cosim_eth_proto_dev_intro di;
-    struct cosim_eth_proto_net_intro ni;
-    int cfd, shm_fd;
-    void *p;
+    int sync;
 
     if (argc != 2) {
         fprintf(stderr, "Usage: net_tap TAP_DEVICE_NAME\n");
@@ -239,33 +138,12 @@ int main(int argc, char *argv[])
         return -1;
     }
 
-    if ((cfd = uxsocket_connect("/tmp/cosim-eth")) < 0) {
-        return -1;
-    }
-
-    memset(&ni, 0, sizeof(ni));
-    if (send(cfd, &ni, sizeof(ni), 0) != sizeof(ni)) {
-        perror("sending net intro failed");
+    sync = 0;
+    if (netsim_init(&nsif, "/tmp/cosim-eth", &sync) != 0) {
+        close(tap_fd);
         return -1;
     }
 
-    if (uxsocket_recv(cfd, &di, sizeof(di), &shm_fd)) {
-        return -1;
-    }
-
-    if ((p = shm_map(shm_fd)) == NULL) {
-        return -1;
-    }
-    close(shm_fd);
-
-    d2n_queue = (uint8_t *) p + di.d2n_offset;
-    n2d_queue = (uint8_t *) p + di.n2d_offset;
-    d2n_elen = di.d2n_elen;
-    n2d_elen = di.n2d_elen;
-    d2n_enum = di.d2n_nentries;
-    n2d_enum = di.n2d_nentries;
-    d2n_pos = n2d_pos = 0;
-
     pthread_t worker;
     if (pthread_create(&worker, NULL, rx_handler, NULL) != 0) {
         return EXIT_FAILURE;

net_wire/Makefile

+CPPFLAGS += -I../proto -I../netsim_common/include
+CFLAGS += -Wall -Wextra -Wno-unused-parameter -O3
+
+net_wire: net_wire.o ../netsim_common/libnetsim_common.a
+
+all: net_wire
+
+clean:
+	rm -f *.o net_wire

net_wire/net_wire.c

+/*
+ * Copyright 2020 Max Planck Institute for Software Systems
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <fcntl.h>
+#include <pthread.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include <linux/if.h>
+#include <linux/if_tun.h>
+
+#include <netsim.h>
+
+static void move_pkt(struct netsim_interface *from, struct netsim_interface *to)
+{
+    volatile union cosim_eth_proto_d2n *msg_from = netsim_d2n_poll(from);
+    volatile union cosim_eth_proto_n2d *msg_to;
+    volatile struct cosim_eth_proto_d2n_send *tx;
+    volatile struct cosim_eth_proto_n2d_recv *rx;
+    uint8_t type;
+
+    if (msg_from == NULL)
+        return;
+
+    type = msg_from->dummy.own_type & COSIM_ETH_PROTO_D2N_MSG_MASK;
+    if (type == COSIM_ETH_PROTO_D2N_MSG_SEND) {
+        tx = &msg_from->send;
+
+        msg_to = netsim_n2d_alloc(to);
+        if (msg_to != NULL) {
+            rx = &msg_to->recv;
+            rx->len = tx->len;
+            rx->port = 0;
+            memcpy((void *) rx->data, (void *) tx->data, tx->len);
+
+            // WMB();
+            rx->own_type = COSIM_ETH_PROTO_N2D_MSG_RECV |
+                COSIM_ETH_PROTO_N2D_OWN_DEV;
+        } else {
+            fprintf(stderr, "move_pkt: dropping packet\n");
+        }
+    } else {
+        fprintf(stderr, "move_pkt: unsupported type=%u\n", type);
+        abort();
+    }
+
+    netsim_d2n_done(from, msg_from);
+}
+
+int main(int argc, char *argv[])
+{
+    struct netsim_interface nsif_a, nsif_b;
+    int sync;
+
+    if (argc != 3) {
+        fprintf(stderr, "Usage: net_tap SOCKET-A SOCKET-B\n");
+        return EXIT_FAILURE;
+    }
+
+    sync = 0;
+    if (netsim_init(&nsif_a, argv[1], &sync) != 0) {
+        return -1;
+    }
+    if (netsim_init(&nsif_b, argv[2], &sync) != 0) {
+        return -1;
+    }
+
+    printf("start polling\n");
+    while (1) {
+        move_pkt(&nsif_a, &nsif_b);
+        move_pkt(&nsif_b, &nsif_a);
+    }
+    return 0;
+}

netsim_common/Makefile

+CPPFLAGS += -I include/ -I../proto/
+CFLAGS += -Wall -Wextra -Wno-unused-parameter -O3
+
+all: libnetsim_common.a
+
+libnetsim_common.a: netsim.o utils.o
+	$(AR) rcs $@ $^
+
+clean:
+	rm -rf libnetsim_common.a *.o

netsim_common/include/netsim.h

+/*
+ * Copyright 2020 Max Planck Institute for Software Systems
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef COSIM_NETSIM_H_
+#define COSIM_NETSIM_H_
+
+#include <stddef.h>
+#include <stdint.h>
+#include <cosim_eth_proto.h>
+
+struct netsim_interface {
+    uint8_t *d2n_queue;
+    size_t d2n_pos;
+    size_t d2n_elen;
+    size_t d2n_enum;
+
+    uint8_t *n2d_queue;
+    size_t n2d_pos;
+    size_t n2d_elen;
+    size_t n2d_enum;
+};
+
+int netsim_init(struct netsim_interface *nsif,
+        const char *eth_socket_path, int *sync_eth);
+void netsim_cleanup(struct netsim_interface *nsif);
+
+volatile union cosim_eth_proto_d2n *netsim_d2n_poll(
+        struct netsim_interface *nsif);
+void netsim_d2n_done(struct netsim_interface *nsif,
+        volatile union cosim_eth_proto_d2n *msg);
+
+volatile union cosim_eth_proto_n2d *netsim_n2d_alloc(
+        struct netsim_interface *nsif);
+
+#endif /* ndef COSIM_NETSIM_H_ */

netsim_common/internal.h

+/*
+ * Copyright 2020 Max Planck Institute for Software Systems
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <stddef.h>
+#include <stdio.h>
+
+int uxsocket_connect(const char *path);
+int uxsocket_recv(int fd, void *data, size_t len, int *pfd);
+void *shm_map(int shm_fd);

netsim_common/netsim.c

+/*
+ * Copyright 2020 Max Planck Institute for Software Systems
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+#include <netsim.h>
+#include "internal.h"
+
+int netsim_init(struct netsim_interface *nsif,
+        const char *eth_socket_path, int *sync_eth)
+{
+    struct cosim_eth_proto_dev_intro di;
+    struct cosim_eth_proto_net_intro ni;
+    int cfd, shm_fd;
+    void *p;
+
+    if ((cfd = uxsocket_connect(eth_socket_path)) < 0) {
+        return -1;
+    }
+
+    memset(&ni, 0, sizeof(ni));
+
+    if (*sync_eth)
+        ni.flags |= COSIM_ETH_PROTO_FLAGS_NI_SYNC;
+
+    if (send(cfd, &ni, sizeof(ni), 0) != sizeof(ni)) {
+        perror("sending net intro failed");
+        return -1;
+    }
+
+    if (uxsocket_recv(cfd, &di, sizeof(di), &shm_fd)) {
+        return -1;
+    }
+
+    if ((p = shm_map(shm_fd)) == NULL) {
+        return -1;
+    }
+    close(shm_fd);
+
+    if ((di.flags & COSIM_ETH_PROTO_FLAGS_DI_SYNC) == 0)
+        *sync_eth = 0;
+
+    nsif->d2n_queue = (uint8_t *) p + di.d2n_offset;
+    nsif->n2d_queue = (uint8_t *) p + di.n2d_offset;
+    nsif->d2n_elen = di.d2n_elen;
+    nsif->n2d_elen = di.n2d_elen;
+    nsif->d2n_enum = di.d2n_nentries;
+    nsif->n2d_enum = di.n2d_nentries;
+    nsif->d2n_pos = nsif->n2d_pos = 0;
+
+    return 0;
+}
+
+void netsim_cleanup(struct netsim_interface *nsif)
+{
+    fprintf(stderr, "netsim_cleanup: TODO\n");
+    abort();
+}
+
+volatile union cosim_eth_proto_d2n *netsim_d2n_poll(
+        struct netsim_interface *nsif)
+{
+    volatile union cosim_eth_proto_d2n *msg =
+        (volatile union cosim_eth_proto_d2n *)
+        (nsif->d2n_queue + nsif->d2n_pos * nsif->d2n_elen);
+
+    /* message not ready */
+    if ((msg->dummy.own_type & COSIM_ETH_PROTO_D2N_OWN_MASK) !=
+            COSIM_ETH_PROTO_D2N_OWN_NET)
+        return NULL;
+
+    nsif->d2n_pos = (nsif->d2n_pos + 1) % nsif->d2n_enum;
+    return msg;
+}
+
+void netsim_d2n_done(struct netsim_interface *nsif,
+        volatile union cosim_eth_proto_d2n *msg)
+{
+    msg->dummy.own_type = (msg->dummy.own_type & COSIM_ETH_PROTO_D2N_MSG_MASK)
+        | COSIM_ETH_PROTO_D2N_OWN_DEV;
+}
+
+volatile union cosim_eth_proto_n2d *netsim_n2d_alloc(
+        struct netsim_interface *nsif)
+{
+    volatile union cosim_eth_proto_n2d *msg =
+        (volatile union cosim_eth_proto_n2d *)
+        (nsif->n2d_queue + nsif->n2d_pos * nsif->n2d_elen);
+
+    if ((msg->dummy.own_type & COSIM_ETH_PROTO_N2D_OWN_MASK) !=
+            COSIM_ETH_PROTO_N2D_OWN_NET)
+    {
+        return NULL;
+    }
+
+    nsif->n2d_pos = (nsif->n2d_pos + 1) % nsif->n2d_enum;
+    return msg;
+}

netsim_common/utils.c

+/*
+ * Copyright 2020 Max Planck Institute for Software Systems
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <fcntl.h>
+#include <pthread.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/un.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "internal.h"
+
+int uxsocket_connect(const char *path)
+{
+    int fd;
+    struct sockaddr_un saun;
+
+    /* prepare and connect socket */
+    memset(&saun, 0, sizeof(saun));
+    saun.sun_family = AF_UNIX;
+    strcpy(saun.sun_path, path);
+
+    if ((fd = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) {
+        perror("socket failed");
+        return -1;
+    }
+
+    if (connect(fd, (struct sockaddr *) &saun, sizeof(saun)) != 0) {
+        perror("connect failed");
+        return -1;
+    }
+
+    return fd;
+}
+
+int uxsocket_recv(int fd, void *data, size_t len, int *pfd)
+{
+    int *ppfd;
+    ssize_t ret;
+    struct cmsghdr *cmsg;
+    union {
+        char buf[CMSG_SPACE(sizeof(int))];
+        struct cmsghdr align;
+    } u;
+    struct iovec iov = {
+        .iov_base = data,
+        .iov_len = len,
+    };
+    struct msghdr msg = {
+        .msg_name = NULL,
+        .msg_namelen = 0,
+        .msg_iov = &iov,
+        .msg_iovlen = 1,
+        .msg_control = u.buf,
+        .msg_controllen = sizeof(u.buf),
+        .msg_flags = 0,
+    };
+
+    if ((ret = recvmsg(fd, &msg, 0)) != (ssize_t) len) {
+        perror("recvmsg failed");
+        return -1;
+    }
+
+    cmsg = CMSG_FIRSTHDR(&msg);
+    ppfd = (int *) CMSG_DATA(cmsg);
+    if (msg.msg_controllen <= 0 || cmsg->cmsg_len != CMSG_LEN(sizeof(int))) {
+        fprintf(stderr, "accessing ancillary data failed\n");
+        return -1;
+    }
+
+    *pfd = *ppfd;
+    return 0;
+}
+
+void *shm_map(int shm_fd)
+{
+    void *p;
+    struct stat statbuf;
+
+    if (fstat(shm_fd, &statbuf) != 0) {
+        perror("shm_map: fstat failed");
+        return NULL;
+    }
+
+    p = mmap(NULL, statbuf.st_size, PROT_READ | PROT_WRITE, MAP_SHARED, shm_fd,
+            0);
+    if (p == MAP_FAILED) {
+        perror("shm_map: mmap failed");
+        return NULL;
+    }
+
+    return p;
+}