Skip to content

GitLab

"example/vscode:/vscode.git/clone" did not exist on "246723392ba7351dcc713bc472d585cb549dd6cb"
Commit 87f7ac7a authored by Antoine Kaufmann's avatar Antoine Kaufmann
Browse files

nicsim_common,corundum(_bm): move sync support to library

parent c35116cf
Hide whitespace changes
Inline Side-by-side
Showing with 201 additions and 84 deletions
corundum/corundum_verilator.cpp
View file @ 87f7ac7a
......@@ -22,17 +22,13 @@ extern "C" {
#define CLOCK_PERIOD (100 * 1000ULL) // 100ns -> 10MHz
#define SYNC_PERIOD (500 * 1000ULL) // 100ns
#define PCI_LATENCY (1 * 1000 * 1000ULL) // 1us
#define ETH_ASYNCHRONY (1 * 1000 * 1000ULL) // 1us
#define ETH_LATENCY (1 * 1000 * 1000ULL) // 1us
struct DMAOp;
static volatile int exiting = 0;
static int sync_pci_en, sync_eth_en;
static uint64_t main_time = 0;
static uint64_t pci_last_rx_time = 0;
static uint64_t pci_last_tx_time = 0;
static uint64_t eth_last_rx_time = 0;
static uint64_t eth_last_tx_time = 0;
static struct nicsim_params nsparams;
#ifdef TRACE_ENABLED
static VerilatedVcdC* trace;
#endif
......@@ -541,18 +537,14 @@ static void h2d_write(MMIOInterface &mmio,
static void poll_h2d(MMIOInterface &mmio)
{
volatile union cosim_pcie_proto_h2d *msg = nicif_h2d_poll();
volatile union cosim_pcie_proto_h2d *msg =
nicif_h2d_poll(&nsparams, main_time);
uint8_t t;
if (msg == NULL)
return;
t = msg->dummy.own_type & COSIM_PCIE_PROTO_H2D_MSG_MASK;
pci_last_rx_time = msg->dummy.timestamp;
// we're running synchronized but it's not time for this message yet
if (sync_pci_en && pci_last_rx_time > main_time)
return;
//std::cerr << "poll_h2d: polled type=" << (int) t << std::endl;
switch (t) {
......@@ -586,10 +578,7 @@ static void poll_h2d(MMIOInterface &mmio)
static volatile union cosim_pcie_proto_d2h *d2h_alloc(void)
{
volatile union cosim_pcie_proto_d2h *msg = nicsim_d2h_alloc();
msg->dummy.timestamp = main_time + PCI_LATENCY;
pci_last_tx_time = main_time;
return msg;
return nicsim_d2h_alloc(&nsparams, main_time);
}
class EthernetTx {
......@@ -606,7 +595,8 @@ class EthernetTx {
void packet_done()
{
volatile union cosim_eth_proto_d2n *msg = nicsim_d2n_alloc();
volatile union cosim_eth_proto_d2n *msg =
nicsim_d2n_alloc(&nsparams, main_time);
volatile struct cosim_eth_proto_d2n_send *send;
if (!msg)
......@@ -615,6 +605,7 @@ class EthernetTx {
send = &msg->send;
memcpy((void *) send->data, packet_buf, packet_len);
send->len = packet_len;
send->timestamp = main_time + ETH_LATENCY;
//WMB();
send->own_type = COSIM_ETH_PROTO_D2N_MSG_SEND |
......@@ -749,18 +740,23 @@ static void n2d_recv(EthernetRx &rx,
static void poll_n2d(EthernetRx &rx)
{
volatile union cosim_eth_proto_n2d *msg = nicif_n2d_poll();
volatile union cosim_eth_proto_n2d *msg =
nicif_n2d_poll(&nsparams, main_time);
uint8_t t;
if (msg == NULL)
return;
t = msg->dummy.own_type & COSIM_ETH_PROTO_N2D_MSG_MASK;
switch (t) {
case COSIM_ETH_PROTO_N2D_MSG_RECV:
n2d_recv(rx, &msg->recv);
break;
case COSIM_ETH_PROTO_N2D_MSG_SYNC:
break;
default:
std::cerr << "poll_n2d: unsupported type=" << t << std::endl;
}
......@@ -878,24 +874,6 @@ static void msi_step(Vinterface &top, PCICoordinator &coord)
}
}
static void send_sync_pci(MMIOInterface &mmio)
{
volatile union cosim_pcie_proto_d2h *msg;
volatile struct cosim_pcie_proto_d2h_sync *sync;
if (pci_last_tx_time == 0 || main_time - pci_last_tx_time >= SYNC_PERIOD) {
msg = d2h_alloc();
sync = &msg->sync;
// WMB();
sync->own_type = COSIM_PCIE_PROTO_D2H_MSG_SYNC |
COSIM_PCIE_PROTO_D2H_OWN_HOST;
}
}
static void sync_eth(EthernetRx &rx)
{
}
int main(int argc, char *argv[])
{
char *vargs[2] = { argv[0], NULL };
......@@ -925,15 +903,19 @@ int main(int argc, char *argv[])
di.pci_revision = 0x00;
di.pci_msi_nvecs = 32;
sync_pci_en = 1;
sync_eth_en = 1;
if (nicsim_init(&di, argv[1], &sync_pci_en, argv[2], &sync_eth_en,
argv[3]))
{
nsparams.sync_pci = 1;
nsparams.sync_eth = 1;
nsparams.pci_socket_path = argv[1];
nsparams.eth_socket_path = argv[2];
nsparams.shm_path = argv[3];
nsparams.pci_latency = PCI_LATENCY;
nsparams.eth_latency = ETH_LATENCY;
nsparams.sync_delay = SYNC_PERIOD;
if (nicsim_init(&nsparams, &di)) {
return EXIT_FAILURE;
}
std::cout << "sync_pci=" << sync_pci_en << " sync_eth=" << sync_eth_en <<
std::endl;
std::cout << "sync_pci=" << nsparams.sync_pci <<
" sync_eth=" << nsparams.sync_eth << std::endl;
signal(SIGINT, sigint_handler);
......@@ -1044,15 +1026,16 @@ int main(int argc, char *argv[])
top->rst = 0;
while (!exiting) {
if (sync_pci_en)
send_sync_pci(mmio);
if (sync_eth_en)
sync_eth(rx);
while (nicsim_sync(&nsparams, main_time)) {
std::cerr << "warn: nicsim_sync failed (t=" << main_time << ")" <<
std::endl;
}
do {
poll_h2d(mmio);
poll_n2d(rx);
} while (sync_pci_en && pci_last_rx_time < main_time && !exiting);
} while ((nsparams.sync_pci || nsparams.sync_eth) &&
netsim_next_timestamp(&nsparams) <= main_time && !exiting);
/* falling edge */
top->clk = !top->clk;
......
corundum_bm/corundum_bm.cc
View file @ 87f7ac7a
......@@ -13,10 +13,17 @@ extern "C" {
#include <netsim.h>
}
#define SYNC_PERIOD (500 * 1000ULL) // 100ns
#define PCI_LATENCY (1 * 1000 * 1000ULL) // 1us
#define ETH_LATENCY (1 * 1000 * 1000ULL) // 1us
static void issue_dma_op(corundum::DMAOp *op);
static void msi_issue(uint8_t vec);
static void eth_send(void *data, size_t len);
static uint64_t main_time = 0;
static struct nicsim_params nsparams;
namespace corundum {
DescRing::DescRing()
......@@ -757,7 +764,8 @@ sigint_handler(int dummy)
static volatile union cosim_pcie_proto_d2h *
d2h_alloc(void)
{
volatile union cosim_pcie_proto_d2h *msg = nicsim_d2h_alloc();
volatile union cosim_pcie_proto_d2h *msg =
nicsim_d2h_alloc(&nsparams, main_time);
if (msg == NULL) {
fprintf(stderr, "d2h_alloc: no entry available\n");
abort();
......@@ -768,7 +776,8 @@ d2h_alloc(void)
static volatile union cosim_eth_proto_d2n *
d2n_alloc(void)
{
volatile union cosim_eth_proto_d2n *msg = nicsim_d2n_alloc();
volatile union cosim_eth_proto_d2n *msg =
nicsim_d2n_alloc(&nsparams, main_time);
if (msg == NULL) {
fprintf(stderr, "d2n_alloc: no entry available\n");
abort();
......@@ -893,7 +902,8 @@ static void eth_send(void *data, size_t len)
static void poll_h2d(Corundum &nic)
{
volatile union cosim_pcie_proto_h2d *msg = nicif_h2d_poll();
volatile union cosim_pcie_proto_h2d *msg =
nicif_h2d_poll(&nsparams, main_time);
uint8_t type;
if (msg == NULL)
......@@ -927,7 +937,8 @@ static void poll_h2d(Corundum &nic)
static void poll_n2d(Corundum &nic)
{
volatile union cosim_eth_proto_n2d *msg = nicif_n2d_poll();
volatile union cosim_eth_proto_n2d *msg =
nicif_n2d_poll(&nsparams, main_time);
uint8_t t;
if (msg == NULL)
......@@ -949,6 +960,17 @@ static void poll_n2d(Corundum &nic)
int main(int argc, char *argv[])
{
uint64_t next_ts;
if (argc != 4 && argc != 5) {
fprintf(stderr, "Usage: corundum_bm PCI-SOCKET ETH-SOCKET "
"SHM [START-TICK]\n");
return EXIT_FAILURE;
}
if (argc == 5)
main_time = strtoull(argv[4], NULL, 0);
signal(SIGINT, sigint_handler);
struct cosim_pcie_proto_dev_intro di;
......@@ -962,18 +984,34 @@ int main(int argc, char *argv[])
di.pci_revision = 0x00;
di.pci_msi_nvecs = 32;
int sync_pci_en = 0, sync_eth_en = 0;
if (nicsim_init(&di, "/tmp/cosim-pci", &sync_pci_en,
"/tmp/cosim-eth", &sync_eth_en,
"/dev/shm/dummy_nic_shm")) {
nsparams.sync_pci = 1;
nsparams.sync_eth = 1;
nsparams.pci_socket_path = argv[1];
nsparams.eth_socket_path = argv[2];
nsparams.shm_path = argv[3];
nsparams.pci_latency = PCI_LATENCY;
nsparams.eth_latency = ETH_LATENCY;
nsparams.sync_delay = SYNC_PERIOD;
if (nicsim_init(&nsparams, &di)) {
return EXIT_FAILURE;
}
fprintf(stderr, "sync_pci=%d sync_eth=%d\n", nsparams.sync_pci,
nsparams.sync_eth);
Corundum nic;
while (!exiting) {
poll_h2d(nic);
poll_n2d(nic);
while (nicsim_sync(&nsparams, main_time)) {
fprintf(stderr, "warn: nicsim_sync failed (t=%llu)\n", main_time);
}
do {
poll_h2d(nic);
poll_n2d(nic);
next_ts = netsim_next_timestamp(&nsparams);
} while ((nsparams.sync_pci || nsparams.sync_eth) &&
next_ts <= main_time && !exiting);
main_time = next_ts;
}
nicsim_cleanup();
......
nicsim_common/include/nicsim.h
View file @ 87f7ac7a
......@@ -27,24 +27,41 @@
#include <cosim_pcie_proto.h>
#include <cosim_eth_proto.h>
int nicsim_init(struct cosim_pcie_proto_dev_intro *di,
const char *pci_socket_path, int *sync_pci,
const char *eth_socket_path, int *sync_eth,
const char *shm_path);
struct nicsim_params {
const char *pci_socket_path;
const char *eth_socket_path;
const char *shm_path;
uint64_t pci_latency;
uint64_t eth_latency;
uint64_t sync_delay;
int sync_pci;
int sync_eth;
};
int nicsim_init(struct nicsim_params *params,
struct cosim_pcie_proto_dev_intro *di);
void nicsim_cleanup(void);
int nicsim_sync(struct nicsim_params *params, uint64_t timestamp);
uint64_t netsim_next_timestamp(struct nicsim_params *params);
volatile union cosim_pcie_proto_h2d *nicif_h2d_poll(void);
volatile union cosim_pcie_proto_h2d *nicif_h2d_poll(
struct nicsim_params *params, uint64_t timestamp);
void nicif_h2d_done(volatile union cosim_pcie_proto_h2d *msg);
void nicif_h2d_next(void);
volatile union cosim_pcie_proto_d2h *nicsim_d2h_alloc(void);
volatile union cosim_pcie_proto_d2h *nicsim_d2h_alloc(
struct nicsim_params *params, uint64_t timestamp);
volatile union cosim_eth_proto_n2d *nicif_n2d_poll(void);
volatile union cosim_eth_proto_n2d *nicif_n2d_poll(
struct nicsim_params *params, uint64_t timestamp);
void nicif_n2d_done(volatile union cosim_eth_proto_n2d *msg);
void nicif_n2d_next(void);
volatile union cosim_eth_proto_d2n *nicsim_d2n_alloc(void);
volatile union cosim_eth_proto_d2n *nicsim_d2n_alloc(
struct nicsim_params *params, uint64_t timestamp);
#endif /* ndef COSIM_NICSIM_H_ */
nicsim_common/nicsim.c
View file @ 87f7ac7a
......@@ -61,6 +61,10 @@ static uint8_t *n2d_queue;
static size_t n2d_pos;
static size_t n2d_off; /* offset in shm region */
static uint64_t pci_last_rx_time = 0;
static uint64_t pci_last_tx_time = 0;
static uint64_t eth_last_rx_time = 0;
static uint64_t eth_last_tx_time = 0;
static int shm_fd = -1;
static int pci_cfd = -1;
......@@ -173,16 +177,17 @@ static int accept_conns(struct cosim_pcie_proto_dev_intro *di,
return 0;
}
int nicsim_init(struct cosim_pcie_proto_dev_intro *di,
const char *pci_socket_path, int *sync_pci,
const char *eth_socket_path, int *sync_eth,
const char *shm_path)
int nicsim_init(struct nicsim_params *params,
struct cosim_pcie_proto_dev_intro *di)
{
int pci_lfd = -1, eth_lfd = -1;
void *shmptr;
/* ready in memory queues */
if ((shm_fd = shm_create(shm_path, 32 * 1024 * 1024, &shmptr)) < 0) {
if ((shm_fd = shm_create(params->shm_path, 32 * 1024 * 1024, &shmptr))
< 0)
{
return -1;
}
......@@ -199,39 +204,41 @@ int nicsim_init(struct cosim_pcie_proto_dev_intro *di,
d2h_pos = h2d_pos = d2n_pos = n2d_pos = 0;
/* get listening sockets ready */
if (pci_socket_path != NULL) {
if ((pci_lfd = uxsocket_init(pci_socket_path)) < 0) {
if (params->pci_socket_path != NULL) {
if ((pci_lfd = uxsocket_init(params->pci_socket_path)) < 0) {
return -1;
}
}
if (eth_socket_path != NULL) {
if ((eth_lfd = uxsocket_init(eth_socket_path)) < 0) {
if (params->eth_socket_path != NULL) {
if ((eth_lfd = uxsocket_init(params->eth_socket_path)) < 0) {
return -1;
}
}
/* accept connection fds */
if (accept_conns(di, pci_lfd, sync_pci, eth_lfd, sync_eth) != 0) {
if (accept_conns(di, pci_lfd, &params->sync_pci, eth_lfd,
&params->sync_eth) != 0)
{
return -1;
}
/* receive introductions from other end */
if (pci_socket_path != NULL) {
if (params->pci_socket_path != NULL) {
struct cosim_pcie_proto_host_intro hi;
if (recv(pci_cfd, &hi, sizeof(hi), 0) != sizeof(hi)) {
return -1;
}
if ((hi.flags & COSIM_PCIE_PROTO_FLAGS_HI_SYNC) == 0)
*sync_pci = 0;
params->sync_pci = 0;
printf("pci host info received\n");
}
if (eth_socket_path != NULL) {
if (params->eth_socket_path != NULL) {
struct cosim_eth_proto_net_intro ni;
if (recv(eth_cfd, &ni, sizeof(ni), 0) != sizeof(ni)) {
return -1;
}
if ((ni.flags & COSIM_ETH_PROTO_FLAGS_NI_SYNC) == 0)
*sync_eth = 0;
params->sync_eth = 0;
printf("eth net info received\n");
}
......@@ -244,10 +251,63 @@ void nicsim_cleanup(void)
close(eth_cfd);
}
/******************************************************************************/
/* Sync */
int nicsim_sync(struct nicsim_params *params, uint64_t timestamp)
{
int ret = 0;
volatile union cosim_pcie_proto_d2h *d2h;
volatile union cosim_eth_proto_d2n *d2n;
/* sync PCI if necessary */
if (params->sync_pci &&
timestamp - pci_last_tx_time >= params->sync_delay)
{
d2h = nicsim_d2h_alloc(params, timestamp);
if (d2h == NULL) {
ret = -1;
} else {
d2h->sync.own_type = COSIM_PCIE_PROTO_D2H_MSG_SYNC |
COSIM_PCIE_PROTO_D2H_OWN_HOST;
}
}
/* sync Ethernet if necessary */
if (params->sync_eth &&
timestamp - eth_last_tx_time >= params->sync_delay)
{
d2n = nicsim_d2n_alloc(params, timestamp);
if (d2n == NULL) {
ret = -1;
} else {
d2n->sync.own_type = COSIM_ETH_PROTO_D2N_MSG_SYNC |
COSIM_ETH_PROTO_D2N_OWN_NET;
}
}
return ret;
}
uint64_t netsim_next_timestamp(struct nicsim_params *params)
{
if (params->sync_pci && params->sync_eth) {
return (pci_last_rx_time <= eth_last_rx_time ? pci_last_rx_time :
eth_last_rx_time);
} else if (params->sync_pci) {
return pci_last_rx_time;
} else if (params->sync_eth) {
return eth_last_rx_time;
} else {
return 0;
}
}
/******************************************************************************/
/* PCI */
volatile union cosim_pcie_proto_h2d *nicif_h2d_poll(void)
volatile union cosim_pcie_proto_h2d *nicif_h2d_poll(
struct nicsim_params *params, uint64_t timestamp)
{
volatile union cosim_pcie_proto_h2d *msg =
(volatile union cosim_pcie_proto_h2d *)
......@@ -258,6 +318,11 @@ volatile union cosim_pcie_proto_h2d *nicif_h2d_poll(void)
COSIM_PCIE_PROTO_H2D_OWN_DEV)
return NULL;
/* if in sync mode, wait till message is ready */
pci_last_rx_time = msg->dummy.timestamp;
if (params->sync_pci && pci_last_rx_time > timestamp)
return NULL;
return msg;
}
......@@ -272,7 +337,8 @@ void nicif_h2d_next(void)
h2d_pos = (h2d_pos + 1) % H2D_ENUM;
}
volatile union cosim_pcie_proto_d2h *nicsim_d2h_alloc(void)
volatile union cosim_pcie_proto_d2h *nicsim_d2h_alloc(
struct nicsim_params *params, uint64_t timestamp)
{
volatile union cosim_pcie_proto_d2h *msg =
(volatile union cosim_pcie_proto_d2h *)
......@@ -284,6 +350,9 @@ volatile union cosim_pcie_proto_d2h *nicsim_d2h_alloc(void)
return NULL;
}
msg->dummy.timestamp = timestamp + params->pci_latency;;
pci_last_tx_time = timestamp;
d2h_pos = (d2h_pos + 1) % D2H_ENUM;
return msg;
}
......@@ -291,7 +360,8 @@ volatile union cosim_pcie_proto_d2h *nicsim_d2h_alloc(void)
/******************************************************************************/
/* Ethernet */
volatile union cosim_eth_proto_n2d *nicif_n2d_poll(void)
volatile union cosim_eth_proto_n2d *nicif_n2d_poll(
struct nicsim_params *params, uint64_t timestamp)
{
volatile union cosim_eth_proto_n2d *msg =
(volatile union cosim_eth_proto_n2d *)
......@@ -302,6 +372,11 @@ volatile union cosim_eth_proto_n2d *nicif_n2d_poll(void)
COSIM_ETH_PROTO_N2D_OWN_DEV)
return NULL;
/* if in sync mode, wait till message is ready */
eth_last_rx_time = msg->dummy.timestamp;
if (params->sync_eth && eth_last_rx_time > timestamp)
return NULL;
return msg;
}
......@@ -316,7 +391,8 @@ void nicif_n2d_next(void)
n2d_pos = (n2d_pos + 1) % N2D_ENUM;
}
volatile union cosim_eth_proto_d2n *nicsim_d2n_alloc(void)
volatile union cosim_eth_proto_d2n *nicsim_d2n_alloc(
struct nicsim_params *params, uint64_t timestamp)
{
volatile union cosim_eth_proto_d2n *msg =
(volatile union cosim_eth_proto_d2n *)
......@@ -328,6 +404,9 @@ volatile union cosim_eth_proto_d2n *nicsim_d2n_alloc(void)
return NULL;
}
msg->dummy.timestamp = timestamp + params->eth_latency;
eth_last_tx_time = timestamp;
d2n_pos = (d2n_pos + 1) % D2N_ENUM;
return msg;
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment