Skip to content

GitLab

Commit 5728d26c authored by Jialin Li's avatar Jialin Li
Browse files

netsim/nicsim: add global barrier synchronization mode

parent f5216010
Show whitespace changes
Inline Side-by-side
Showing with 181 additions and 47 deletions
libnicbm/nicbm.cc
View file @ 5728d26c
......@@ -391,21 +391,23 @@ int Runner::runMain(int argc, char *argv[])
uint64_t sync_period = 100 * 1000ULL;
uint64_t pci_latency = 500 * 1000ULL;
uint64_t eth_latency = 500 * 1000ULL;
int sync_mode = SYNC_MODES;
if (argc < 4 && argc > 8) {
if (argc < 4 && argc > 9) {
fprintf(stderr, "Usage: corundum_bm PCI-SOCKET ETH-SOCKET "
"SHM [START-TICK] [SYNC-PERIOD] [PCI-LATENCY] [ETH-LATENCY]\n");
"SHM [SYNC-MODE] [START-TICK] [SYNC-PERIOD] [PCI-LATENCY] [ETH-LATENCY]\n");
return EXIT_FAILURE;
}
if (argc >= 5)
main_time = strtoull(argv[4], NULL, 0);
sync_mode = strtol(argv[4], NULL, 0);
if (argc >= 6)
sync_period = strtoull(argv[5], NULL, 0) * 1000ULL;
main_time = strtoull(argv[5], NULL, 0);
if (argc >= 7)
pci_latency = strtoull(argv[6], NULL, 0) * 1000ULL;
sync_period = strtoull(argv[6], NULL, 0) * 1000ULL;
if (argc >= 8)
eth_latency = strtoull(argv[7], NULL, 0) * 1000ULL;
pci_latency = strtoull(argv[7], NULL, 0) * 1000ULL;
if (argc >= 9)
eth_latency = strtoull(argv[8], NULL, 0) * 1000ULL;
signal(SIGINT, sigint_handler);
signal(SIGUSR1, sigusr1_handler);
......@@ -421,6 +423,9 @@ int Runner::runMain(int argc, char *argv[])
nsparams.pci_latency = pci_latency;
nsparams.eth_latency = eth_latency;
nsparams.sync_delay = sync_period;
assert(sync_mode == SYNC_MODES || sync_mode == SYNC_BARRIER);
nsparams.sync_mode = sync_mode;
if (nicsim_init(&nsparams, &dintro)) {
return EXIT_FAILURE;
}
......@@ -433,6 +438,7 @@ int Runner::runMain(int argc, char *argv[])
while (nicsim_sync(&nsparams, main_time)) {
fprintf(stderr, "warn: nicsim_sync failed (t=%lu)\n", main_time);
}
nicsim_advance_epoch(&nsparams, main_time);
do {
poll_h2d();
......@@ -440,7 +446,7 @@ int Runner::runMain(int argc, char *argv[])
event_trigger();
if (is_sync) {
next_ts = netsim_next_timestamp(&nsparams);
next_ts = nicsim_next_timestamp(&nsparams);
if (next_ts > main_time + max_step)
next_ts = main_time + max_step;
} else {
......@@ -452,7 +458,7 @@ int Runner::runMain(int argc, char *argv[])
next_ts = ev_ts;
} while (next_ts <= main_time && !exiting);
main_time = next_ts;
main_time = nicsim_advance_time(&nsparams, next_ts);
}
fprintf(stderr, "exit main_time: %lu\n", main_time);
......
net_switch/net_switch.cc
View file @ 5728d26c
......@@ -6,6 +6,7 @@
#include <unistd.h>
#include <vector>
#include <unordered_map>
#include <cassert>
extern "C" {
#include <netsim.h>
......@@ -118,9 +119,10 @@ int main(int argc, char *argv[])
{
int c;
int bad_option = 0;
int sync_mode = SYNC_MODES;
// Parse command line argument
while ((c = getopt(argc, argv, "s:S:E:")) != -1 && !bad_option) {
while ((c = getopt(argc, argv, "s:S:E:m:")) != -1 && !bad_option) {
switch (c) {
case 's': {
struct netsim_interface nsif;
......@@ -141,6 +143,11 @@ int main(int argc, char *argv[])
eth_latency = strtoull(optarg, NULL, 0) * 1000ULL;
break;
case 'm':
sync_mode = strtol(optarg, NULL, 0);
assert(sync_mode == SYNC_MODES || sync_mode == SYNC_BARRIER);
break;
default:
fprintf(stderr, "unknown option %c\n", c);
bad_option = 1;
......@@ -161,11 +168,14 @@ int main(int argc, char *argv[])
while (!exiting) {
// Sync all interfaces
for (auto &nsif : nsifs) {
if (netsim_n2d_sync(&nsif, cur_ts, eth_latency, sync_period) != 0) {
if (netsim_n2d_sync(&nsif, cur_ts, eth_latency,
sync_period, sync_mode) != 0) {
fprintf(stderr, "netsim_n2d_sync failed\n");
abort();
}
}
netsim_advance_epoch(cur_ts, sync_period, sync_mode);
// Switch packets
uint64_t min_ts;
do {
......@@ -182,7 +192,7 @@ int main(int argc, char *argv[])
// Update cur_ts
if (min_ts < ULLONG_MAX) {
cur_ts = min_ts;
cur_ts = netsim_advance_time(min_ts, sync_period, sync_mode);
}
}
......
net_wire/net_wire.c
View file @ 5728d26c
......@@ -33,6 +33,7 @@
#include <linux/if.h>
#include <linux/if_tun.h>
#include <pcap/pcap.h>
#include <assert.h>
#include <netsim.h>
......@@ -107,9 +108,10 @@ int main(int argc, char *argv[])
uint64_t ts_a, ts_b;
int sync_a, sync_b;
pcap_t *pc = NULL;
int sync_mode = SYNC_MODES;
if (argc < 3 && argc > 6) {
fprintf(stderr, "Usage: net_wire SOCKET-A SOCKET-B [SYNC-PERIOD] "
if (argc < 3 && argc > 7) {
fprintf(stderr, "Usage: net_wire SOCKET-A SOCKET-B [SYNC-MODE] [SYNC-PERIOD] "
"[ETH-LATENCY] [PCAP-FILE]\n");
return EXIT_FAILURE;
}
......@@ -119,12 +121,15 @@ int main(int argc, char *argv[])
signal(SIGUSR1, sigusr1_handler);
if (argc >= 4)
sync_period = strtoull(argv[3], NULL, 0) * 1000ULL;
sync_mode = strtol(argv[3], NULL, 0);
if (argc >= 5)
eth_latency = strtoull(argv[4], NULL, 0) * 1000ULL;
sync_period = strtoull(argv[4], NULL, 0) * 1000ULL;
if (argc >= 6) {
if (argc >= 6)
eth_latency = strtoull(argv[5], NULL, 0) * 1000ULL;
if (argc >= 7) {
pc = pcap_open_dead_with_tstamp_precision(DLT_EN10MB, 65535,
PCAP_TSTAMP_PRECISION_NANO);
if (pc == NULL) {
......@@ -132,9 +137,11 @@ int main(int argc, char *argv[])
return EXIT_FAILURE;
}
dumpfile = pcap_dump_open(pc, argv[5]);
dumpfile = pcap_dump_open(pc, argv[6]);
}
assert(sync_mode == SYNC_MODES || sync_mode == SYNC_BARRIER);
sync_a = sync_b = 1;
if (netsim_init(&nsif_a, argv[1], &sync_a) != 0) {
return -1;
......@@ -145,14 +152,15 @@ int main(int argc, char *argv[])
printf("start polling\n");
while (!exiting) {
if (netsim_n2d_sync(&nsif_a, cur_ts, eth_latency, sync_period) != 0) {
if (netsim_n2d_sync(&nsif_a, cur_ts, eth_latency, sync_period, sync_mode) != 0) {
fprintf(stderr, "netsim_n2d_sync(nsif_a) failed\n");
abort();
}
if (netsim_n2d_sync(&nsif_b, cur_ts, eth_latency, sync_period) != 0) {
if (netsim_n2d_sync(&nsif_b, cur_ts, eth_latency, sync_period, sync_mode) != 0) {
fprintf(stderr, "netsim_n2d_sync(nsif_a) failed\n");
abort();
}
netsim_advance_epoch(cur_ts, sync_period, sync_mode);
do {
move_pkt(&nsif_a, &nsif_b);
......@@ -164,11 +172,12 @@ int main(int argc, char *argv[])
(sync_b && ts_b <= cur_ts)));
if (sync_a && sync_b)
cur_ts = (ts_a <= ts_b ? ts_a : ts_b);
cur_ts = netsim_advance_time(ts_a <= ts_b ? ts_a : ts_b,
sync_period, sync_mode);
else if (sync_a)
cur_ts = ts_a;
cur_ts = netsim_advance_time(ts_a, sync_period, sync_mode);
else if (sync_b)
cur_ts = ts_b;
cur_ts = netsim_advance_time(ts_b, sync_period, sync_mode);
}
if (dumpfile)
......
netsim_common/include/netsim.h
View file @ 5728d26c
......@@ -28,6 +28,9 @@
#include <stdint.h>
#include <cosim_eth_proto.h>
#define SYNC_MODES 0
#define SYNC_BARRIER 1
struct netsim_interface {
uint8_t *d2n_queue;
size_t d2n_pos;
......@@ -61,6 +64,8 @@ volatile union cosim_eth_proto_n2d *netsim_n2d_alloc(
struct netsim_interface *nsif, uint64_t timestamp,
uint64_t latency);
int netsim_n2d_sync(struct netsim_interface *nsif, uint64_t timestamp,
uint64_t latency, uint64_t sync_delay);
uint64_t latency, uint64_t sync_delay, int sync_mode);
void netsim_advance_epoch(uint64_t timestamp, uint64_t sync_delay, int sync_mode);
uint64_t netsim_advance_time(uint64_t timestamp, uint64_t sync_delay, int sync_mode);
#endif /* ndef COSIM_NETSIM_H_ */
netsim_common/netsim.c
View file @ 5728d26c
......@@ -31,6 +31,8 @@
#include <netsim.h>
#include "internal.h"
static uint64_t current_epoch = 0;
int netsim_init(struct netsim_interface *nsif,
const char *eth_socket_path, int *sync_eth)
{
......@@ -137,17 +139,32 @@ volatile union cosim_eth_proto_n2d *netsim_n2d_alloc(
}
int netsim_n2d_sync(struct netsim_interface *nsif, uint64_t timestamp,
uint64_t latency, uint64_t sync_delay)
uint64_t latency, uint64_t sync_delay, int sync_mode)
{
volatile union cosim_eth_proto_n2d *msg;
volatile struct cosim_eth_proto_n2d_sync *sync;
int do_sync;
if (!nsif->sync)
return 0;
if (nsif->n2d_timestamp != 0 &&
timestamp - nsif->n2d_timestamp < sync_delay)
switch (sync_mode) {
case SYNC_MODES:
do_sync = nsif->n2d_timestamp == 0 ||
timestamp - nsif->n2d_timestamp >= sync_delay;
break;
case SYNC_BARRIER:
do_sync = current_epoch == 0 ||
timestamp - current_epoch >= sync_delay;
break;
default:
fprintf(stderr, "unsupported sync mode=%u\n", sync_mode);
return 0;
}
if (!do_sync) {
return 0;
}
msg = netsim_n2d_alloc(nsif, timestamp, latency);
if (msg == NULL)
......@@ -159,3 +176,26 @@ int netsim_n2d_sync(struct netsim_interface *nsif, uint64_t timestamp,
return 0;
}
void netsim_advance_epoch(uint64_t timestamp, uint64_t sync_delay, int sync_mode)
{
if (sync_mode == SYNC_BARRIER) {
if (timestamp - current_epoch >= sync_delay) {
current_epoch = timestamp;
}
}
}
uint64_t netsim_advance_time(uint64_t timestamp, uint64_t sync_delay, int sync_mode)
{
switch (sync_mode) {
case SYNC_MODES:
return timestamp;
case SYNC_BARRIER:
return timestamp < current_epoch + sync_delay ?
timestamp : current_epoch + sync_delay;
default:
fprintf(stderr, "unsupported sync mode=%u\n", sync_mode);
return timestamp;
}
}
nicsim_common/include/nicsim.h
View file @ 5728d26c
......@@ -27,6 +27,9 @@
#include <cosim_pcie_proto.h>
#include <cosim_eth_proto.h>
#define SYNC_MODES 0 // ModES style synchronization
#define SYNC_BARRIER 1 // Global barrier style synchronization
struct nicsim_params {
const char *pci_socket_path;
const char *eth_socket_path;
......@@ -38,6 +41,7 @@ struct nicsim_params {
int sync_pci;
int sync_eth;
int sync_mode;
};
int nicsim_init(struct nicsim_params *params,
......@@ -45,7 +49,9 @@ int nicsim_init(struct nicsim_params *params,
void nicsim_cleanup(void);
int nicsim_sync(struct nicsim_params *params, uint64_t timestamp);
uint64_t netsim_next_timestamp(struct nicsim_params *params);
void nicsim_advance_epoch(struct nicsim_params *params, uint64_t timestamp);
uint64_t nicsim_advance_time(struct nicsim_params *params, uint64_t timestamp);
uint64_t nicsim_next_timestamp(struct nicsim_params *params);
volatile union cosim_pcie_proto_h2d *nicif_h2d_poll(
struct nicsim_params *params, uint64_t timestamp);
......
nicsim_common/nicsim.c
View file @ 5728d26c
......@@ -66,6 +66,8 @@ static uint64_t pci_last_tx_time = 0;
static uint64_t eth_last_rx_time = 0;
static uint64_t eth_last_tx_time = 0;
static uint64_t current_epoch = 0;
static int shm_fd = -1;
static int pci_cfd = -1;
static int eth_cfd = -1;
......@@ -266,8 +268,23 @@ int nicsim_sync(struct nicsim_params *params, uint64_t timestamp)
volatile union cosim_eth_proto_d2n *d2n;
/* sync PCI if necessary */
if (params->sync_pci && (pci_last_tx_time == 0 ||
timestamp - pci_last_tx_time >= params->sync_delay))
if (params->sync_pci) {
int sync;
switch (params->sync_mode) {
case SYNC_MODES:
sync = pci_last_tx_time == 0 ||
timestamp - pci_last_tx_time >= params->sync_delay;
break;
case SYNC_BARRIER:
sync = current_epoch == 0 ||
timestamp - current_epoch >= params->sync_delay;
break;
default:
fprintf(stderr, "unsupported sync mode=%u\n", params->sync_mode);
return ret;
}
if (sync)
{
d2h = nicsim_d2h_alloc(params, timestamp);
if (d2h == NULL) {
......@@ -277,10 +294,26 @@ int nicsim_sync(struct nicsim_params *params, uint64_t timestamp)
COSIM_PCIE_PROTO_D2H_OWN_HOST;
}
}
}
/* sync Ethernet if necessary */
if (params->sync_eth && (eth_last_tx_time == 0 ||
timestamp - eth_last_tx_time >= params->sync_delay))
if (params->sync_eth) {
int sync;
switch (params->sync_mode) {
case SYNC_MODES:
sync = eth_last_tx_time == 0 ||
timestamp - eth_last_tx_time >= params->sync_delay;
break;
case SYNC_BARRIER:
sync = current_epoch == 0 ||
timestamp - current_epoch >= params->sync_delay;
break;
default:
fprintf(stderr, "unsupported sync mode=%u\n", params->sync_mode);
return ret;
}
if (sync)
{
d2n = nicsim_d2n_alloc(params, timestamp);
if (d2n == NULL) {
......@@ -290,11 +323,36 @@ int nicsim_sync(struct nicsim_params *params, uint64_t timestamp)
COSIM_ETH_PROTO_D2N_OWN_NET;
}
}
}
return ret;
}
uint64_t netsim_next_timestamp(struct nicsim_params *params)
void nicsim_advance_epoch(struct nicsim_params *params, uint64_t timestamp)
{
if (params->sync_mode == SYNC_BARRIER) {
if ((params->sync_pci || params->sync_eth) &&
timestamp - current_epoch >= params->sync_delay) {
current_epoch = timestamp;
}
}
}
uint64_t nicsim_advance_time(struct nicsim_params *params, uint64_t timestamp)
{
switch (params->sync_mode) {
case SYNC_MODES:
return timestamp;
case SYNC_BARRIER:
return timestamp < current_epoch + params->sync_delay ?
timestamp : current_epoch + params->sync_delay;
default:
fprintf(stderr, "unsupported sync mode=%u\n", params->sync_mode);
return timestamp;
}
}
uint64_t nicsim_next_timestamp(struct nicsim_params *params)
{
if (params->sync_pci && params->sync_eth) {
return (pci_last_rx_time <= eth_last_rx_time ? pci_last_rx_time :
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment