Skip to content

GitLab

Commit 3ff6bad1 authored by Hejing Li's avatar Hejing Li
Browse files

Merge branch 'master' into experiments

parents 68f7d33f 8073a95e
Hide whitespace changes
Inline Side-by-side
Showing with 208 additions and 62 deletions
corundum/corundum_verilator.cpp
View file @ 3ff6bad1
......@@ -888,26 +888,29 @@ int main(int argc, char *argv[])
{
char *vargs[2] = { argv[0], NULL };
Verilated::commandArgs(1, vargs);
int sync_mode = SYNC_MODES;
#ifdef TRACE_ENABLED
Verilated::traceEverOn(true);
#endif
if (argc < 4 && argc > 9) {
if (argc < 4 && argc > 10) {
fprintf(stderr, "Usage: corundum_verilator PCI-SOCKET ETH-SOCKET "
"SHM [START-TICK] [SYNC-PERIOD] [PCI-LATENCY] [ETH-LATENCY] "
"SHM [SYNC-MODE] [START-TICK] [SYNC-PERIOD] [PCI-LATENCY] [ETH-LATENCY] "
"[CLOCK-FREQ-MHZ]\n");
return EXIT_FAILURE;
}
if (argc >= 5)
main_time = strtoull(argv[4], NULL, 0);
sync_mode = strtol(argv[4], NULL, 0);
if (argc >= 6)
sync_period = strtoull(argv[5], NULL, 0) * 1000ULL;
main_time = strtoull(argv[5], NULL, 0);
if (argc >= 7)
pci_latency = strtoull(argv[6], NULL, 0) * 1000ULL;
sync_period = strtoull(argv[6], NULL, 0) * 1000ULL;
if (argc >= 8)
eth_latency = strtoull(argv[7], NULL, 0) * 1000ULL;
pci_latency = strtoull(argv[7], NULL, 0) * 1000ULL;
if (argc >= 9)
clock_period = 1000000ULL / strtoull(argv[8], NULL, 0);
eth_latency = strtoull(argv[8], NULL, 0) * 1000ULL;
if (argc >= 10)
clock_period = 1000000ULL / strtoull(argv[9], NULL, 0);
struct cosim_pcie_proto_dev_intro di;
memset(&di, 0, sizeof(di));
......@@ -930,6 +933,9 @@ int main(int argc, char *argv[])
nsparams.pci_latency = pci_latency;
nsparams.eth_latency = eth_latency;
nsparams.sync_delay = sync_period;
assert(sync_mode == SYNC_MODES || sync_mode == SYNC_BARRIER);
nsparams.sync_mode = sync_mode;
if (nicsim_init(&nsparams, &di)) {
return EXIT_FAILURE;
}
......@@ -1056,12 +1062,13 @@ int main(int argc, char *argv[])
std::cerr << "warn: nicsim_sync failed (t=" << main_time << ")" <<
std::endl;
}
nicsim_advance_epoch(&nsparams, main_time);
do {
poll_h2d(mmio);
poll_n2d(rx);
} while ((nsparams.sync_pci || nsparams.sync_eth) &&
netsim_next_timestamp(&nsparams) <= main_time && !exiting);
nicsim_next_timestamp(&nsparams) <= main_time && !exiting);
/* falling edge */
top->clk = !top->clk;
......
experiments/modes/simulators.py
View file @ 3ff6bad1
......@@ -22,6 +22,7 @@ class HostSim(Simulator):
sleep = 0
cpu_freq = '3GHz'
sync_mode = 0
sync_period = 500
pci_latency = 500
......@@ -42,6 +43,7 @@ class NICSim(Simulator):
network = None
name = ''
sync_mode = 0
sync_period = 500
pci_latency = 500
eth_latency = 500
......@@ -51,10 +53,10 @@ class NICSim(Simulator):
net.nics.append(self)
def basic_run_cmd(self, env, name, extra=None):
cmd = '%s/%s %s %s %s 0 %d %d %d' % \
cmd = '%s/%s %s %s %s %d 0 %d %d %d' % \
(env.repodir, name, env.nic_pci_path(self), env.nic_eth_path(self),
env.nic_shm_path(self), self.sync_period, self.pci_latency,
self.eth_latency)
env.nic_shm_path(self), self.sync_mode, self.sync_period,
self.pci_latency, self.eth_latency)
if extra is not None:
cmd += ' ' + extra
......@@ -66,6 +68,7 @@ class NICSim(Simulator):
class NetSim(Simulator):
name = ''
opt = ''
sync_mode = 0
sync_period = 500
eth_latency = 500
......@@ -126,6 +129,7 @@ class QemuHost(HostSim):
cmd += f'-device cosim-pci,chardev=cosimcd'
if self.sync:
cmd += ',sync=on'
cmd += f',sync-mode={self.sync_mode}'
cmd += f',pci-latency={self.pci_latency}'
cmd += f',sync-period={self.sync_period}'
else:
......@@ -179,6 +183,7 @@ class Gem5Host(HostSim):
cmd += f'--cosim-shm={env.nic_shm_path(nic)} '
if cpu_type == 'TimingSimpleCPU':
cmd += '--cosim-sync '
cmd += f'--cosim-sync_mode={self.sync_mode} '
cmd += f'--cosim-pci-lat={self.pci_latency} '
cmd += f'--cosim-sync-int={self.sync_period} '
if isinstance(nic, I40eNIC):
......@@ -211,15 +216,15 @@ class I40eNIC(NICSim):
class WireNet(NetSim):
def run_cmd(self, env):
assert len(self.nics) == 2
return '%s/net_wire/net_wire %s %s %d %d' % \
return '%s/net_wire/net_wire %s %s %d %d %d' % \
(env.repodir, env.nic_eth_path(self.nics[0]),
env.nic_eth_path(self.nics[1]),
self.sync_period, self.eth_latency)
self.sync_mode, self.sync_period, self.eth_latency)
class SwitchNet(NetSim):
def run_cmd(self, env):
cmd = env.repodir + '/net_switch/net_switch'
cmd += f' -S {self.sync_period} -E {self.eth_latency}'
cmd += f' -m {self.sync_mode} -S {self.sync_period} -E {self.eth_latency}'
for n in self.nics:
cmd += ' -s ' + env.nic_eth_path(n)
return cmd
......@@ -315,4 +320,4 @@ def create_dctcp_hosts(e, num, name_prefix, net, nic_class, host_class,
hosts.append(host)
return hosts
\ No newline at end of file
return hosts
libnicbm/nicbm.cc
View file @ 3ff6bad1
......@@ -391,21 +391,23 @@ int Runner::runMain(int argc, char *argv[])
uint64_t sync_period = 100 * 1000ULL;
uint64_t pci_latency = 500 * 1000ULL;
uint64_t eth_latency = 500 * 1000ULL;
int sync_mode = SYNC_MODES;
if (argc < 4 && argc > 8) {
if (argc < 4 && argc > 9) {
fprintf(stderr, "Usage: corundum_bm PCI-SOCKET ETH-SOCKET "
"SHM [START-TICK] [SYNC-PERIOD] [PCI-LATENCY] [ETH-LATENCY]\n");
"SHM [SYNC-MODE] [START-TICK] [SYNC-PERIOD] [PCI-LATENCY] [ETH-LATENCY]\n");
return EXIT_FAILURE;
}
if (argc >= 5)
main_time = strtoull(argv[4], NULL, 0);
sync_mode = strtol(argv[4], NULL, 0);
if (argc >= 6)
sync_period = strtoull(argv[5], NULL, 0) * 1000ULL;
main_time = strtoull(argv[5], NULL, 0);
if (argc >= 7)
pci_latency = strtoull(argv[6], NULL, 0) * 1000ULL;
sync_period = strtoull(argv[6], NULL, 0) * 1000ULL;
if (argc >= 8)
eth_latency = strtoull(argv[7], NULL, 0) * 1000ULL;
pci_latency = strtoull(argv[7], NULL, 0) * 1000ULL;
if (argc >= 9)
eth_latency = strtoull(argv[8], NULL, 0) * 1000ULL;
signal(SIGINT, sigint_handler);
signal(SIGUSR1, sigusr1_handler);
......@@ -421,6 +423,9 @@ int Runner::runMain(int argc, char *argv[])
nsparams.pci_latency = pci_latency;
nsparams.eth_latency = eth_latency;
nsparams.sync_delay = sync_period;
assert(sync_mode == SYNC_MODES || sync_mode == SYNC_BARRIER);
nsparams.sync_mode = sync_mode;
if (nicsim_init(&nsparams, &dintro)) {
return EXIT_FAILURE;
}
......@@ -433,6 +438,7 @@ int Runner::runMain(int argc, char *argv[])
while (nicsim_sync(&nsparams, main_time)) {
fprintf(stderr, "warn: nicsim_sync failed (t=%lu)\n", main_time);
}
nicsim_advance_epoch(&nsparams, main_time);
do {
poll_h2d();
......@@ -440,7 +446,7 @@ int Runner::runMain(int argc, char *argv[])
event_trigger();
if (is_sync) {
next_ts = netsim_next_timestamp(&nsparams);
next_ts = nicsim_next_timestamp(&nsparams);
if (next_ts > main_time + max_step)
next_ts = main_time + max_step;
} else {
......@@ -452,7 +458,7 @@ int Runner::runMain(int argc, char *argv[])
next_ts = ev_ts;
} while (next_ts <= main_time && !exiting);
main_time = next_ts;
main_time = nicsim_advance_time(&nsparams, next_ts);
}
fprintf(stderr, "exit main_time: %lu\n", main_time);
......
net_switch/net_switch.cc
View file @ 3ff6bad1
......@@ -6,6 +6,7 @@
#include <unistd.h>
#include <vector>
#include <unordered_map>
#include <cassert>
extern "C" {
#include <netsim.h>
......@@ -118,9 +119,10 @@ int main(int argc, char *argv[])
{
int c;
int bad_option = 0;
int sync_mode = SYNC_MODES;
// Parse command line argument
while ((c = getopt(argc, argv, "s:S:E:")) != -1 && !bad_option) {
while ((c = getopt(argc, argv, "s:S:E:m:")) != -1 && !bad_option) {
switch (c) {
case 's': {
struct netsim_interface nsif;
......@@ -141,6 +143,11 @@ int main(int argc, char *argv[])
eth_latency = strtoull(optarg, NULL, 0) * 1000ULL;
break;
case 'm':
sync_mode = strtol(optarg, NULL, 0);
assert(sync_mode == SYNC_MODES || sync_mode == SYNC_BARRIER);
break;
default:
fprintf(stderr, "unknown option %c\n", c);
bad_option = 1;
......@@ -161,11 +168,14 @@ int main(int argc, char *argv[])
while (!exiting) {
// Sync all interfaces
for (auto &nsif : nsifs) {
if (netsim_n2d_sync(&nsif, cur_ts, eth_latency, sync_period) != 0) {
if (netsim_n2d_sync(&nsif, cur_ts, eth_latency,
sync_period, sync_mode) != 0) {
fprintf(stderr, "netsim_n2d_sync failed\n");
abort();
}
}
netsim_advance_epoch(cur_ts, sync_period, sync_mode);
// Switch packets
uint64_t min_ts;
do {
......@@ -182,7 +192,7 @@ int main(int argc, char *argv[])
// Update cur_ts
if (min_ts < ULLONG_MAX) {
cur_ts = min_ts;
cur_ts = netsim_advance_time(min_ts, sync_period, sync_mode);
}
}
......
net_wire/net_wire.c
View file @ 3ff6bad1
......@@ -33,6 +33,7 @@
#include <linux/if.h>
#include <linux/if_tun.h>
#include <pcap/pcap.h>
#include <assert.h>
#include <netsim.h>
......@@ -107,9 +108,10 @@ int main(int argc, char *argv[])
uint64_t ts_a, ts_b;
int sync_a, sync_b;
pcap_t *pc = NULL;
int sync_mode = SYNC_MODES;
if (argc < 3 && argc > 6) {
fprintf(stderr, "Usage: net_wire SOCKET-A SOCKET-B [SYNC-PERIOD] "
if (argc < 3 && argc > 7) {
fprintf(stderr, "Usage: net_wire SOCKET-A SOCKET-B [SYNC-MODE] [SYNC-PERIOD] "
"[ETH-LATENCY] [PCAP-FILE]\n");
return EXIT_FAILURE;
}
......@@ -119,12 +121,15 @@ int main(int argc, char *argv[])
signal(SIGUSR1, sigusr1_handler);
if (argc >= 4)
sync_period = strtoull(argv[3], NULL, 0) * 1000ULL;
sync_mode = strtol(argv[3], NULL, 0);
if (argc >= 5)
eth_latency = strtoull(argv[4], NULL, 0) * 1000ULL;
sync_period = strtoull(argv[4], NULL, 0) * 1000ULL;
if (argc >= 6) {
if (argc >= 6)
eth_latency = strtoull(argv[5], NULL, 0) * 1000ULL;
if (argc >= 7) {
pc = pcap_open_dead_with_tstamp_precision(DLT_EN10MB, 65535,
PCAP_TSTAMP_PRECISION_NANO);
if (pc == NULL) {
......@@ -132,9 +137,11 @@ int main(int argc, char *argv[])
return EXIT_FAILURE;
}
dumpfile = pcap_dump_open(pc, argv[5]);
dumpfile = pcap_dump_open(pc, argv[6]);
}
assert(sync_mode == SYNC_MODES || sync_mode == SYNC_BARRIER);
sync_a = sync_b = 1;
if (netsim_init(&nsif_a, argv[1], &sync_a) != 0) {
return -1;
......@@ -145,14 +152,15 @@ int main(int argc, char *argv[])
printf("start polling\n");
while (!exiting) {
if (netsim_n2d_sync(&nsif_a, cur_ts, eth_latency, sync_period) != 0) {
if (netsim_n2d_sync(&nsif_a, cur_ts, eth_latency, sync_period, sync_mode) != 0) {
fprintf(stderr, "netsim_n2d_sync(nsif_a) failed\n");
abort();
}
if (netsim_n2d_sync(&nsif_b, cur_ts, eth_latency, sync_period) != 0) {
if (netsim_n2d_sync(&nsif_b, cur_ts, eth_latency, sync_period, sync_mode) != 0) {
fprintf(stderr, "netsim_n2d_sync(nsif_a) failed\n");
abort();
}
netsim_advance_epoch(cur_ts, sync_period, sync_mode);
do {
move_pkt(&nsif_a, &nsif_b);
......@@ -164,11 +172,12 @@ int main(int argc, char *argv[])
(sync_b && ts_b <= cur_ts)));
if (sync_a && sync_b)
cur_ts = (ts_a <= ts_b ? ts_a : ts_b);
cur_ts = netsim_advance_time(ts_a <= ts_b ? ts_a : ts_b,
sync_period, sync_mode);
else if (sync_a)
cur_ts = ts_a;
cur_ts = netsim_advance_time(ts_a, sync_period, sync_mode);
else if (sync_b)
cur_ts = ts_b;
cur_ts = netsim_advance_time(ts_b, sync_period, sync_mode);
}
if (dumpfile)
......
netsim_common/include/netsim.h
View file @ 3ff6bad1
......@@ -28,6 +28,9 @@
#include <stdint.h>
#include <cosim_eth_proto.h>
#define SYNC_MODES 0
#define SYNC_BARRIER 1
struct netsim_interface {
uint8_t *d2n_queue;
size_t d2n_pos;
......@@ -61,6 +64,8 @@ volatile union cosim_eth_proto_n2d *netsim_n2d_alloc(
struct netsim_interface *nsif, uint64_t timestamp,
uint64_t latency);
int netsim_n2d_sync(struct netsim_interface *nsif, uint64_t timestamp,
uint64_t latency, uint64_t sync_delay);
uint64_t latency, uint64_t sync_delay, int sync_mode);
void netsim_advance_epoch(uint64_t timestamp, uint64_t sync_delay, int sync_mode);
uint64_t netsim_advance_time(uint64_t timestamp, uint64_t sync_delay, int sync_mode);
#endif /* ndef COSIM_NETSIM_H_ */
netsim_common/netsim.c
View file @ 3ff6bad1
......@@ -31,6 +31,8 @@
#include <netsim.h>
#include "internal.h"
static uint64_t current_epoch = 0;
int netsim_init(struct netsim_interface *nsif,
const char *eth_socket_path, int *sync_eth)
{
......@@ -137,17 +139,32 @@ volatile union cosim_eth_proto_n2d *netsim_n2d_alloc(
}
int netsim_n2d_sync(struct netsim_interface *nsif, uint64_t timestamp,
uint64_t latency, uint64_t sync_delay)
uint64_t latency, uint64_t sync_delay, int sync_mode)
{
volatile union cosim_eth_proto_n2d *msg;
volatile struct cosim_eth_proto_n2d_sync *sync;
int do_sync;
if (!nsif->sync)
return 0;
if (nsif->n2d_timestamp != 0 &&
timestamp - nsif->n2d_timestamp < sync_delay)
switch (sync_mode) {
case SYNC_MODES:
do_sync = nsif->n2d_timestamp == 0 ||
timestamp - nsif->n2d_timestamp >= sync_delay;
break;
case SYNC_BARRIER:
do_sync = current_epoch == 0 ||
timestamp - current_epoch >= sync_delay;
break;
default:
fprintf(stderr, "unsupported sync mode=%u\n", sync_mode);
return 0;
}
if (!do_sync) {
return 0;
}
msg = netsim_n2d_alloc(nsif, timestamp, latency);
if (msg == NULL)
......@@ -159,3 +176,26 @@ int netsim_n2d_sync(struct netsim_interface *nsif, uint64_t timestamp,
return 0;
}
void netsim_advance_epoch(uint64_t timestamp, uint64_t sync_delay, int sync_mode)
{
if (sync_mode == SYNC_BARRIER) {
if (timestamp - current_epoch >= sync_delay) {
current_epoch = timestamp;
}
}
}
uint64_t netsim_advance_time(uint64_t timestamp, uint64_t sync_delay, int sync_mode)
{
switch (sync_mode) {
case SYNC_MODES:
return timestamp;
case SYNC_BARRIER:
return timestamp < current_epoch + sync_delay ?
timestamp : current_epoch + sync_delay;
default:
fprintf(stderr, "unsupported sync mode=%u\n", sync_mode);
return timestamp;
}
}
nicsim_common/include/nicsim.h
View file @ 3ff6bad1
......@@ -27,6 +27,9 @@
#include <cosim_pcie_proto.h>
#include <cosim_eth_proto.h>
#define SYNC_MODES 0 // ModES style synchronization
#define SYNC_BARRIER 1 // Global barrier style synchronization
struct nicsim_params {
const char *pci_socket_path;
const char *eth_socket_path;
......@@ -38,6 +41,7 @@ struct nicsim_params {
int sync_pci;
int sync_eth;
int sync_mode;
};
int nicsim_init(struct nicsim_params *params,
......@@ -45,7 +49,9 @@ int nicsim_init(struct nicsim_params *params,
void nicsim_cleanup(void);
int nicsim_sync(struct nicsim_params *params, uint64_t timestamp);
uint64_t netsim_next_timestamp(struct nicsim_params *params);
void nicsim_advance_epoch(struct nicsim_params *params, uint64_t timestamp);
uint64_t nicsim_advance_time(struct nicsim_params *params, uint64_t timestamp);
uint64_t nicsim_next_timestamp(struct nicsim_params *params);
volatile union cosim_pcie_proto_h2d *nicif_h2d_poll(
struct nicsim_params *params, uint64_t timestamp);
......
nicsim_common/nicsim.c
View file @ 3ff6bad1
......@@ -66,6 +66,8 @@ static uint64_t pci_last_tx_time = 0;
static uint64_t eth_last_rx_time = 0;
static uint64_t eth_last_tx_time = 0;
static uint64_t current_epoch = 0;
static int shm_fd = -1;
static int pci_cfd = -1;
static int eth_cfd = -1;
......@@ -266,35 +268,91 @@ int nicsim_sync(struct nicsim_params *params, uint64_t timestamp)
volatile union cosim_eth_proto_d2n *d2n;
/* sync PCI if necessary */
if (params->sync_pci && (pci_last_tx_time == 0 ||
timestamp - pci_last_tx_time >= params->sync_delay))
{
d2h = nicsim_d2h_alloc(params, timestamp);
if (d2h == NULL) {
ret = -1;
} else {
d2h->sync.own_type = COSIM_PCIE_PROTO_D2H_MSG_SYNC |
COSIM_PCIE_PROTO_D2H_OWN_HOST;
if (params->sync_pci) {
int sync;
switch (params->sync_mode) {
case SYNC_MODES:
sync = pci_last_tx_time == 0 ||
timestamp - pci_last_tx_time >= params->sync_delay;
break;
case SYNC_BARRIER:
sync = current_epoch == 0 ||
timestamp - current_epoch >= params->sync_delay;
break;
default:
fprintf(stderr, "unsupported sync mode=%u\n", params->sync_mode);
return ret;
}
if (sync)
{
d2h = nicsim_d2h_alloc(params, timestamp);
if (d2h == NULL) {
ret = -1;
} else {
d2h->sync.own_type = COSIM_PCIE_PROTO_D2H_MSG_SYNC |
COSIM_PCIE_PROTO_D2H_OWN_HOST;
}
}
}
/* sync Ethernet if necessary */
if (params->sync_eth && (eth_last_tx_time == 0 ||
timestamp - eth_last_tx_time >= params->sync_delay))
{
d2n = nicsim_d2n_alloc(params, timestamp);
if (d2n == NULL) {
ret = -1;
} else {
d2n->sync.own_type = COSIM_ETH_PROTO_D2N_MSG_SYNC |
COSIM_ETH_PROTO_D2N_OWN_NET;
if (params->sync_eth) {
int sync;
switch (params->sync_mode) {
case SYNC_MODES:
sync = eth_last_tx_time == 0 ||
timestamp - eth_last_tx_time >= params->sync_delay;
break;
case SYNC_BARRIER:
sync = current_epoch == 0 ||
timestamp - current_epoch >= params->sync_delay;
break;
default:
fprintf(stderr, "unsupported sync mode=%u\n", params->sync_mode);
return ret;
}
if (sync)
{
d2n = nicsim_d2n_alloc(params, timestamp);
if (d2n == NULL) {
ret = -1;
} else {
d2n->sync.own_type = COSIM_ETH_PROTO_D2N_MSG_SYNC |
COSIM_ETH_PROTO_D2N_OWN_NET;
}
}
}
return ret;
}
uint64_t netsim_next_timestamp(struct nicsim_params *params)
void nicsim_advance_epoch(struct nicsim_params *params, uint64_t timestamp)
{
if (params->sync_mode == SYNC_BARRIER) {
if ((params->sync_pci || params->sync_eth) &&
timestamp - current_epoch >= params->sync_delay) {
current_epoch = timestamp;
}
}
}
uint64_t nicsim_advance_time(struct nicsim_params *params, uint64_t timestamp)
{
switch (params->sync_mode) {
case SYNC_MODES:
return timestamp;
case SYNC_BARRIER:
return timestamp < current_epoch + params->sync_delay ?
timestamp : current_epoch + params->sync_delay;
default:
fprintf(stderr, "unsupported sync mode=%u\n", params->sync_mode);
return timestamp;
}
}
uint64_t nicsim_next_timestamp(struct nicsim_params *params)
{
if (params->sync_pci && params->sync_eth) {
return (pci_last_rx_time <= eth_last_rx_time ? pci_last_rx_time :
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment