Skip to content

GitLab

Commit d4666c97 authored by Antoine Kaufmann's avatar Antoine Kaufmann
Browse files

Reformat closer to google style

parent eb125a88
Expand all Show whitespace changes
Inline Side-by-side
Showing with 3623 additions and 3840 deletions
sims/nic/corundum_bm/corundum_bm.h
View file @ d4666c97
......@@ -24,9 +24,10 @@
#pragma once
#include <stdint.h>
#include <list>
#include <vector>
#include <stdint.h>
extern "C" {
#include <simbricks/proto/pcie.h>
}
......@@ -188,7 +189,7 @@ struct DMAOp : public nicbm::DMAOp {
};
class DescRing {
public:
public:
DescRing();
~DescRing();
......@@ -207,7 +208,7 @@ public:
virtual void dmaDone(DMAOp *op) = 0;
protected:
protected:
bool empty();
bool full();
bool updatePtr(ptr_t ptr, bool head);
......@@ -227,7 +228,7 @@ protected:
};
class EventRing : public DescRing {
public:
public:
EventRing();
~EventRing();
......@@ -236,14 +237,14 @@ public:
};
class CplRing : public DescRing {
public:
public:
CplRing(EventRing *eventRing);
~CplRing();
void dmaDone(DMAOp *op) override;
void complete(unsigned index, size_t len, bool tx);
private:
private:
struct CplData {
unsigned index;
size_t len;
......@@ -254,31 +255,31 @@ private:
};
class TxRing : public DescRing {
public:
public:
TxRing(CplRing *cplRing);
~TxRing();
void setHeadPtr(ptr_t ptr) override;
void dmaDone(DMAOp *op) override;
private:
private:
CplRing *txCplRing;
};
class RxRing : public DescRing {
public:
public:
RxRing(CplRing *cplRing);
~RxRing();
void dmaDone(DMAOp *op) override;
void rx(RxData *rx_data);
private:
private:
CplRing *rxCplRing;
};
class Port {
public:
public:
Port();
~Port();
......@@ -304,7 +305,7 @@ public:
void queueEnable();
void queueDisable();
private:
private:
unsigned _id;
unsigned _features;
size_t _mtu;
......@@ -318,7 +319,7 @@ private:
};
class Corundum : public nicbm::SimpleDevice<reg_t> {
public:
public:
Corundum();
~Corundum();
......@@ -328,7 +329,7 @@ public:
virtual void dma_complete(nicbm::DMAOp &op);
virtual void eth_rx(uint8_t port, const void *data, size_t len);
private:
private:
EventRing eventRing;
TxRing txRing;
CplRing txCplRing;
......
sims/nic/corundum_bm/tester.c
View file @ d4666c97
......@@ -22,19 +22,18 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include <stdlib.h>
#include <assert.h>
#include <cosim_pcie_proto.h>
#include <fcntl.h>
#include <nicsim.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <sys/mman.h>
#include <unistd.h>
#include <signal.h>
#include <fcntl.h>
#include <assert.h>
#include <nicsim.h>
#include <cosim_pcie_proto.h>
static uint8_t *d2h_queue;
static size_t d2h_pos;
......@@ -46,13 +45,11 @@ static size_t h2d_pos;
static size_t h2d_elen;
static size_t h2d_enum;
static void sigint_handler(int dummy)
{
static void sigint_handler(int dummy) {
exit(1);
}
static int uxsocket_init()
{
static int uxsocket_init() {
int cfd;
if ((cfd = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) {
......@@ -72,8 +69,7 @@ static int uxsocket_init()
return cfd;
}
static int queue_create(const struct cosim_pcie_proto_dev_intro di)
{
static int queue_create(const struct cosim_pcie_proto_dev_intro di) {
int fd = -1;
if ((fd = open("/dev/shm/dummy_nic_shm", O_RDWR)) == -1) {
perror("Failed to open shm file");
......@@ -106,11 +102,9 @@ error:
return -1;
}
volatile union cosim_pcie_proto_h2d *h2d_alloc()
{
volatile union cosim_pcie_proto_h2d *h2d_alloc() {
volatile union cosim_pcie_proto_h2d *msg =
(volatile union cosim_pcie_proto_h2d *)
(h2d_queue + h2d_pos * h2d_elen);
(volatile union cosim_pcie_proto_h2d *)(h2d_queue + h2d_pos * h2d_elen);
if ((msg->dummy.own_type & COSIM_PCIE_PROTO_H2D_OWN_MASK) !=
COSIM_PCIE_PROTO_H2D_OWN_HOST) {
......@@ -122,12 +116,10 @@ volatile union cosim_pcie_proto_h2d *h2d_alloc()
return msg;
}
volatile union cosim_pcie_proto_d2h *d2h_poll()
{
volatile union cosim_pcie_proto_d2h *d2h_poll() {
volatile union cosim_pcie_proto_d2h *msg;
msg = (volatile union cosim_pcie_proto_d2h *)
(d2h_queue + d2h_pos * d2h_elen);
msg = (volatile union cosim_pcie_proto_d2h *)(d2h_queue + d2h_pos * d2h_elen);
if ((msg->dummy.own_type & COSIM_PCIE_PROTO_D2H_OWN_MASK) ==
COSIM_PCIE_PROTO_D2H_OWN_DEV) {
return NULL;
......@@ -135,23 +127,20 @@ volatile union cosim_pcie_proto_d2h *d2h_poll()
return msg;
}
void d2h_done(volatile union cosim_pcie_proto_d2h *msg)
{
msg->dummy.own_type = (msg->dummy.own_type & COSIM_PCIE_PROTO_D2H_MSG_MASK)
| COSIM_PCIE_PROTO_D2H_OWN_DEV;
void d2h_done(volatile union cosim_pcie_proto_d2h *msg) {
msg->dummy.own_type = (msg->dummy.own_type & COSIM_PCIE_PROTO_D2H_MSG_MASK) |
COSIM_PCIE_PROTO_D2H_OWN_DEV;
d2h_pos = (d2h_pos + 1) % d2h_enum;
}
static void dev_read(uint64_t offset, uint16_t len)
{
static void dev_read(uint64_t offset, uint16_t len) {
volatile union cosim_pcie_proto_h2d *h2d_msg = h2d_alloc();
volatile struct cosim_pcie_proto_h2d_read *read = &h2d_msg->read;
read->req_id = 0xF;
read->offset = offset;
read->len = len;
read->bar = 0;
read->own_type = COSIM_PCIE_PROTO_H2D_MSG_READ |
COSIM_PCIE_PROTO_H2D_OWN_DEV;
read->own_type = COSIM_PCIE_PROTO_H2D_MSG_READ | COSIM_PCIE_PROTO_H2D_OWN_DEV;
volatile union cosim_pcie_proto_d2h *d2h_msg = NULL;
while (d2h_msg == NULL) {
......@@ -169,8 +158,7 @@ static void dev_read(uint64_t offset, uint16_t len)
d2h_done(d2h_msg);
}
int main(int argc, char *argv[])
{
int main(int argc, char *argv[]) {
signal(SIGINT, sigint_handler);
int cfd;
......
sims/nic/i40e_bm/headers.h
View file @ d4666c97
......@@ -46,7 +46,6 @@ struct eth_hdr {
uint16_t type;
} __attribute__((packed));
/******************************************************************************/
/* IPv4 */
......@@ -97,7 +96,6 @@ struct ip_hdr {
uint32_t dest;
} __attribute__((packed));
/******************************************************************************/
/* ARP */
......@@ -118,7 +116,6 @@ struct arp_hdr {
uint32_t tpa;
} __attribute__((packed));
/******************************************************************************/
/* TCP */
......@@ -140,21 +137,24 @@ struct arp_hdr {
#define TCPH_HDRLEN(phdr) (ntohs((phdr)->_hdrlen_rsvd_flags) >> 12)
#define TCPH_FLAGS(phdr) (ntohs((phdr)->_hdrlen_rsvd_flags) & TCP_FLAGS)
#define TCPH_HDRLEN_SET(phdr, len) (phdr)->_hdrlen_rsvd_flags = \
htons(((len) << 12) | TCPH_FLAGS(phdr))
#define TCPH_FLAGS_SET(phdr, flags) (phdr)->_hdrlen_rsvd_flags = \
(((phdr)->_hdrlen_rsvd_flags & PP_HTONS( \
(uint16_t)(~(uint16_t)(TCP_FLAGS)))) | htons(flags))
#define TCPH_HDRLEN_SET(phdr, len) \
(phdr)->_hdrlen_rsvd_flags = htons(((len) << 12) | TCPH_FLAGS(phdr))
#define TCPH_FLAGS_SET(phdr, flags) \
(phdr)->_hdrlen_rsvd_flags = \
(((phdr)->_hdrlen_rsvd_flags & \
PP_HTONS((uint16_t)(~(uint16_t)(TCP_FLAGS)))) | \
htons(flags))
#define TCPH_HDRLEN_FLAGS_SET(phdr, len, flags) \
(phdr)->_hdrlen_rsvd_flags = htons(((len) << 12) | (flags))
#define TCPH_SET_FLAG(phdr, flags) (phdr)->_hdrlen_rsvd_flags = \
((phdr)->_hdrlen_rsvd_flags | htons(flags))
#define TCPH_UNSET_FLAG(phdr, flags) (phdr)->_hdrlen_rsvd_flags = \
htons(ntohs((phdr)->_hdrlen_rsvd_flags) | (TCPH_FLAGS(phdr) & ~(flags)) )
#define TCPH_SET_FLAG(phdr, flags) \
(phdr)->_hdrlen_rsvd_flags = ((phdr)->_hdrlen_rsvd_flags | htons(flags))
#define TCPH_UNSET_FLAG(phdr, flags) \
(phdr)->_hdrlen_rsvd_flags = \
htons(ntohs((phdr)->_hdrlen_rsvd_flags) | (TCPH_FLAGS(phdr) & ~(flags)))
#define TCP_TCPLEN(seg) ((seg)->len + ((TCPH_FLAGS((seg)->tcphdr) & \
(TCP_FIN | TCP_SYN)) != 0))
#define TCP_TCPLEN(seg) \
((seg)->len + ((TCPH_FLAGS((seg)->tcphdr) & (TCP_FIN | TCP_SYN)) != 0))
struct tcp_hdr {
uint16_t src;
......@@ -167,7 +167,6 @@ struct tcp_hdr {
uint16_t urgp;
} __attribute__((packed));
/******************************************************************************/
/* UDP */
......@@ -178,7 +177,6 @@ struct udp_hdr {
uint16_t chksum;
} __attribute__((packed));
/******************************************************************************/
/* whole packets */
......
sims/nic/i40e_bm/i40e_adminq.cc
View file @ d4666c97
......@@ -24,40 +24,39 @@
#include <stdlib.h>
#include <string.h>
#include <cassert>
#include <iostream>
#include "sims/nic/i40e_bm/i40e_bm.h"
#include "sims/nic/i40e_bm/i40e_base_wrapper.h"
#include "sims/nic/i40e_bm/i40e_bm.h"
using namespace i40e;
extern nicbm::Runner *runner;
queue_admin_tx::queue_admin_tx(i40e_bm &dev_, uint64_t &reg_base_,
uint32_t &reg_len_, uint32_t &reg_head_, uint32_t &reg_tail_)
: queue_base("atx", reg_head_, reg_tail_), dev(dev_), reg_base(reg_base_),
reg_len(reg_len_)
{
uint32_t &reg_len_, uint32_t &reg_head_,
uint32_t &reg_tail_)
: queue_base("atx", reg_head_, reg_tail_),
dev(dev_),
reg_base(reg_base_),
reg_len(reg_len_) {
desc_len = 32;
ctxs_init();
}
queue_base::desc_ctx &queue_admin_tx::desc_ctx_create()
{
queue_base::desc_ctx &queue_admin_tx::desc_ctx_create() {
return *new admin_desc_ctx(*this, dev);
}
void queue_admin_tx::reg_updated()
{
void queue_admin_tx::reg_updated() {
base = reg_base;
len = (reg_len & I40E_GL_ATQLEN_ATQLEN_MASK) >> I40E_GL_ATQLEN_ATQLEN_SHIFT;
if (!enabled && (reg_len & I40E_GL_ATQLEN_ATQENABLE_MASK)) {
#ifdef DEBUG_ADMINQ
log << " enable base=" << base << " len=" << len <<
logger::endl;
log << " enable base=" << base << " len=" << len << logger::endl;
#endif
enabled = true;
} else if (enabled && !(reg_len & I40E_GL_ATQLEN_ATQENABLE_MASK)) {
......@@ -72,19 +71,16 @@ void queue_admin_tx::reg_updated()
queue_admin_tx::admin_desc_ctx::admin_desc_ctx(queue_admin_tx &queue_,
i40e_bm &dev_)
: i40e::queue_base::desc_ctx(queue_), aq(queue_), dev(dev_)
{
d = reinterpret_cast <struct i40e_aq_desc *> (desc);
: i40e::queue_base::desc_ctx(queue_), aq(queue_), dev(dev_) {
d = reinterpret_cast<struct i40e_aq_desc *>(desc);
}
void queue_admin_tx::admin_desc_ctx::data_written(uint64_t addr, size_t len)
{
void queue_admin_tx::admin_desc_ctx::data_written(uint64_t addr, size_t len) {
processed();
}
void queue_admin_tx::admin_desc_ctx::desc_compl_prepare(uint16_t retval,
uint16_t extra_flags)
{
uint16_t extra_flags) {
d->flags &= ~0x1ff;
d->flags |= I40E_AQ_FLAG_DD | I40E_AQ_FLAG_CMP | extra_flags;
if (retval)
......@@ -92,24 +88,25 @@ void queue_admin_tx::admin_desc_ctx::desc_compl_prepare(uint16_t retval,
d->retval = retval;
#ifdef DEBUG_ADMINQ
queue.log << " desc_compl_prepare index=" << index << " retval=" <<
retval << logger::endl;
queue.log << " desc_compl_prepare index=" << index << " retval=" << retval
<< logger::endl;
#endif
}
void queue_admin_tx::admin_desc_ctx::desc_complete(uint16_t retval,
uint16_t extra_flags)
{
uint16_t extra_flags) {
desc_compl_prepare(retval, extra_flags);
processed();
}
void queue_admin_tx::admin_desc_ctx::desc_complete_indir(uint16_t retval,
const void *data, size_t len, uint16_t extra_flags, bool ignore_datalen)
{
const void *data,
size_t len,
uint16_t extra_flags,
bool ignore_datalen) {
if (!ignore_datalen && len > d->datalen) {
queue.log << "queue_admin_tx::desc_complete_indir: data too long ("
<< len << ") got buffer for (" << d->datalen << ")" << logger::endl;
queue.log << "queue_admin_tx::desc_complete_indir: data too long (" << len
<< ") got buffer for (" << d->datalen << ")" << logger::endl;
abort();
}
d->datalen = len;
......@@ -117,18 +114,17 @@ void queue_admin_tx::admin_desc_ctx::desc_complete_indir(uint16_t retval,
desc_compl_prepare(retval, extra_flags);
uint64_t addr = d->params.external.addr_low |
(((uint64_t) d->params.external.addr_high) << 32);
(((uint64_t)d->params.external.addr_high) << 32);
data_write(addr, len, data);
}
void queue_admin_tx::admin_desc_ctx::prepare()
{
void queue_admin_tx::admin_desc_ctx::prepare() {
if ((d->flags & I40E_AQ_FLAG_RD)) {
uint64_t addr = d->params.external.addr_low |
(((uint64_t) d->params.external.addr_high) << 32);
(((uint64_t)d->params.external.addr_high) << 32);
#ifdef DEBUG_ADMINQ
queue.log << " desc with buffer opc=" << d->opcode << " addr=" <<
addr << logger::endl;
queue.log << " desc with buffer opc=" << d->opcode << " addr=" << addr
<< logger::endl;
#endif
data_fetch(addr, d->datalen);
} else {
......@@ -136,8 +132,7 @@ void queue_admin_tx::admin_desc_ctx::prepare()
}
}
void queue_admin_tx::admin_desc_ctx::process()
{
void queue_admin_tx::admin_desc_ctx::process() {
#ifdef DEBUG_ADMINQ
queue.log << " descriptor " << index << " fetched" << logger::endl;
#endif
......@@ -161,8 +156,7 @@ void queue_admin_tx::admin_desc_ctx::process()
queue.log << " request resource" << logger::endl;
#endif
struct i40e_aqc_request_resource *rr =
reinterpret_cast<struct i40e_aqc_request_resource *>(
d->params.raw);
reinterpret_cast<struct i40e_aqc_request_resource *>(d->params.raw);
rr->timeout = 180000;
#ifdef DEBUG_ADMINQ
queue.log << " res_id=" << rr->resource_id << logger::endl;
......@@ -175,8 +169,7 @@ void queue_admin_tx::admin_desc_ctx::process()
#endif
#ifdef DEBUG_ADMINQ
struct i40e_aqc_request_resource *rr =
reinterpret_cast<struct i40e_aqc_request_resource *>(
d->params.raw);
reinterpret_cast<struct i40e_aqc_request_resource *>(d->params.raw);
queue.log << " res_id=" << rr->resource_id << logger::endl;
queue.log << " res_nu=" << rr->resource_number << logger::endl;
#endif
......@@ -193,16 +186,15 @@ void queue_admin_tx::admin_desc_ctx::process()
queue.log << " get dev/fun caps" << logger::endl;
#endif
struct i40e_aqc_list_capabilites *lc =
reinterpret_cast<struct i40e_aqc_list_capabilites *>(
d->params.raw);
reinterpret_cast<struct i40e_aqc_list_capabilites *>(d->params.raw);
struct i40e_aqc_list_capabilities_element_resp caps[] = {
{ I40E_AQ_CAP_ID_RSS, 1, 0, 512, 6, 0, {} },
{ I40E_AQ_CAP_ID_RXQ, 1, 0, dev.NUM_QUEUES, 0, 0, {} },
{ I40E_AQ_CAP_ID_TXQ, 1, 0, dev.NUM_QUEUES, 0, 0, {} },
{ I40E_AQ_CAP_ID_MSIX, 1, 0, dev.NUM_PFINTS, 0, 0, {} },
{ I40E_AQ_CAP_ID_VSI, 1, 0, dev.NUM_VSIS, 0, 0, {} },
{ I40E_AQ_CAP_ID_DCB, 1, 0, 1, 1, 1, {} },
{I40E_AQ_CAP_ID_RSS, 1, 0, 512, 6, 0, {}},
{I40E_AQ_CAP_ID_RXQ, 1, 0, dev.NUM_QUEUES, 0, 0, {}},
{I40E_AQ_CAP_ID_TXQ, 1, 0, dev.NUM_QUEUES, 0, 0, {}},
{I40E_AQ_CAP_ID_MSIX, 1, 0, dev.NUM_PFINTS, 0, 0, {}},
{I40E_AQ_CAP_ID_VSI, 1, 0, dev.NUM_VSIS, 0, 0, {}},
{I40E_AQ_CAP_ID_DCB, 1, 0, 1, 1, 1, {}},
};
size_t num_caps = sizeof(caps) / sizeof(caps[0]);
......@@ -231,8 +223,7 @@ void queue_admin_tx::admin_desc_ctx::process()
queue.log << " read mac" << logger::endl;
#endif
struct i40e_aqc_mac_address_read *ar =
reinterpret_cast<struct i40e_aqc_mac_address_read *>(
d->params.raw);
reinterpret_cast<struct i40e_aqc_mac_address_read *>(d->params.raw);
struct i40e_aqc_mac_address_read_data ard;
uint64_t mac = runner->get_mac_addr();
......@@ -253,8 +244,7 @@ void queue_admin_tx::admin_desc_ctx::process()
par.phy_type = (1ULL << I40E_PHY_TYPE_40GBASE_CR4_CU);
par.link_speed = I40E_LINK_SPEED_40GB;
par.abilities = I40E_AQ_PHY_LINK_ENABLED |
I40E_AQ_PHY_AN_ENABLED;
par.abilities = I40E_AQ_PHY_LINK_ENABLED | I40E_AQ_PHY_AN_ENABLED;
par.eee_capability = 0;
d->params.external.param0 = 0;
......@@ -266,8 +256,7 @@ void queue_admin_tx::admin_desc_ctx::process()
queue.log << " link status" << logger::endl;
#endif
struct i40e_aqc_get_link_status *gls =
reinterpret_cast<struct i40e_aqc_get_link_status *>(
d->params.raw);
reinterpret_cast<struct i40e_aqc_get_link_status *>(d->params.raw);
gls->command_flags &= I40E_AQ_LSE_IS_ENABLED; // should actually return
// status of link status
......@@ -279,8 +268,7 @@ void queue_admin_tx::admin_desc_ctx::process()
// might need qualified module
gls->an_info = I40E_AQ_AN_COMPLETED | I40E_AQ_LP_AN_ABILITY;
gls->ext_info = 0;
gls->loopback = I40E_AQ_LINK_POWER_CLASS_4 <<
I40E_AQ_PWR_CLASS_SHIFT_LB;
gls->loopback = I40E_AQ_LINK_POWER_CLASS_4 << I40E_AQ_PWR_CLASS_SHIFT_LB;
gls->max_frame_size = dev.MAX_MTU;
gls->config = I40E_AQ_CONFIG_CRC_ENA;
......@@ -289,8 +277,8 @@ void queue_admin_tx::admin_desc_ctx::process()
#ifdef DEBUG_ADMINQ
queue.log << " get switch config" << logger::endl;
#endif
struct i40e_aqc_switch_seid *sw = reinterpret_cast<
struct i40e_aqc_switch_seid *>(d->params.raw);
struct i40e_aqc_switch_seid *sw =
reinterpret_cast<struct i40e_aqc_switch_seid *>(d->params.raw);
struct i40e_aqc_get_switch_config_header_resp hr;
/* Not sure why dpdk doesn't like this?
struct i40e_aqc_switch_config_element_resp els[] = {
......@@ -312,15 +300,22 @@ void queue_admin_tx::admin_desc_ctx::process()
};*/
struct i40e_aqc_switch_config_element_resp els[] = {
// VSI PF
{ I40E_AQ_SW_ELEM_TYPE_VSI, I40E_AQ_SW_ELEM_REV_1, 512, 2, 16, {},
I40E_AQ_CONN_TYPE_REGULAR, 0, 0},
{I40E_AQ_SW_ELEM_TYPE_VSI,
I40E_AQ_SW_ELEM_REV_1,
512,
2,
16,
{},
I40E_AQ_CONN_TYPE_REGULAR,
0,
0},
};
// find start idx
size_t cnt = sizeof(els) / sizeof(els[0]);
size_t first = 0;
for (first = 0; first < cnt && els[first].seid < sw->seid; first++) {}
for (first = 0; first < cnt && els[first].seid < sw->seid; first++) {
}
// figure out how many fit in the buffer
size_t max = (d->datalen - sizeof(hr)) / sizeof(els[0]);
......@@ -337,8 +332,8 @@ void queue_admin_tx::admin_desc_ctx::process()
hr.num_reported = report;
hr.num_total = cnt;
#ifdef DEBUG_ADMINQ
queue.log << " report=" << report << " cnt=" << cnt <<
" seid=" << sw->seid << logger::endl;
queue.log << " report=" << report << " cnt=" << cnt
<< " seid=" << sw->seid << logger::endl;
#endif
// create temporary contiguous buffer
......@@ -369,10 +364,9 @@ void queue_admin_tx::admin_desc_ctx::process()
struct i40e_aqc_vsi_properties_data pd;
memset(&pd, 0, sizeof(pd));
pd.valid_sections |= I40E_AQ_VSI_PROP_SWITCH_VALID |
I40E_AQ_VSI_PROP_QUEUE_MAP_VALID |
I40E_AQ_VSI_PROP_QUEUE_OPT_VALID |
I40E_AQ_VSI_PROP_SCHED_VALID;
pd.valid_sections |=
I40E_AQ_VSI_PROP_SWITCH_VALID | I40E_AQ_VSI_PROP_QUEUE_MAP_VALID |
I40E_AQ_VSI_PROP_QUEUE_OPT_VALID | I40E_AQ_VSI_PROP_SCHED_VALID;
desc_complete_indir(0, &pd, sizeof(pd));
} else if (d->opcode == i40e_aqc_opc_update_vsi_parameters) {
#ifdef DEBUG_ADMINQ
......@@ -397,8 +391,7 @@ void queue_admin_tx::admin_desc_ctx::process()
#endif
struct i40e_aqc_query_vsi_bw_config_resp bwc;
memset(&bwc, 0, sizeof(bwc));
for (size_t i = 0; i < 8; i++)
bwc.qs_handles[i] = 0xffff;
for (size_t i = 0; i < 8; i++) bwc.qs_handles[i] = 0xffff;
desc_complete_indir(0, &bwc, sizeof(bwc));
} else if (d->opcode == i40e_aqc_opc_query_vsi_ets_sla_config) {
#ifdef DEBUG_ADMINQ
......@@ -406,18 +399,16 @@ void queue_admin_tx::admin_desc_ctx::process()
#endif
struct i40e_aqc_query_vsi_ets_sla_config_resp sla;
memset(&sla, 0, sizeof(sla));
for (size_t i = 0; i < 8; i++)
sla.share_credits[i] = 127;
for (size_t i = 0; i < 8; i++) sla.share_credits[i] = 127;
desc_complete_indir(0, &sla, sizeof(sla));
} else if (d->opcode == i40e_aqc_opc_remove_macvlan) {
#ifdef DEBUG_ADMINQ
queue.log << " remove macvlan" << logger::endl;
#endif
struct i40e_aqc_macvlan *m = reinterpret_cast<
struct i40e_aqc_macvlan *>(d->params.raw);
struct i40e_aqc_macvlan *m =
reinterpret_cast<struct i40e_aqc_macvlan *>(d->params.raw);
struct i40e_aqc_remove_macvlan_element_data *rve =
reinterpret_cast<struct i40e_aqc_remove_macvlan_element_data *>(
data);
reinterpret_cast<struct i40e_aqc_remove_macvlan_element_data *>(data);
for (uint16_t i = 0; i < m->num_addresses; i++)
rve[i].error_code = I40E_AQC_REMOVE_MACVLAN_SUCCESS;
......
sims/nic/i40e_bm/i40e_base_wrapper.h
View file @ d4666c97
......@@ -17,9 +17,9 @@ typedef uint16_t __le16;
typedef uint32_t __le32;
typedef uint64_t __le64;
#include "sims/nic/i40e_bm/base/i40e_adminq_cmd.h"
#include "sims/nic/i40e_bm/base/i40e_devids.h"
#include "sims/nic/i40e_bm/base/i40e_register.h"
#include "sims/nic/i40e_bm/base/i40e_adminq_cmd.h"
#include "sims/nic/i40e_bm/base/i40e_rxtxq.h"
/* from i40e_types.h */
......
sims/nic/i40e_bm/i40e_bm.cc
View file @ d4666c97
......@@ -22,12 +22,14 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include "sims/nic/i40e_bm/i40e_bm.h"
#include <stdlib.h>
#include <string.h>
#include <cassert>
#include <iostream>
#include "sims/nic/i40e_bm/i40e_bm.h"
#include "sims/nic/i40e_bm/i40e_base_wrapper.h"
nicbm::Runner *runner;
......@@ -37,24 +39,23 @@ namespace i40e {
i40e_bm::i40e_bm()
: log("i40e"),
pf_atq(*this, regs.pf_atqba, regs.pf_atqlen, regs.pf_atqh, regs.pf_atqt),
hmc(*this), shram(*this), lanmgr(*this, NUM_QUEUES)
{
hmc(*this),
shram(*this),
lanmgr(*this, NUM_QUEUES) {
reset(false);
}
i40e_bm::~i40e_bm()
{
i40e_bm::~i40e_bm() {
}
void i40e_bm::setup_intro(struct cosim_pcie_proto_dev_intro &di)
{
void i40e_bm::setup_intro(struct cosim_pcie_proto_dev_intro &di) {
di.bars[BAR_REGS].len = 4 * 1024 * 1024;
di.bars[BAR_REGS].flags = COSIM_PCIE_PROTO_BAR_64;
di.bars[BAR_IO].len = 32;
di.bars[BAR_IO].flags = COSIM_PCIE_PROTO_BAR_IO;
di.bars[BAR_MSIX].len = 32 * 1024;
di.bars[BAR_MSIX].flags = COSIM_PCIE_PROTO_BAR_64 |
COSIM_PCIE_PROTO_BAR_DUMMY;
di.bars[BAR_MSIX].flags =
COSIM_PCIE_PROTO_BAR_64 | COSIM_PCIE_PROTO_BAR_DUMMY;
di.pci_vendor_id = I40E_INTEL_VENDOR_ID;
di.pci_device_id = I40E_DEV_ID_QSFP_A;
......@@ -71,8 +72,7 @@ void i40e_bm::setup_intro(struct cosim_pcie_proto_dev_intro &di)
di.psi_msix_cap_offset = 0x70;
}
void i40e_bm::dma_complete(nicbm::DMAOp &op)
{
void i40e_bm::dma_complete(nicbm::DMAOp &op) {
dma_base &dma = dynamic_cast<dma_base &>(op);
#ifdef DEBUG_DEV
log << "dma_complete(" << &op << ")" << logger::endl;
......@@ -80,17 +80,15 @@ void i40e_bm::dma_complete(nicbm::DMAOp &op)
dma.done();
}
void i40e_bm::eth_rx(uint8_t port, const void *data, size_t len)
{
void i40e_bm::eth_rx(uint8_t port, const void *data, size_t len) {
#ifdef DEBUG_DEV
log << "i40e: received packet len=" << len << logger::endl;
#endif
lanmgr.packet_received(data, len);
}
void i40e_bm::reg_read(uint8_t bar, uint64_t addr, void *dest, size_t len)
{
uint32_t *dest_p = reinterpret_cast <uint32_t *> (dest);
void i40e_bm::reg_read(uint8_t bar, uint64_t addr, void *dest, size_t len) {
uint32_t *dest_p = reinterpret_cast<uint32_t *>(dest);
if (len == 4) {
dest_p[0] = reg_read32(bar, addr);
......@@ -104,21 +102,20 @@ void i40e_bm::reg_read(uint8_t bar, uint64_t addr, void *dest, size_t len)
}
}
uint32_t i40e_bm::reg_read32(uint8_t bar, uint64_t addr)
{
uint32_t i40e_bm::reg_read32(uint8_t bar, uint64_t addr) {
if (bar == BAR_REGS) {
return reg_mem_read32(addr);
} else if (bar == BAR_IO) {
return reg_io_read(addr);
} else {
log << "invalid BAR " << (int) bar << logger::endl;
log << "invalid BAR " << (int)bar << logger::endl;
abort();
}
}
void i40e_bm::reg_write(uint8_t bar, uint64_t addr, const void *src, size_t len)
{
const uint32_t *src_p = reinterpret_cast<const uint32_t *> (src);
void i40e_bm::reg_write(uint8_t bar, uint64_t addr, const void *src,
size_t len) {
const uint32_t *src_p = reinterpret_cast<const uint32_t *>(src);
if (len == 4) {
reg_write32(bar, addr, src_p[0]);
......@@ -132,32 +129,27 @@ void i40e_bm::reg_write(uint8_t bar, uint64_t addr, const void *src, size_t len)
}
}
void i40e_bm::reg_write32(uint8_t bar, uint64_t addr, uint32_t val)
{
void i40e_bm::reg_write32(uint8_t bar, uint64_t addr, uint32_t val) {
if (bar == BAR_REGS) {
reg_mem_write32(addr, val);
} else if (bar == BAR_IO) {
reg_io_write(addr, val);
} else {
log << "invalid BAR " << (int) bar << logger::endl;
log << "invalid BAR " << (int)bar << logger::endl;
abort();
}
}
uint32_t i40e_bm::reg_io_read(uint64_t addr)
{
uint32_t i40e_bm::reg_io_read(uint64_t addr) {
log << "unhandled io read addr=" << addr << logger::endl;
return 0;
}
void i40e_bm::reg_io_write(uint64_t addr, uint32_t val)
{
log << "unhandled io write addr=" << addr << " val="
<< val << logger::endl;
void i40e_bm::reg_io_write(uint64_t addr, uint32_t val) {
log << "unhandled io write addr=" << addr << " val=" << val << logger::endl;
}
uint32_t i40e_bm::reg_mem_read32(uint64_t addr)
{
uint32_t i40e_bm::reg_mem_read32(uint64_t addr) {
uint32_t val = 0;
if (addr >= I40E_PFINT_DYN_CTLN(0) &&
......@@ -176,20 +168,17 @@ uint32_t i40e_bm::reg_mem_read32(uint64_t addr)
} else if (addr >= I40E_QINT_TQCTL(0) &&
addr <= I40E_QINT_TQCTL(NUM_QUEUES - 1)) {
val = regs.qint_tqctl[(addr - I40E_QINT_TQCTL(0)) / 4];
} else if (addr >= I40E_QTX_ENA(0) &&
addr <= I40E_QTX_ENA(NUM_QUEUES - 1)) {
} else if (addr >= I40E_QTX_ENA(0) && addr <= I40E_QTX_ENA(NUM_QUEUES - 1)) {
val = regs.qtx_ena[(addr - I40E_QTX_ENA(0)) / 4];
} else if (addr >= I40E_QTX_TAIL(0) &&
addr <= I40E_QTX_TAIL(NUM_QUEUES - 1)) {
val = regs.qtx_tail[(addr - I40E_QTX_TAIL(0)) / 4];
} else if (addr >= I40E_QTX_CTL(0) &&
addr <= I40E_QTX_CTL(NUM_QUEUES - 1)) {
} else if (addr >= I40E_QTX_CTL(0) && addr <= I40E_QTX_CTL(NUM_QUEUES - 1)) {
val = regs.qtx_ctl[(addr - I40E_QTX_CTL(0)) / 4];
} else if (addr >= I40E_QINT_RQCTL(0) &&
addr <= I40E_QINT_RQCTL(NUM_QUEUES - 1)) {
val = regs.qint_rqctl[(addr - I40E_QINT_RQCTL(0)) / 4];
} else if (addr >= I40E_QRX_ENA(0) &&
addr <= I40E_QRX_ENA(NUM_QUEUES - 1)) {
} else if (addr >= I40E_QRX_ENA(0) && addr <= I40E_QRX_ENA(NUM_QUEUES - 1)) {
val = regs.qrx_ena[(addr - I40E_QRX_ENA(0)) / 4];
} else if (addr >= I40E_QRX_TAIL(0) &&
addr <= I40E_QRX_TAIL(NUM_QUEUES - 1)) {
......@@ -323,7 +312,6 @@ uint32_t i40e_bm::reg_mem_read32(uint64_t addr)
val = regs.gllan_rctl_0;
break;
case I40E_GLHMC_LANTXOBJSZ:
val = 7; // 128 B
break;
......@@ -440,8 +428,7 @@ uint32_t i40e_bm::reg_mem_read32(uint64_t addr)
default:
#ifdef DEBUG_DEV
log << "unhandled mem read addr=" << addr
<< logger::endl;
log << "unhandled mem read addr=" << addr << logger::endl;
#endif
break;
}
......@@ -450,8 +437,7 @@ uint32_t i40e_bm::reg_mem_read32(uint64_t addr)
return val;
}
void i40e_bm::reg_mem_write32(uint64_t addr, uint32_t val)
{
void i40e_bm::reg_mem_write32(uint64_t addr, uint32_t val) {
if (addr >= I40E_PFINT_DYN_CTLN(0) &&
addr <= I40E_PFINT_DYN_CTLN(NUM_PFINTS - 1)) {
regs.pfint_dyn_ctln[(addr - I40E_PFINT_DYN_CTLN(0)) / 4] = val;
......@@ -467,8 +453,7 @@ void i40e_bm::reg_mem_write32(uint64_t addr, uint32_t val)
} else if (addr >= I40E_QINT_TQCTL(0) &&
addr <= I40E_QINT_TQCTL(NUM_QUEUES - 1)) {
regs.qint_tqctl[(addr - I40E_QINT_TQCTL(0)) / 4] = val;
} else if (addr >= I40E_QTX_ENA(0) &&
addr <= I40E_QTX_ENA(NUM_QUEUES - 1)) {
} else if (addr >= I40E_QTX_ENA(0) && addr <= I40E_QTX_ENA(NUM_QUEUES - 1)) {
size_t idx = (addr - I40E_QTX_ENA(0)) / 4;
regs.qtx_ena[idx] = val;
lanmgr.qena_updated(idx, false);
......@@ -477,14 +462,12 @@ void i40e_bm::reg_mem_write32(uint64_t addr, uint32_t val)
size_t idx = (addr - I40E_QTX_TAIL(0)) / 4;
regs.qtx_tail[idx] = val;
lanmgr.tail_updated(idx, false);
} else if (addr >= I40E_QTX_CTL(0) &&
addr <= I40E_QTX_CTL(NUM_QUEUES - 1)) {
} else if (addr >= I40E_QTX_CTL(0) && addr <= I40E_QTX_CTL(NUM_QUEUES - 1)) {
regs.qtx_ctl[(addr - I40E_QTX_CTL(0)) / 4] = val;
} else if (addr >= I40E_QINT_RQCTL(0) &&
addr <= I40E_QINT_RQCTL(NUM_QUEUES - 1)) {
regs.qint_rqctl[(addr - I40E_QINT_RQCTL(0)) / 4] = val;
} else if (addr >= I40E_QRX_ENA(0) &&
addr <= I40E_QRX_ENA(NUM_QUEUES - 1)) {
} else if (addr >= I40E_QRX_ENA(0) && addr <= I40E_QRX_ENA(NUM_QUEUES - 1)) {
size_t idx = (addr - I40E_QRX_ENA(0)) / 4;
regs.qrx_ena[idx] = val;
lanmgr.qena_updated(idx, true);
......@@ -524,8 +507,7 @@ void i40e_bm::reg_mem_write32(uint64_t addr, uint32_t val)
} else {
switch (addr) {
case I40E_PFGEN_CTRL:
if ((val & I40E_PFGEN_CTRL_PFSWR_MASK) ==
I40E_PFGEN_CTRL_PFSWR_MASK)
if ((val & I40E_PFGEN_CTRL_PFSWR_MASK) == I40E_PFGEN_CTRL_PFSWR_MASK)
reset(true);
break;
......@@ -598,8 +580,7 @@ void i40e_bm::reg_mem_write32(uint64_t addr, uint32_t val)
pf_atq.reg_updated();
break;
case I40E_PF_ATQBAH:
regs.pf_atqba = ((uint64_t) val << 32) |
(regs.pf_atqba & 0xffffffffULL);
regs.pf_atqba = ((uint64_t)val << 32) | (regs.pf_atqba & 0xffffffffULL);
pf_atq.reg_updated();
break;
case I40E_PF_ATQLEN:
......@@ -619,8 +600,7 @@ void i40e_bm::reg_mem_write32(uint64_t addr, uint32_t val)
regs.pf_arqba = val | (regs.pf_atqba & 0xffffffff00000000ULL);
break;
case I40E_PF_ARQBAH:
regs.pf_arqba = ((uint64_t) val << 32) |
(regs.pf_arqba & 0xffffffffULL);
regs.pf_arqba = ((uint64_t)val << 32) | (regs.pf_arqba & 0xffffffffULL);
break;
case I40E_PF_ARQLEN:
regs.pf_arqlen = val;
......@@ -663,20 +643,19 @@ void i40e_bm::reg_mem_write32(uint64_t addr, uint32_t val)
break;
default:
#ifdef DEBUG_DEV
log << "unhandled mem write addr=" << addr
<< " val=" << val << logger::endl;
log << "unhandled mem write addr=" << addr << " val=" << val
<< logger::endl;
#endif
break;
}
}
}
void i40e_bm::timed_event(nicbm::TimedEvent &ev)
{
int_ev &iev = *((int_ev *) &ev);
void i40e_bm::timed_event(nicbm::TimedEvent &ev) {
int_ev &iev = *((int_ev *)&ev);
#ifdef DEBUG_DEV
log << "timed_event: triggering interrupt (" << iev.vec << ")" <<
logger::endl;
log << "timed_event: triggering interrupt (" << iev.vec << ")"
<< logger::endl;
#endif
iev.armed = false;
......@@ -690,8 +669,7 @@ void i40e_bm::timed_event(nicbm::TimedEvent &ev)
}
}
void i40e_bm::signal_interrupt(uint16_t vec, uint8_t itr)
{
void i40e_bm::signal_interrupt(uint16_t vec, uint8_t itr) {
int_ev &iev = intevs[vec];
uint64_t mindelay;
......@@ -715,8 +693,8 @@ void i40e_bm::signal_interrupt(uint16_t vec, uint8_t itr)
if (iev.armed && iev.time <= newtime) {
// already armed and this is not scheduled sooner
#ifdef DEBUG_DEV
log << "signal_interrupt: vec " << vec << " already scheduled" <<
logger::endl;
log << "signal_interrupt: vec " << vec << " already scheduled"
<< logger::endl;
#endif
return;
} else if (iev.armed) {
......@@ -728,15 +706,14 @@ void i40e_bm::signal_interrupt(uint16_t vec, uint8_t itr)
iev.time = newtime;
#ifdef DEBUG_DEV
log << "signal_interrupt: scheduled vec " << vec << " for time=" <<
newtime << " (itr " << itr << ")" << logger::endl;
log << "signal_interrupt: scheduled vec " << vec << " for time=" << newtime
<< " (itr " << itr << ")" << logger::endl;
#endif
runner->event_schedule(iev);
}
void i40e_bm::reset(bool indicate_done)
{
void i40e_bm::reset(bool indicate_done) {
#ifdef DEBUG_DEV
std::cout << "reset triggered" << logger::endl;
#endif
......@@ -778,13 +755,10 @@ void i40e_bm::reset(bool indicate_done)
regs.glrpb_plw = 0x0846;
}
shadow_ram::shadow_ram(i40e_bm &dev_)
: dev(dev_), log("sram")
{
shadow_ram::shadow_ram(i40e_bm &dev_) : dev(dev_), log("sram") {
}
void shadow_ram::reg_updated()
{
void shadow_ram::reg_updated() {
uint32_t val = dev.regs.glnvm_srctl;
uint32_t addr;
bool is_write;
......@@ -792,31 +766,27 @@ void shadow_ram::reg_updated()
if (!(val & I40E_GLNVM_SRCTL_START_MASK))
return;
addr = (val & I40E_GLNVM_SRCTL_ADDR_MASK)
>> I40E_GLNVM_SRCTL_ADDR_SHIFT;
addr = (val & I40E_GLNVM_SRCTL_ADDR_MASK) >> I40E_GLNVM_SRCTL_ADDR_SHIFT;
is_write = (val & I40E_GLNVM_SRCTL_WRITE_MASK);
#ifdef DEBUG_DEV
log << "shadow ram op addr=" << addr << " w=" << is_write
<< logger::endl;
log << "shadow ram op addr=" << addr << " w=" << is_write << logger::endl;
#endif
if (is_write) {
write(addr,
(dev.regs.glnvm_srdata & I40E_GLNVM_SRDATA_WRDATA_MASK)
>> I40E_GLNVM_SRDATA_WRDATA_SHIFT);
write(addr, (dev.regs.glnvm_srdata & I40E_GLNVM_SRDATA_WRDATA_MASK) >>
I40E_GLNVM_SRDATA_WRDATA_SHIFT);
} else {
dev.regs.glnvm_srdata &= ~I40E_GLNVM_SRDATA_RDDATA_MASK;
dev.regs.glnvm_srdata |= ((uint32_t) read(addr)) <<
I40E_GLNVM_SRDATA_RDDATA_SHIFT;
dev.regs.glnvm_srdata |= ((uint32_t)read(addr))
<< I40E_GLNVM_SRDATA_RDDATA_SHIFT;
}
dev.regs.glnvm_srctl &= ~I40E_GLNVM_SRCTL_START_MASK;
dev.regs.glnvm_srctl |= I40E_GLNVM_SRCTL_DONE_MASK;
}
uint16_t shadow_ram::read(uint16_t addr)
{
uint16_t shadow_ram::read(uint16_t addr) {
switch (addr) {
/* for any of these hopefully return 0 should be fine */
/* they are read by drivers but not used */
......@@ -834,8 +804,7 @@ uint16_t shadow_ram::read(uint16_t addr)
default:
#ifdef DEBUG_DEV
log << "TODO shadow memory read addr=" << addr
<< logger::endl;
log << "TODO shadow memory read addr=" << addr << logger::endl;
#endif
break;
}
......@@ -843,16 +812,14 @@ uint16_t shadow_ram::read(uint16_t addr)
return 0;
}
void shadow_ram::write(uint16_t addr, uint16_t val)
{
void shadow_ram::write(uint16_t addr, uint16_t val) {
#ifdef DEBUG_DEV
log << "TODO shadow memory write addr=" << addr <<
" val=" << val << logger::endl;
log << "TODO shadow memory write addr=" << addr << " val=" << val
<< logger::endl;
#endif
}
int_ev::int_ev()
{
int_ev::int_ev() {
armed = false;
time = 0;
}
......@@ -861,8 +828,7 @@ int_ev::int_ev()
using namespace i40e;
int main(int argc, char *argv[])
{
int main(int argc, char *argv[]) {
i40e_bm dev;
runner = new nicbm::Runner(dev);
return runner->runMain(argc, argv);
......
sims/nic/i40e_bm/i40e_bm.h
View file @ d4666c97
......@@ -24,10 +24,11 @@
#pragma once
#include <stdint.h>
#include <deque>
#include <sstream>
#include <string>
#include <stdint.h>
extern "C" {
#include <simbricks/proto/pcie.h>
}
......@@ -61,7 +62,6 @@ class int_ev : public nicbm::TimedEvent {
int_ev();
};
class logger : public std::ostream {
public:
static const char endl = '\n';
......@@ -111,8 +111,10 @@ class queue_base {
class desc_ctx {
friend class queue_base;
public:
enum state {
enum state
{
DESC_EMPTY,
DESC_FETCHING,
DESC_PREPARING,
......@@ -154,6 +156,7 @@ class queue_base {
class dma_fetch : public dma_base {
protected:
queue_base &queue;
public:
uint32_t pos;
dma_fetch(queue_base &queue_, size_t len);
......@@ -164,6 +167,7 @@ class queue_base {
class dma_wb : public dma_base {
protected:
queue_base &queue;
public:
uint32_t pos;
dma_wb(queue_base &queue_, size_t len);
......@@ -186,6 +190,7 @@ class queue_base {
class dma_data_wb : public dma_base {
protected:
desc_ctx &ctx;
public:
size_t total_len;
size_t part_offset;
......@@ -260,8 +265,8 @@ class queue_admin_tx : public queue_base {
// complete direct response
void desc_complete(uint16_t retval, uint16_t extra_flags = 0);
// complete indirect response
void desc_complete_indir(uint16_t retval, const void *data,
size_t len, uint16_t extra_flags = 0,
void desc_complete_indir(uint16_t retval, const void *data, size_t len,
uint16_t extra_flags = 0,
bool ignore_datalen = false);
public:
......@@ -278,8 +283,8 @@ class queue_admin_tx : public queue_base {
virtual desc_ctx &desc_ctx_create();
public:
queue_admin_tx(i40e_bm &dev_, uint64_t &reg_base_,
uint32_t &reg_len_, uint32_t &reg_head_, uint32_t &reg_tail_);
queue_admin_tx(i40e_bm &dev_, uint64_t &reg_base_, uint32_t &reg_len_,
uint32_t &reg_head_, uint32_t &reg_tail_);
void reg_updated();
};
......@@ -322,7 +327,6 @@ class lan_queue_base : public queue_base {
virtual void done();
};
lan &lanmgr;
void ctx_fetched();
......@@ -342,10 +346,9 @@ class lan_queue_base : public queue_base {
uint32_t reg_dummy_head;
lan_queue_base(lan &lanmgr_, const std::string &qtype,
uint32_t &reg_tail, size_t idx_, uint32_t &reg_ena_,
uint32_t &fpm_basereg, uint32_t &reg_intqctl,
uint16_t ctx_size);
lan_queue_base(lan &lanmgr_, const std::string &qtype, uint32_t &reg_tail,
size_t idx_, uint32_t &reg_ena_, uint32_t &fpm_basereg,
uint32_t &reg_intqctl, uint16_t ctx_size);
virtual void reset();
void enable();
void disable();
......@@ -369,10 +372,10 @@ class lan_queue_tx : public lan_queue_base {
virtual void processed();
};
class dma_hwb : public dma_base {
protected:
lan_queue_tx &queue;
public:
uint32_t pos;
uint32_t cnt;
......@@ -400,9 +403,8 @@ class lan_queue_tx : public lan_queue_base {
void trigger_tx();
public:
lan_queue_tx(lan &lanmgr_, uint32_t &reg_tail, size_t idx,
uint32_t &reg_ena, uint32_t &fpm_basereg,
uint32_t &reg_intqctl);
lan_queue_tx(lan &lanmgr_, uint32_t &reg_tail, size_t idx, uint32_t &reg_ena,
uint32_t &fpm_basereg, uint32_t &reg_intqctl);
virtual void reset();
};
......@@ -430,9 +432,8 @@ class lan_queue_rx : public lan_queue_base {
virtual desc_ctx &desc_ctx_create();
public:
lan_queue_rx(lan &lanmgr_, uint32_t &reg_tail, size_t idx,
uint32_t &reg_ena, uint32_t &fpm_basereg,
uint32_t &reg_intqctl);
lan_queue_rx(lan &lanmgr_, uint32_t &reg_tail, size_t idx, uint32_t &reg_ena,
uint32_t &fpm_basereg, uint32_t &reg_intqctl);
virtual void reset();
void packet_received(const void *data, size_t len, uint32_t hash);
};
......@@ -451,8 +452,7 @@ class rss_key_cache {
public:
rss_key_cache(const uint32_t (&key_)[key_len / 4]);
void set_dirty();
uint32_t hash_ipv4(uint32_t sip, uint32_t dip, uint16_t sp,
uint16_t dp);
uint32_t hash_ipv4(uint32_t sip, uint32_t dip, uint16_t sp, uint16_t dp);
};
// rx tx management
......@@ -471,6 +471,7 @@ class lan {
bool rss_steering(const void *data, size_t len, uint16_t &queue,
uint32_t &hash);
public:
lan(i40e_bm &dev, size_t num_qs);
void reset();
......@@ -493,7 +494,7 @@ class shadow_ram {
};
class i40e_bm : public nicbm::Runner::Device {
protected:
protected:
friend class queue_admin_tx;
friend class host_mem_cache;
friend class lan;
......@@ -578,7 +579,7 @@ protected:
uint32_t glrpb_plw;
};
public:
public:
i40e_bm();
~i40e_bm();
......@@ -594,7 +595,7 @@ public:
void signal_interrupt(uint16_t vector, uint8_t itr);
protected:
protected:
logger log;
i40e_regs regs;
queue_admin_tx pf_atq;
......@@ -622,8 +623,7 @@ void xsum_tcp(void *tcphdr, size_t l4len);
// calculates the full ipv4 & tcp checksum without assuming any pseudo header
// xsums
void xsum_tcpip_tso(void *iphdr, uint8_t iplen, uint8_t l4len,
uint16_t paylen);
void xsum_tcpip_tso(void *iphdr, uint8_t iplen, uint8_t l4len, uint16_t paylen);
void tso_postupdate_header(void *iphdr, uint8_t iplen, uint8_t l4len,
uint16_t paylen);
......
sims/nic/i40e_bm/i40e_hmc.cc
View file @ d4666c97
......@@ -24,24 +24,22 @@
#include <stdlib.h>
#include <string.h>
#include <cassert>
#include <iostream>
#include "sims/nic/i40e_bm/i40e_bm.h"
#include "sims/nic/i40e_bm/i40e_base_wrapper.h"
#include "sims/nic/i40e_bm/i40e_bm.h"
using namespace i40e;
extern nicbm::Runner *runner;
host_mem_cache::host_mem_cache(i40e_bm &dev_)
: dev(dev_)
{
host_mem_cache::host_mem_cache(i40e_bm &dev_) : dev(dev_) {
reset();
}
void host_mem_cache::reset()
{
void host_mem_cache::reset() {
for (size_t i = 0; i < MAX_SEGMENTS; i++) {
segs[i].addr = 0;
segs[i].pgcount = 0;
......@@ -50,13 +48,12 @@ void host_mem_cache::reset()
}
}
void host_mem_cache::reg_updated(uint64_t addr)
{
void host_mem_cache::reg_updated(uint64_t addr) {
if (addr == I40E_PFHMC_SDCMD) {
// read/write command for descriptor
uint32_t cmd = dev.regs.pfhmc_sdcmd;
uint16_t idx = (cmd & I40E_PFHMC_SDCMD_PMSDIDX_MASK) >>
I40E_PFHMC_SDCMD_PMSDIDX_SHIFT;
uint16_t idx =
(cmd & I40E_PFHMC_SDCMD_PMSDIDX_MASK) >> I40E_PFHMC_SDCMD_PMSDIDX_SHIFT;
uint32_t lo = dev.regs.pfhmc_sddatalow;
uint32_t hi = dev.regs.pfhmc_sddatahigh;
......@@ -67,17 +64,19 @@ void host_mem_cache::reg_updated(uint64_t addr)
#endif
segs[idx].addr = ((lo & I40E_PFHMC_SDDATALOW_PMSDDATALOW_MASK) >>
I40E_PFHMC_SDDATALOW_PMSDDATALOW_SHIFT) << 12;
segs[idx].addr |= ((uint64_t) hi) << 32;
I40E_PFHMC_SDDATALOW_PMSDDATALOW_SHIFT)
<< 12;
segs[idx].addr |= ((uint64_t)hi) << 32;
segs[idx].pgcount = (lo & I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_MASK) >>
I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT;
segs[idx].valid = !!(lo & I40E_PFHMC_SDDATALOW_PMSDVALID_MASK);
segs[idx].direct = !!(lo & I40E_PFHMC_SDDATALOW_PMSDTYPE_MASK);
#ifdef DEBUG_HMC
std::cerr << " addr=" << segs[idx].addr << " pgcount=" <<
segs[idx].pgcount << " valid=" << segs[idx].valid <<
" direct=" << segs[idx].direct << std::endl;
std::cerr << " addr=" << segs[idx].addr
<< " pgcount=" << segs[idx].pgcount
<< " valid=" << segs[idx].valid
<< " direct=" << segs[idx].direct << std::endl;
#endif
} else {
// read
......@@ -85,11 +84,11 @@ void host_mem_cache::reg_updated(uint64_t addr)
std::cerr << "hmc: reading descriptor " << idx << std::endl;
#endif
dev.regs.pfhmc_sddatalow = ((segs[idx].addr >> 12) <<
I40E_PFHMC_SDDATALOW_PMSDDATALOW_SHIFT) &
dev.regs.pfhmc_sddatalow =
((segs[idx].addr >> 12) << I40E_PFHMC_SDDATALOW_PMSDDATALOW_SHIFT) &
I40E_PFHMC_SDDATALOW_PMSDDATALOW_MASK;
dev.regs.pfhmc_sddatalow |= (segs[idx].pgcount <<
I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT) &
dev.regs.pfhmc_sddatalow |=
(segs[idx].pgcount << I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT) &
I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_MASK;
if (segs[idx].valid)
dev.regs.pfhmc_sddatalow |= I40E_PFHMC_SDDATALOW_PMSDVALID_MASK;
......@@ -100,8 +99,7 @@ void host_mem_cache::reg_updated(uint64_t addr)
}
}
void host_mem_cache::issue_mem_op(mem_op &op)
{
void host_mem_cache::issue_mem_op(mem_op &op) {
uint64_t addr = op.dma_addr;
uint16_t seg_idx = addr >> 21;
uint16_t seg_idx_last = (addr + op.len - 1) >> 21;
......@@ -109,28 +107,26 @@ void host_mem_cache::issue_mem_op(mem_op &op)
struct segment *seg = &segs[seg_idx];
if (seg_idx >= MAX_SEGMENTS) {
std::cerr << "hmc issue_mem_op: seg index too high " << seg_idx <<
std::endl;
std::cerr << "hmc issue_mem_op: seg index too high " << seg_idx
<< std::endl;
abort();
}
if (!seg->valid) {
// TODO(antoinek): errorinfo and data registers
std::cerr << "hmc issue_mem_op: segment invalid addr=" << addr <<
std::endl;
std::cerr << "hmc issue_mem_op: segment invalid addr=" << addr << std::endl;
op.failed = true;
return;
}
if (seg_idx != seg_idx_last) {
std::cerr << "hmc issue_mem_op: operation crosses segs addr=" <<
addr << " len=" << op.len << std::endl;
std::cerr << "hmc issue_mem_op: operation crosses segs addr=" << addr
<< " len=" << op.len << std::endl;
abort();
}
if (!seg->direct) {
std::cerr << "hmc issue_mem_op: TODO paged ops addr=" << addr <<
std::endl;
std::cerr << "hmc issue_mem_op: TODO paged ops addr=" << addr << std::endl;
abort();
}
......@@ -138,8 +134,8 @@ void host_mem_cache::issue_mem_op(mem_op &op)
op.dma_addr = seg->addr + dir_off;
#ifdef DEBUG_HMC
std::cerr << "hmc issue_mem_op: hmc_addr=" << addr << " dma_addr=" <<
op.dma_addr << " len=" << op.len << std::endl;
std::cerr << "hmc issue_mem_op: hmc_addr=" << addr
<< " dma_addr=" << op.dma_addr << " len=" << op.len << std::endl;
#endif
runner->issue_dma(op);
}
sims/nic/i40e_bm/i40e_lan.cc
View file @ d4666c97
This diff is collapsed.
sims/nic/i40e_bm/i40e_queues.cc
View file @ d4666c97
......@@ -24,12 +24,13 @@
#include <stdlib.h>
#include <string.h>
#include <algorithm>
#include <cassert>
#include <iostream>
#include <algorithm>
#include "sims/nic/i40e_bm/i40e_bm.h"
#include "sims/nic/i40e_bm/i40e_base_wrapper.h"
#include "sims/nic/i40e_bm/i40e_bm.h"
using namespace i40e;
......@@ -37,24 +38,29 @@ extern nicbm::Runner *runner;
queue_base::queue_base(const std::string &qname_, uint32_t &reg_head_,
uint32_t &reg_tail_)
: qname(qname_), log(qname_), active_first_pos(0), active_first_idx(0),
active_cnt(0), base(0), len(0), reg_head(reg_head_), reg_tail(reg_tail_),
enabled(false), desc_len(0)
{
: qname(qname_),
log(qname_),
active_first_pos(0),
active_first_idx(0),
active_cnt(0),
base(0),
len(0),
reg_head(reg_head_),
reg_tail(reg_tail_),
enabled(false),
desc_len(0) {
for (size_t i = 0; i < MAX_ACTIVE_DESCS; i++) {
desc_ctxs[i] = nullptr;
}
}
void queue_base::ctxs_init()
{
void queue_base::ctxs_init() {
for (size_t i = 0; i < MAX_ACTIVE_DESCS; i++) {
desc_ctxs[i] = &desc_ctx_create();
}
}
void queue_base::trigger_fetch()
{
void queue_base::trigger_fetch() {
if (!enabled)
return;
......@@ -71,8 +77,8 @@ void queue_base::trigger_fetch()
fetch_cnt = len - next_idx;
#ifdef DEBUG_QUEUES
log << "fetching avail=" << desc_avail << " cnt=" << fetch_cnt << " idx=" <<
next_idx << logger::endl;
log << "fetching avail=" << desc_avail << " cnt=" << fetch_cnt
<< " idx=" << next_idx << logger::endl;
#endif
// abort if nothign to fetch
......@@ -101,23 +107,21 @@ void queue_base::trigger_fetch()
runner->issue_dma(*dma);
}
void queue_base::trigger_process()
{
void queue_base::trigger_process() {
if (!enabled)
return;
// first skip over descriptors that are already done processing
uint32_t i;
for (i = 0; i < active_cnt; i++)
if (desc_ctxs[(active_first_pos + i) % MAX_ACTIVE_DESCS]->state
<= desc_ctx::DESC_PREPARED)
if (desc_ctxs[(active_first_pos + i) % MAX_ACTIVE_DESCS]->state <=
desc_ctx::DESC_PREPARED)
break;
// then run all prepared contexts
uint32_t j;
for (j = 0; i + j < active_cnt; j++) {
desc_ctx &ctx = *desc_ctxs[(active_first_pos + i + j)
% MAX_ACTIVE_DESCS];
desc_ctx &ctx = *desc_ctxs[(active_first_pos + i + j) % MAX_ACTIVE_DESCS];
if (ctx.state != desc_ctx::DESC_PREPARED)
break;
......@@ -129,16 +133,15 @@ void queue_base::trigger_process()
}
}
void queue_base::trigger_writeback()
{
void queue_base::trigger_writeback() {
if (!enabled)
return;
// from first pos count number of processed descriptors
uint32_t avail;
for (avail = 0; avail < active_cnt; avail++)
if (desc_ctxs[(active_first_pos + avail) % MAX_ACTIVE_DESCS]->state
!= desc_ctx::DESC_PROCESSED)
if (desc_ctxs[(active_first_pos + avail) % MAX_ACTIVE_DESCS]->state !=
desc_ctx::DESC_PROCESSED)
break;
uint32_t cnt = std::min(avail, max_writeback_capacity());
......@@ -146,8 +149,8 @@ void queue_base::trigger_writeback()
cnt = len - active_first_idx;
#ifdef DEBUG_QUEUES
log << "writing back avail=" << avail << " cnt=" << cnt <<
" idx=" << active_first_idx << logger::endl;
log << "writing back avail=" << avail << " cnt=" << cnt
<< " idx=" << active_first_idx << logger::endl;
#endif
if (cnt == 0)
......@@ -162,15 +165,13 @@ void queue_base::trigger_writeback()
do_writeback(active_first_idx, active_first_pos, cnt);
}
void queue_base::trigger()
{
void queue_base::trigger() {
trigger_fetch();
trigger_process();
trigger_writeback();
}
void queue_base::reset()
{
void queue_base::reset() {
#ifdef DEBUG_QUEUES
log << "reset" << logger::endl;
#endif
......@@ -185,11 +186,10 @@ void queue_base::reset()
}
}
void queue_base::reg_updated()
{
void queue_base::reg_updated() {
#ifdef DEBUG_QUEUES
log << "reg_updated: tail=" << reg_tail << " enabled=" << (int) enabled <<
logger::endl;
log << "reg_updated: tail=" << reg_tail << " enabled=" << (int)enabled
<< logger::endl;
#endif
if (!enabled)
return;
......@@ -197,39 +197,33 @@ void queue_base::reg_updated()
trigger();
}
bool queue_base::is_enabled()
{
bool queue_base::is_enabled() {
return enabled;
}
uint32_t queue_base::max_fetch_capacity()
{
uint32_t queue_base::max_fetch_capacity() {
return UINT32_MAX;
}
uint32_t queue_base::max_active_capacity()
{
uint32_t queue_base::max_active_capacity() {
return UINT32_MAX;
}
uint32_t queue_base::max_writeback_capacity()
{
uint32_t queue_base::max_writeback_capacity() {
return UINT32_MAX;
}
void queue_base::interrupt()
{
void queue_base::interrupt() {
}
void queue_base::do_writeback(uint32_t first_idx, uint32_t first_pos,
uint32_t cnt)
{
uint32_t cnt) {
dma_wb *dma = new dma_wb(*this, desc_len * cnt);
dma->write = true;
dma->dma_addr = base + first_idx * desc_len;
dma->pos = first_pos;
uint8_t *buf = reinterpret_cast<uint8_t *> (dma->data);
uint8_t *buf = reinterpret_cast<uint8_t *>(dma->data);
for (uint32_t i = 0; i < cnt; i++) {
desc_ctx &ctx = *desc_ctxs[(first_pos + i) % MAX_ACTIVE_DESCS];
assert(ctx.state == desc_ctx::DESC_WRITING_BACK);
......@@ -239,8 +233,7 @@ void queue_base::do_writeback(uint32_t first_idx, uint32_t first_pos,
runner->issue_dma(*dma);
}
void queue_base::writeback_done(uint32_t first_pos, uint32_t cnt)
{
void queue_base::writeback_done(uint32_t first_pos, uint32_t cnt) {
if (!enabled)
return;
......@@ -252,17 +245,17 @@ void queue_base::writeback_done(uint32_t first_pos, uint32_t cnt)
}
#ifdef DEBUG_QUEUES
log << "written back afi=" << active_first_idx << " afp=" <<
active_first_pos << " acnt=" << active_cnt << " pos=" <<
first_pos << " cnt=" << cnt << logger::endl;
log << "written back afi=" << active_first_idx << " afp=" << active_first_pos
<< " acnt=" << active_cnt << " pos=" << first_pos << " cnt=" << cnt
<< logger::endl;
#endif
// then start at the beginning and check how many are written back and then
// free those
uint32_t bump_cnt = 0;
for (bump_cnt = 0; bump_cnt < active_cnt; bump_cnt++) {
desc_ctx &ctx = *desc_ctxs[(active_first_pos + bump_cnt) %
MAX_ACTIVE_DESCS];
desc_ctx &ctx =
*desc_ctxs[(active_first_pos + bump_cnt) % MAX_ACTIVE_DESCS];
if (ctx.state != desc_ctx::DESC_WRITTEN_BACK)
break;
......@@ -281,26 +274,26 @@ void queue_base::writeback_done(uint32_t first_pos, uint32_t cnt)
}
queue_base::desc_ctx::desc_ctx(queue_base &queue_)
: queue(queue_), state(DESC_EMPTY), index(0), data(nullptr), data_len(0),
data_capacity(0)
{
: queue(queue_),
state(DESC_EMPTY),
index(0),
data(nullptr),
data_len(0),
data_capacity(0) {
desc = new uint8_t[queue_.desc_len];
}
queue_base::desc_ctx::~desc_ctx()
{
delete[] ((uint8_t *) desc);
queue_base::desc_ctx::~desc_ctx() {
delete[]((uint8_t *)desc);
if (data_capacity > 0)
delete[] ((uint8_t *) data);
delete[]((uint8_t *)data);
}
void queue_base::desc_ctx::prepare()
{
void queue_base::desc_ctx::prepare() {
prepared();
}
void queue_base::desc_ctx::prepared()
{
void queue_base::desc_ctx::prepared() {
#ifdef DEBUG_QUEUES
queue.log << "prepared desc " << index << logger::endl;
#endif
......@@ -308,8 +301,7 @@ void queue_base::desc_ctx::prepared()
state = DESC_PREPARED;
}
void queue_base::desc_ctx::processed()
{
void queue_base::desc_ctx::processed() {
#ifdef DEBUG_QUEUES
queue.log << "processed desc " << index << logger::endl;
#endif
......@@ -317,47 +309,44 @@ void queue_base::desc_ctx::processed()
state = DESC_PROCESSED;
}
#define MAX_DMA_SIZE ((size_t) 9024)
#define MAX_DMA_SIZE ((size_t)9024)
void queue_base::desc_ctx::data_fetch(uint64_t addr, size_t data_len)
{
void queue_base::desc_ctx::data_fetch(uint64_t addr, size_t data_len) {
if (data_capacity < data_len) {
#ifdef DEBUG_QUEUES
queue.log << "data_fetch allocating" << logger::endl;
#endif
if (data_capacity != 0)
delete[] ((uint8_t *) data);
delete[]((uint8_t *)data);
data = new uint8_t[data_len];
data_capacity = data_len;
}
dma_data_fetch *dma = new dma_data_fetch(*this, std::min(data_len,
MAX_DMA_SIZE), data);
dma_data_fetch *dma =
new dma_data_fetch(*this, std::min(data_len, MAX_DMA_SIZE), data);
dma->part_offset = 0;
dma->total_len = data_len;
dma->write = false;
dma->dma_addr = addr;
#ifdef DEBUG_QUEUES
queue.log << "fetching data idx=" << index << " addr=" << addr << " len=" <<
data_len << logger::endl;
queue.log << "fetching data idx=" << index << " addr=" << addr
<< " len=" << data_len << logger::endl;
queue.log << " dma = " << dma << " data=" << data << logger::endl;
#endif
runner->issue_dma(*dma);
}
void queue_base::desc_ctx::data_fetched(uint64_t addr, size_t len)
{
void queue_base::desc_ctx::data_fetched(uint64_t addr, size_t len) {
prepared();
}
void queue_base::desc_ctx::data_write(uint64_t addr, size_t data_len,
const void *buf)
{
const void *buf) {
#ifdef DEBUG_QUEUES
queue.log << "data_write(addr=" << addr << " datalen=" << data_len << ")" <<
logger::endl;
queue.log << "data_write(addr=" << addr << " datalen=" << data_len << ")"
<< logger::endl;
#endif
dma_data_wb *data_dma = new dma_data_wb(*this, data_len);
data_dma->write = true;
......@@ -367,30 +356,26 @@ void queue_base::desc_ctx::data_write(uint64_t addr, size_t data_len,
runner->issue_dma(*data_dma);
}
void queue_base::desc_ctx::data_written(uint64_t addr, size_t len)
{
void queue_base::desc_ctx::data_written(uint64_t addr, size_t len) {
#ifdef DEBUG_QUEUES
queue.log << "data_written(addr=" << addr << " datalen=" << len << ")" <<
logger::endl;
queue.log << "data_written(addr=" << addr << " datalen=" << len << ")"
<< logger::endl;
#endif
processed();
}
queue_base::dma_fetch::dma_fetch(queue_base &queue_, size_t len_)
: queue(queue_)
{
: queue(queue_) {
data = new char[len_];
len = len_;
}
queue_base::dma_fetch::~dma_fetch()
{
delete[] ((char *) data);
queue_base::dma_fetch::~dma_fetch() {
delete[]((char *)data);
}
void queue_base::dma_fetch::done()
{
uint8_t *buf = reinterpret_cast <uint8_t *> (data);
void queue_base::dma_fetch::done() {
uint8_t *buf = reinterpret_cast<uint8_t *>(data);
for (uint32_t i = 0; i < len / queue.desc_len; i++) {
desc_ctx &ctx = *queue.desc_ctxs[(pos + i) % queue.MAX_ACTIVE_DESCS];
memcpy(ctx.desc, buf + queue.desc_len * i, queue.desc_len);
......@@ -407,26 +392,22 @@ void queue_base::dma_fetch::done()
queue_base::dma_data_fetch::dma_data_fetch(desc_ctx &ctx_, size_t len_,
void *buffer)
: ctx(ctx_)
{
: ctx(ctx_) {
data = buffer;
len = len_;
}
queue_base::dma_data_fetch::~dma_data_fetch()
{
queue_base::dma_data_fetch::~dma_data_fetch() {
}
void queue_base::dma_data_fetch::done()
{
void queue_base::dma_data_fetch::done() {
part_offset += len;
dma_addr += len;
data = (uint8_t *) data + len;
data = (uint8_t *)data + len;
if (part_offset < total_len) {
#ifdef DEBUG_QUEUES
ctx.queue.log << " dma_fetch: next part of multi part dma" <<
logger::endl;
ctx.queue.log << " dma_fetch: next part of multi part dma" << logger::endl;
#endif
len = std::min(total_len - part_offset, MAX_DMA_SIZE);
runner->issue_dma(*this);
......@@ -437,40 +418,31 @@ void queue_base::dma_data_fetch::done()
delete this;
}
queue_base::dma_wb::dma_wb(queue_base &queue_, size_t len_)
: queue(queue_)
{
queue_base::dma_wb::dma_wb(queue_base &queue_, size_t len_) : queue(queue_) {
data = new char[len_];
len = len_;
}
queue_base::dma_wb::~dma_wb()
{
delete[] ((char *) data);
queue_base::dma_wb::~dma_wb() {
delete[]((char *)data);
}
void queue_base::dma_wb::done()
{
void queue_base::dma_wb::done() {
queue.writeback_done(pos, len / queue.desc_len);
queue.trigger();
delete this;
}
queue_base::dma_data_wb::dma_data_wb(desc_ctx &ctx_, size_t len_)
: ctx(ctx_)
{
queue_base::dma_data_wb::dma_data_wb(desc_ctx &ctx_, size_t len_) : ctx(ctx_) {
data = new char[len_];
len = len_;
}
queue_base::dma_data_wb::~dma_data_wb()
{
delete[] ((char *) data);
queue_base::dma_data_wb::~dma_data_wb() {
delete[]((char *)data);
}
void queue_base::dma_data_wb::done()
{
void queue_base::dma_data_wb::done() {
ctx.data_written(dma_addr, len);
ctx.queue.trigger();
delete this;
......
sims/nic/i40e_bm/logger.cc
View file @ d4666c97
This diff is collapsed.
sims/nic/i40e_bm/rss.cc
View file @ d4666c97
This diff is collapsed.
sims/nic/i40e_bm/xsums.cc
View file @ d4666c97
This diff is collapsed.
trace/events.h
View file @ d4666c97
This diff is collapsed.
trace/gem5.cc
View file @ d4666c97
This diff is collapsed.
trace/log_parser.cc
View file @ d4666c97
This diff is collapsed.
trace/nicbm.cc
View file @ d4666c97
This diff is collapsed.
trace/parser.h
View file @ d4666c97
This diff is collapsed.
trace/process.cc
View file @ d4666c97
This diff is collapsed.
trace/process.h
View file @ d4666c97
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment