i40e: now rx and tx with xsum offload works

i40e_bm/Makefile

@@ -3,7 +3,7 @@ CPPFLAGS += -I../libnicbm/include/
 CFLAGS += -Wall -Wextra -Wno-unused-parameter -O3 -g
 LDFLGAS = -g
 
-OBJS := i40e_bm.o i40e_queues.o i40e_adminq.o i40e_hmc.o i40e_lan.o
+OBJS := i40e_bm.o i40e_queues.o i40e_adminq.o i40e_hmc.o i40e_lan.o xsums.o
 
 all: i40e_bm
 

i40e_bm/base/i40e_rxtxq.h

@@ -379,11 +379,13 @@ enum i40e_tx_desc_cmd_bits {
 	I40E_TX_DESC_CMD_IIPT_IPV6		= 0x0020, /* 2 BITS */
 	I40E_TX_DESC_CMD_IIPT_IPV4		= 0x0040, /* 2 BITS */
 	I40E_TX_DESC_CMD_IIPT_IPV4_CSUM		= 0x0060, /* 2 BITS */
+	I40E_TX_DESC_CMD_IIPT_MASK		= 0x0060, /* 2 BITS */
 	I40E_TX_DESC_CMD_FCOET			= 0x0080,
 	I40E_TX_DESC_CMD_L4T_EOFT_UNK		= 0x0000, /* 2 BITS */
 	I40E_TX_DESC_CMD_L4T_EOFT_TCP		= 0x0100, /* 2 BITS */
 	I40E_TX_DESC_CMD_L4T_EOFT_SCTP		= 0x0200, /* 2 BITS */
 	I40E_TX_DESC_CMD_L4T_EOFT_UDP		= 0x0300, /* 2 BITS */
+	I40E_TX_DESC_CMD_L4T_EOFT_MASK		= 0x0300, /* 2 BITS */
 	I40E_TX_DESC_CMD_L4T_EOFT_EOF_N		= 0x0000, /* 2 BITS */
 	I40E_TX_DESC_CMD_L4T_EOFT_EOF_T		= 0x0100, /* 2 BITS */
 	I40E_TX_DESC_CMD_L4T_EOFT_EOF_NI	= 0x0200, /* 2 BITS */

i40e_bm/i40e_bm.cc

@@ -46,6 +46,7 @@ void i40e_bm::dma_complete(nicbm::DMAOp &op)
 void i40e_bm::eth_rx(uint8_t port, const void *data, size_t len)
 {
     std::cerr << "i40e: received packet len=" << len << std::endl;
+    lanmgr.packet_received(data, len);
 }
 
 void i40e_bm::reg_read(uint8_t bar, uint64_t addr, void *dest, size_t len)

i40e_bm/i40e_bm.h

@@ -81,6 +81,9 @@ class queue_base {
         void desc_writeback_indirect(const void *desc, uint32_t idx,
                 uint64_t data_addr, const void *data, size_t data_len);
 
+        // returns how many descriptors the queue can fetch max during the next
+        // fetch: default UINT32_MAX, but can be overriden by child classes
+        virtual uint32_t max_fetch_capacity();
         // called when a descriptor is fetched
         virtual void desc_fetched(void *desc, uint32_t idx) = 0;
         // called when data is fetched
@@ -223,13 +226,25 @@ class lan_queue_tx : public lan_queue_base {
 
 class lan_queue_rx : public lan_queue_base {
     protected:
+        struct desc_cache {
+            uint64_t buf;
+            uint64_t hbuf;
+        };
+
         uint16_t dbuff_size;
         uint16_t hbuff_size;
         uint16_t rxmax;
         bool crc_strip;
 
+        static const uint16_t DCACHE_SIZE = 128;
+        struct desc_cache dcache[DCACHE_SIZE];
+        uint32_t dcache_first_idx;
+        uint16_t dcache_first_pos;
+        uint16_t dcache_first_cnt;
+
         virtual void initialize();
 
+        virtual uint32_t max_fetch_capacity();
         virtual void desc_fetched(void *desc, uint32_t idx);
         virtual void data_fetched(void *desc, uint32_t idx, void *data);
 
@@ -237,6 +252,7 @@ class lan_queue_rx : public lan_queue_base {
         lan_queue_rx(lan &lanmgr_, uint32_t &reg_tail, size_t idx,
                 uint32_t &reg_ena, uint32_t &fpm_basereg,
                 uint32_t &reg_intqctl);
+        void packet_received(const void *data, size_t len);
 };
 
 // rx tx management
@@ -255,6 +271,7 @@ class lan {
         lan(i40e_bm &dev, size_t num_qs);
         void qena_updated(uint16_t idx, bool rx);
         void tail_updated(uint16_t idx, bool rx);
+        void packet_received(const void *data, size_t len);
 };
 
 class shadow_ram {
@@ -365,4 +382,7 @@ protected:
     void reset();
 };
 
+// places the tcp checksum in the packet (assuming ipv4)
+void xsum_tcp(void *tcphdr, size_t l4len);
+
 } // namespace corundum

i40e_bm/i40e_lan.cc

@@ -52,6 +52,14 @@ void lan::tail_updated(uint16_t idx, bool rx)
         q.reg_updated();
 }
 
+void lan::packet_received(const void *data, size_t len)
+{
+    std::cerr << "lan: packet received len=" << len << std::endl;
+
+    // TODO: steering
+    rxqs[0]->packet_received(data, len);
+}
+
 lan_queue_base::lan_queue_base(lan &lanmgr_, uint32_t &reg_tail_, size_t idx_,
         uint32_t &reg_ena_, uint32_t &fpm_basereg_, uint32_t &reg_intqctl_,
         uint16_t ctx_size_)
@@ -144,7 +152,8 @@ void lan_queue_base::qctx_fetch::done()
 lan_queue_rx::lan_queue_rx(lan &lanmgr_, uint32_t &reg_tail_, size_t idx_,
         uint32_t &reg_ena_, uint32_t &reg_fpmbase_, uint32_t &reg_intqctl_)
     : lan_queue_base(lanmgr_, reg_tail_, idx_, reg_ena_, reg_fpmbase_,
-            reg_intqctl_, 32)
+            reg_intqctl_, 32), dcache_first_idx(0), dcache_first_pos(0),
+        dcache_first_cnt(0)
 {
 }
 
@@ -190,11 +199,27 @@ void lan_queue_rx::initialize()
         " crcstrip=" << crc_strip << " rxmax=" << rxmax << std::endl;
 }
 
+uint32_t lan_queue_rx::max_fetch_capacity()
+{
+    return DCACHE_SIZE - dcache_first_cnt;
+}
+
 void lan_queue_rx::desc_fetched(void *desc_ptr, uint32_t didx)
 {
     std::cerr << "rxq: desc fetched" << std::endl;
+    union i40e_32byte_rx_desc *desc =
+        reinterpret_cast<union i40e_32byte_rx_desc *> (desc_ptr);
+
+    assert(dcache_first_cnt < DCACHE_SIZE);
+    std::cerr << "    idx=" << dcache_first_idx << " cnt=" << dcache_first_cnt <<
+        " didx=" << didx << std::endl;
+    assert((dcache_first_idx + dcache_first_cnt) % len == didx);
 
-    //union i40e_32byte_rx_desc *desc = desc_ptr;
+    uint16_t dci = (dcache_first_pos + dcache_first_cnt) % DCACHE_SIZE;
+    dcache[dci].buf = desc->read.pkt_addr;
+    dcache[dci].hbuf = desc->read.hdr_addr;
+
+    dcache_first_cnt++;
 }
 
 void lan_queue_rx::data_fetched(void *desc, uint32_t didx, void *data)
@@ -202,6 +227,30 @@ void lan_queue_rx::data_fetched(void *desc, uint32_t didx, void *data)
     std::cerr << "rxq: data fetched" << std::endl;
 }
 
+void lan_queue_rx::packet_received(const void *data, size_t pktlen)
+{
+    if (dcache_first_cnt == 0) {
+        std::cerr << "rqx: empty, dropping packet" << std::endl;
+        return;
+    }
+
+    std::cerr << "rxq: packet received" << std::endl;
+    union i40e_32byte_rx_desc rxd;
+    memset(&rxd, 0, sizeof(rxd));
+    rxd.wb.qword1.status_error_len |= (1 << I40E_RX_DESC_STATUS_DD_SHIFT);
+    rxd.wb.qword1.status_error_len |= (1 << I40E_RX_DESC_STATUS_EOF_SHIFT);
+    // TODO: only if checksums are correct
+    rxd.wb.qword1.status_error_len |= (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT);
+    rxd.wb.qword1.status_error_len |= (pktlen << I40E_RXD_QW1_LENGTH_PBUF_SHIFT);
+
+    desc_writeback_indirect(&rxd, dcache_first_idx,
+            dcache[dcache_first_pos].buf, data, pktlen);
+
+    dcache_first_pos = (dcache_first_pos + 1) % DCACHE_SIZE;
+    dcache_first_idx = (dcache_first_idx + 1) % len;
+    dcache_first_cnt--;
+}
+
 lan_queue_tx::lan_queue_tx(lan &lanmgr_, uint32_t &reg_tail_, size_t idx_,
         uint32_t &reg_ena_, uint32_t &reg_fpmbase_, uint32_t &reg_intqctl)
     : lan_queue_base(lanmgr_, reg_tail_, idx_, reg_ena_, reg_fpmbase_,
@@ -270,10 +319,28 @@ void lan_queue_tx::data_fetched(void *desc_buf, uint32_t didx, void *data)
     std::cerr << "txq: data fetched" << std::endl;
     struct i40e_tx_desc *desc = reinterpret_cast<struct i40e_tx_desc *>(desc_buf);
     uint64_t d1 = desc->cmd_type_offset_bsz;
-    uint16_t len = (d1 & I40E_TXD_QW1_TX_BUF_SZ_MASK) >>
+    uint16_t pkt_len = (d1 & I40E_TXD_QW1_TX_BUF_SZ_MASK) >>
         I40E_TXD_QW1_TX_BUF_SZ_SHIFT;
 
-    runner->eth_send(data, len);
+    uint16_t cmd = (d1 & I40E_TXD_QW1_CMD_MASK) >> I40E_TXD_QW1_CMD_SHIFT;
+    uint16_t iipt = cmd & (I40E_TX_DESC_CMD_IIPT_MASK);
+    uint16_t l4t = (cmd & I40E_TX_DESC_CMD_L4T_EOFT_MASK);
+
+    uint32_t off = (d1 & I40E_TXD_QW1_OFFSET_MASK) >> I40E_TXD_QW1_OFFSET_SHIFT;
+    uint16_t maclen = ((off & I40E_TXD_QW1_MACLEN_MASK) >>
+        I40E_TX_DESC_LENGTH_MACLEN_SHIFT) * 2;
+    uint16_t iplen = ((off & I40E_TXD_QW1_IPLEN_MASK) >>
+        I40E_TX_DESC_LENGTH_IPLEN_SHIFT) * 4;
+    /*uint16_t l4len = (off & I40E_TXD_QW1_L4LEN_MASK) >>
+        I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;*/
+
+    if (l4t == I40E_TX_DESC_CMD_L4T_EOFT_TCP) {
+        uint16_t tcp_off = maclen + iplen;
+        xsum_tcp((uint8_t *) data + tcp_off, pkt_len - tcp_off);
+    }
+    std::cerr << "    iipt=" << iipt << " l4t=" << l4t << " maclen=" << maclen << " iplen=" << iplen<< std::endl;
+
+    runner->eth_send(data, pkt_len);
 
     desc->buffer_addr = 0;
     desc->cmd_type_offset_bsz = I40E_TX_DESC_DTYPE_DESC_DONE << I40E_TXD_QW1_DTYPE_SHIFT;

i40e_bm/i40e_queues.cc

@@ -22,6 +22,8 @@ void queue_base::trigger_fetch()
     if (!enabled || fetch_head == reg_tail)
         return;
 
+    if (max_fetch_capacity() == 0)
+        return;
 
     dma_fetch *dma = new dma_fetch(*this, desc_len);
     dma->write = false;
@@ -95,6 +97,11 @@ void queue_base::desc_writeback_indirect(const void *desc, uint32_t idx,
     runner->issue_dma(*data_dma);
 }
 
+uint32_t queue_base::max_fetch_capacity()
+{
+    return UINT32_MAX;
+}
+
 void queue_base::desc_done(uint32_t idx)
 {
     reg_head = (idx + 1) % len;

i40e_bm/xsums.cc

+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 1982, 1986, 1990, 1993
+ *      The Regents of the University of California.
+ * Copyright(c) 2010-2014 Intel Corporation.
+ * Copyright(c) 2014 6WIND S.A.
+ * All rights reserved.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <cassert>
+#include <iostream>
+
+#include "i40e_bm.h"
+
+namespace i40e {
+
+/* from dpdk/lib/librte_net/rte_tcp.h */
+struct rte_tcp_hdr {
+    uint16_t src_port; /**< TCP source port. */
+    uint16_t dst_port; /**< TCP destination port. */
+    uint32_t sent_seq; /**< TX data sequence number. */
+    uint32_t recv_ack; /**< RX data acknowledgment sequence number. */
+    uint8_t  data_off;   /**< Data offset. */
+    uint8_t  tcp_flags;  /**< TCP flags */
+    uint16_t rx_win;   /**< RX flow control window. */
+    uint16_t cksum;    /**< TCP checksum. */
+    uint16_t tcp_urp;  /**< TCP urgent pointer, if any. */
+} __attribute__((packed));
+
+
+/* from dpdk/lib/librte_net/rte_ip.h */
+static inline uint32_t __rte_raw_cksum(const void *buf, size_t len, uint32_t sum)
+{
+    /* workaround gcc strict-aliasing warning */
+    uintptr_t ptr = (uintptr_t)buf;
+    typedef uint16_t __attribute__((__may_alias__)) u16_p;
+    const u16_p *u16_buf = (const u16_p *)ptr;
+
+    while (len >= (sizeof(*u16_buf) * 4)) {
+        sum += u16_buf[0];
+        sum += u16_buf[1];
+        sum += u16_buf[2];
+        sum += u16_buf[3];
+        len -= sizeof(*u16_buf) * 4;
+        u16_buf += 4;
+    }
+    while (len >= sizeof(*u16_buf)) {
+        sum += *u16_buf;
+        len -= sizeof(*u16_buf);
+        u16_buf += 1;
+    }
+
+    /* if length is in odd bytes */
+    if (len == 1) {
+        uint16_t left = 0;
+        *(uint8_t *)&left = *(const uint8_t *)u16_buf;
+        sum += left;
+    }
+
+    return sum;
+}
+
+static inline uint16_t __rte_raw_cksum_reduce(uint32_t sum)
+{
+    sum = ((sum & 0xffff0000) >> 16) + (sum & 0xffff);
+    sum = ((sum & 0xffff0000) >> 16) + (sum & 0xffff);
+    return (uint16_t)sum;
+}
+
+static inline uint16_t rte_raw_cksum(const void *buf, size_t len)
+{
+    uint32_t sum;
+
+    sum = __rte_raw_cksum(buf, len, 0);
+    return __rte_raw_cksum_reduce(sum);
+}
+
+void xsum_tcp(void *tcphdr, size_t l4_len)
+{
+    struct rte_tcp_hdr *tcph = reinterpret_cast<struct rte_tcp_hdr *> (tcphdr);
+    uint32_t cksum = rte_raw_cksum(tcphdr, l4_len);
+	cksum = ((cksum & 0xffff0000) >> 16) + (cksum & 0xffff);
+	cksum = (~cksum) & 0xffff;
+    tcph->cksum = cksum;
+}
+
+}