i40e_lan.cc 21 KB
Newer Older
Antoine Kaufmann's avatar
Antoine Kaufmann committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
/*
 * Copyright 2021 Max Planck Institute for Software Systems, and
 * National University of Singapore
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

25
26
27
28
#include <stdlib.h>
#include <string.h>
#include <cassert>
#include <iostream>
Antoine Kaufmann's avatar
Antoine Kaufmann committed
29
#include <arpa/inet.h>
30
31
32
33

#include "i40e_bm.h"

#include "i40e_base_wrapper.h"
Antoine Kaufmann's avatar
Antoine Kaufmann committed
34
#include "headers.h"
35
36
37
38
39
40

using namespace i40e;

extern nicbm::Runner *runner;

lan::lan(i40e_bm &dev_, size_t num_qs_)
41
    : dev(dev_), log("lan"), rss_kc(dev_.regs.pfqf_hkey), num_qs(num_qs_)
42
{
43
44
45
46
47
    rxqs = new lan_queue_rx *[num_qs];
    txqs = new lan_queue_tx *[num_qs];

    for (size_t i = 0; i < num_qs; i++) {
        rxqs[i] = new lan_queue_rx(*this, dev.regs.qrx_tail[i], i,
48
49
                dev.regs.qrx_ena[i], dev.regs.glhmc_lanrxbase[0],
                dev.regs.qint_rqctl[i]);
50
        txqs[i] = new lan_queue_tx(*this, dev.regs.qtx_tail[i], i,
51
52
                dev.regs.qtx_ena[i], dev.regs.glhmc_lantxbase[0],
                dev.regs.qint_tqctl[i]);
53
    }
54
55
}

56
57
void lan::reset()
{
Antoine Kaufmann's avatar
Antoine Kaufmann committed
58
    rss_kc.set_dirty();
59
60
61
62
63
64
    for (size_t i = 0; i < num_qs; i++) {
        rxqs[i]->reset();
        txqs[i]->reset();
    }
}

65
66
void lan::qena_updated(uint16_t idx, bool rx)
{
67
    uint32_t &reg = (rx ? dev.regs.qrx_ena[idx] : dev.regs.qtx_ena[idx]);
68
#ifdef DEBUG_LAN
69
70
    log << " qena updated idx=" << idx << " rx=" << rx << " reg=" << reg <<
        logger::endl;
71
#endif
72
73
74
75
76
77
78
79
    lan_queue_base &q = (rx ? static_cast<lan_queue_base &>(*rxqs[idx]) :
        static_cast<lan_queue_base &>(*txqs[idx]));

    if ((reg & I40E_QRX_ENA_QENA_REQ_MASK) && !q.is_enabled()) {
        q.enable();
    } else if (!(reg & I40E_QRX_ENA_QENA_REQ_MASK) && q.is_enabled()) {
        q.disable();
    }
80
81
82
83
}

void lan::tail_updated(uint16_t idx, bool rx)
{
84
#ifdef DEBUG_LAN
85
    log << " tail updated idx=" << idx << " rx=" << rx << logger::endl;
86
#endif
87
88
89
90
91
92
93
94

    lan_queue_base &q = (rx ? static_cast<lan_queue_base &>(*rxqs[idx]) :
        static_cast<lan_queue_base &>(*txqs[idx]));

    if (q.is_enabled())
        q.reg_updated();
}

Antoine Kaufmann's avatar
Antoine Kaufmann committed
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
void lan::rss_key_updated()
{
    rss_kc.set_dirty();
}

bool lan::rss_steering(const void *data, size_t len, uint16_t &queue,
        uint32_t &hash)
{
    hash = 0;

    const headers::pkt_tcp *tcp = reinterpret_cast<const headers::pkt_tcp *> (data);
    const headers::pkt_udp *udp = reinterpret_cast<const headers::pkt_udp *> (data);

    // should actually determine packet type and mask with enabled packet types
    // TODO: ipv6
    if (tcp->eth.type == htons(ETH_TYPE_IP) &&
            tcp->ip.proto == IP_PROTO_TCP)
    {
        hash = rss_kc.hash_ipv4(ntohl(tcp->ip.src), ntohl(tcp->ip.dest),
                ntohs(tcp->tcp.src), ntohs(tcp->tcp.dest));
    } else if (udp->eth.type == htons(ETH_TYPE_IP) &&
            udp->ip.proto == IP_PROTO_UDP)
    {
        hash = rss_kc.hash_ipv4(ntohl(udp->ip.src), ntohl(udp->ip.dest),
                ntohs(udp->udp.src), ntohs(udp->udp.dest));
    } else if (udp->eth.type == htons(ETH_TYPE_IP)) {
        hash = rss_kc.hash_ipv4(ntohl(udp->ip.src), ntohl(udp->ip.dest), 0, 0);
    } else {
        return false;
    }

    uint16_t luts = (!(dev.regs.pfqf_ctl_0 & I40E_PFQF_CTL_0_HASHLUTSIZE_MASK) ?
            128 : 512);
    uint16_t idx = hash % luts;
    queue = (dev.regs.pfqf_hlut[idx / 4] >> (8 * (idx % 4))) & 0x3f;
130
131
132
#ifdef DEBUG_LAN
    log << "  q=" << queue << " h=" << hash << " i=" << idx << logger::endl;
#endif
Antoine Kaufmann's avatar
Antoine Kaufmann committed
133
134
135
    return true;
}

136
137
void lan::packet_received(const void *data, size_t len)
{
138
#ifdef DEBUG_LAN
139
    log << " packet received len=" << len << logger::endl;
140
#endif
141

Antoine Kaufmann's avatar
Antoine Kaufmann committed
142
143
144
145
    uint32_t hash = 0;
    uint16_t queue = 0;
    rss_steering(data, len, queue, hash);
    rxqs[queue]->packet_received(data, len, hash);
146
147
}

148
149
lan_queue_base::lan_queue_base(lan &lanmgr_, const std::string &qtype,
        uint32_t &reg_tail_, size_t idx_,
150
151
        uint32_t &reg_ena_, uint32_t &fpm_basereg_, uint32_t &reg_intqctl_,
        uint16_t ctx_size_)
152
153
    : queue_base(qtype + std::to_string(idx_), reg_dummy_head, reg_tail_),
    lanmgr(lanmgr_), enabling(false),
154
    idx(idx_), reg_ena(reg_ena_), fpm_basereg(fpm_basereg_),
155
    reg_intqctl(reg_intqctl_), ctx_size(ctx_size_)
156
157
158
159
{
    ctx = new uint8_t[ctx_size_];
}

160
161
162
163
164
165
void lan_queue_base::reset()
{
    enabling = false;
    queue_base::reset();
}

166
167
168
169
170
void lan_queue_base::enable()
{
    if (enabling || enabled)
        return;

171
#ifdef DEBUG_LAN
172
    log << " lan enabling queue " << idx << logger::endl;
173
#endif
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
    enabling = true;

    qctx_fetch *qf = new qctx_fetch(*this);
    qf->write = false;
    qf->dma_addr = ((fpm_basereg & I40E_GLHMC_LANTXBASE_FPMLANTXBASE_MASK) >>
        I40E_GLHMC_LANTXBASE_FPMLANTXBASE_SHIFT) * 512;
    qf->dma_addr += ctx_size * idx;
    qf->len = ctx_size;
    qf->data = ctx;

    lanmgr.dev.hmc.issue_mem_op(*qf);
}

void lan_queue_base::ctx_fetched()
{
189
#ifdef DEBUG_LAN
190
    log << " lan ctx fetched " << idx << logger::endl;
191
#endif
192
193
194
195
196
197
198
199
200
201
202
203

    initialize();

    enabling = false;
    enabled = true;
    reg_ena |= I40E_QRX_ENA_QENA_STAT_MASK;

    reg_updated();
}

void lan_queue_base::disable()
{
204
#ifdef DEBUG_LAN
205
    log << " lan disabling queue " << idx << logger::endl;
206
#endif
207
208
209
210
211
    enabled = false;
    // TODO: write back
    reg_ena &= ~I40E_QRX_ENA_QENA_STAT_MASK;
}

212
213
214
void lan_queue_base::interrupt()
{
    uint32_t qctl = reg_intqctl;
215
    uint32_t gctl = lanmgr.dev.regs.pfint_dyn_ctl0;
216
#ifdef DEBUG_LAN
217
    log << " interrupt qctl=" << qctl << " gctl=" << gctl << logger::endl;
218
#endif
219
220

    uint16_t msix_idx = (qctl & I40E_QINT_TQCTL_MSIX_INDX_MASK) >>
Antoine Kaufmann's avatar
Antoine Kaufmann committed
221
        I40E_QINT_TQCTL_MSIX_INDX_SHIFT;
222
223
224
    uint8_t msix0_idx = (qctl & I40E_QINT_TQCTL_MSIX0_INDX_MASK) >>
        I40E_QINT_TQCTL_MSIX0_INDX_SHIFT;

225
226
    bool cause_ena = !!(qctl & I40E_QINT_TQCTL_CAUSE_ENA_MASK) &&
      !!(gctl & I40E_PFINT_DYN_CTL0_INTENA_MASK);
227
    if (!cause_ena) {
228
#ifdef DEBUG_LAN
229
        log << " interrupt cause disabled" << logger::endl;
230
#endif
231
232
233
        return;
    }

Antoine Kaufmann's avatar
Antoine Kaufmann committed
234
    if (msix_idx == 0) {
235
#ifdef DEBUG_LAN
Antoine Kaufmann's avatar
Antoine Kaufmann committed
236
        log << "   setting int0.qidx=" << msix0_idx << logger::endl;
237
#endif
Antoine Kaufmann's avatar
Antoine Kaufmann committed
238
239
240
        lanmgr.dev.regs.pfint_icr0 |= I40E_PFINT_ICR0_INTEVENT_MASK |
            (1 << (I40E_PFINT_ICR0_QUEUE_0_SHIFT + msix0_idx));
    }
241
242
243

    uint8_t itr = (qctl & I40E_QINT_TQCTL_ITR_INDX_MASK) >>
        I40E_QINT_TQCTL_ITR_INDX_SHIFT;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
244
    lanmgr.dev.signal_interrupt(msix_idx, itr);
245

246
247
}

248
249
250
251
252
253
254
255
256
257
258
259
lan_queue_base::qctx_fetch::qctx_fetch(lan_queue_base &lq_)
    : lq(lq_)
{
}

void lan_queue_base::qctx_fetch::done()
{
    lq.ctx_fetched();
    delete this;
}

lan_queue_rx::lan_queue_rx(lan &lanmgr_, uint32_t &reg_tail_, size_t idx_,
260
        uint32_t &reg_ena_, uint32_t &reg_fpmbase_, uint32_t &reg_intqctl_)
261
    : lan_queue_base(lanmgr_, "rxq", reg_tail_, idx_, reg_ena_, reg_fpmbase_,
262
            reg_intqctl_, 32)
263
{
264
265
266
    // use larger value for initialization
    desc_len = 32;
    ctxs_init();
267
268
}

269
270
void lan_queue_rx::reset()
{
271
    dcache.clear();
272
273
274
    queue_base::reset();
}

275
276
void lan_queue_rx::initialize()
{
277
#ifdef DEBUG_LAN
278
    log << " initialize()" << logger::endl;
279
#endif
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
    uint8_t *ctx_p = reinterpret_cast<uint8_t *>(ctx);

    uint16_t *head_p = reinterpret_cast<uint16_t *>(ctx_p + 0);
    uint64_t *base_p = reinterpret_cast<uint64_t *>(ctx_p + 4);
    uint16_t *qlen_p = reinterpret_cast<uint16_t *>(ctx_p + 11);
    uint16_t *dbsz_p = reinterpret_cast<uint16_t *>(ctx_p + 12);
    uint16_t *hbsz_p = reinterpret_cast<uint16_t *>(ctx_p + 13);
    uint32_t *rxmax_p = reinterpret_cast<uint32_t *>(ctx_p + 21);

    reg_dummy_head = (*head_p) & ((1 << 13) - 1);

    base = ((*base_p) & ((1ULL << 57) - 1)) * 128;
    len = (*qlen_p >> 1) & ((1 << 13) - 1);

    dbuff_size = (((*dbsz_p) >> 6) & ((1 << 7) - 1)) * 128;
    hbuff_size = (((*hbsz_p) >> 5) & ((1 << 5) - 1)) * 64;
    uint8_t dtype = ((*hbsz_p) >> 10) & ((1 << 2) - 1);
    bool longdesc = !!(((*hbsz_p) >> 12) & 0x1);
    desc_len = (longdesc ? 32 : 16);
    crc_strip = !!(((*hbsz_p) >> 13) & 0x1);
    rxmax = (((*rxmax_p) >> 6) & ((1 << 14) - 1)) * 128;

    if (!longdesc) {
303
304
        log << "lan_queue_rx::initialize: currently only 32B descs "
            " supported" << logger::endl;
305
306
307
        abort();
    }
    if (dtype != 0) {
308
309
        log << "lan_queue_rx::initialize: no header split supported"
            << logger::endl;
310
311
312
        abort();
    }

313
#ifdef DEBUG_LAN
314
    log << "  head=" << reg_dummy_head << " base=" << base <<
315
316
        " len=" << len << " dbsz=" << dbuff_size << " hbsz=" << hbuff_size <<
        " dtype=" << (unsigned) dtype << " longdesc=" << longdesc <<
317
        " crcstrip=" << crc_strip << " rxmax=" << rxmax << logger::endl;
318
#endif
319
320
}

321
queue_base::desc_ctx &lan_queue_rx::desc_ctx_create()
322
{
323
    return *new rx_desc_ctx(*this);
324
325
}

Antoine Kaufmann's avatar
Antoine Kaufmann committed
326
void lan_queue_rx::packet_received(const void *data, size_t pktlen, uint32_t h)
327
{
328
329
    size_t num_descs = (pktlen + dbuff_size - 1) / dbuff_size;

330
331
332
    if (!enabled)
        return;

333
    if (dcache.size() < num_descs) {
334
#ifdef DEBUG_LAN
335
336
        log << " not enough rx descs (" << num_descs << ", dropping packet" <<
            logger::endl;
337
#endif
338
339
        return;
    }
340

341
342
    for (size_t i = 0; i < num_descs; i++) {
        rx_desc_ctx &ctx = *dcache.front();
343

344
#ifdef DEBUG_LAN
345
346
        log << " packet part=" << i <<  " received didx=" << ctx.index <<
            " cnt=" << dcache.size() << logger::endl;
347
#endif
348
349
350
351
352
353
354
355
356
357
        dcache.pop_front();

        const uint8_t *buf = (const uint8_t *) data + (dbuff_size * i);
        if (i == num_descs - 1) {
            // last packet
            ctx.packet_received(buf, pktlen - dbuff_size * i, true);
        } else {
            ctx.packet_received(buf, dbuff_size, false);
        }
    }
358
359
}

360
361
lan_queue_rx::rx_desc_ctx::rx_desc_ctx(lan_queue_rx &queue_)
    : desc_ctx(queue_), rq(queue_)
362
363
364
{
}

365
void lan_queue_rx::rx_desc_ctx::data_written(uint64_t addr, size_t len)
366
{
367
368
    processed();
}
369

370
371
372
373
void lan_queue_rx::rx_desc_ctx::process()
{
    rq.dcache.push_back(this);
}
374

375
376
void lan_queue_rx::rx_desc_ctx::packet_received(const void *data,
        size_t pktlen, bool last)
377
378
379
380
381
382
383
384
385
{
    union i40e_32byte_rx_desc *rxd = reinterpret_cast<
        union i40e_32byte_rx_desc *> (desc);

    uint64_t addr = rxd->read.pkt_addr;

    memset(rxd, 0, sizeof(*rxd));
    rxd->wb.qword1.status_error_len |= (1 << I40E_RX_DESC_STATUS_DD_SHIFT);
    rxd->wb.qword1.status_error_len |= (pktlen << I40E_RXD_QW1_LENGTH_PBUF_SHIFT);
386

387
388
389
390
391
    if (last) {
        rxd->wb.qword1.status_error_len |= (1 << I40E_RX_DESC_STATUS_EOF_SHIFT);
        // TODO: only if checksums are correct
        rxd->wb.qword1.status_error_len |= (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT);
    }
392
    data_write(addr, pktlen, data);
393
394
}

395
lan_queue_tx::lan_queue_tx(lan &lanmgr_, uint32_t &reg_tail_, size_t idx_,
396
        uint32_t &reg_ena_, uint32_t &reg_fpmbase_, uint32_t &reg_intqctl)
397
    : lan_queue_base(lanmgr_, "txq", reg_tail_, idx_, reg_ena_, reg_fpmbase_,
398
            reg_intqctl, 128)
399
400
{
    desc_len = 16;
401
    ctxs_init();
402
403
}

Antoine Kaufmann's avatar
Antoine Kaufmann committed
404
405
void lan_queue_tx::reset()
{
Antoine Kaufmann's avatar
Antoine Kaufmann committed
406
407
    tso_off = 0;
    tso_len = 0;
408
    ready_segments.clear();
Antoine Kaufmann's avatar
Antoine Kaufmann committed
409
410
411
    queue_base::reset();
}

412
413
void lan_queue_tx::initialize()
{
414
#ifdef DEBUG_LAN
415
    log << " initialize()" << logger::endl;
416
#endif
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
    uint8_t *ctx_p = reinterpret_cast<uint8_t *>(ctx);

    uint16_t *head_p = reinterpret_cast<uint16_t *>(ctx_p + 0);
    uint64_t *base_p = reinterpret_cast<uint64_t *>(ctx_p + 4);
    uint16_t *hwb_qlen_p = reinterpret_cast<uint16_t *>(ctx_p + 20);
    uint64_t *hwb_addr_p = reinterpret_cast<uint64_t *>(ctx_p + 24);

    reg_dummy_head = (*head_p) & ((1 << 13) - 1);

    base = ((*base_p) & ((1ULL << 57) - 1)) * 128;
    len = ((*hwb_qlen_p) >> 1) & ((1 << 13) - 1);

    hwb = !!(*hwb_qlen_p & (1 << 0));
    hwb_addr = *hwb_addr_p;

432
#ifdef DEBUG_LAN
433
    log << "  head=" << reg_dummy_head << " base=" << base <<
434
        " len=" << len << " hwb=" << hwb << " hwb_addr=" << hwb_addr <<
435
        logger::endl;
436
#endif
437
}
438

439
queue_base::desc_ctx &lan_queue_tx::desc_ctx_create()
440
{
441
442
    return *new tx_desc_ctx(*this);
}
443

444
445
446
447
448
449
450
451
452
453
454
455
void lan_queue_tx::do_writeback(uint32_t first_idx, uint32_t first_pos,
        uint32_t cnt)
{
    if (!hwb) {
        // if head index writeback is disabled we need to write descriptor back
        lan_queue_base::do_writeback(first_idx, first_pos, cnt);
    } else {
        // else we just need to write the index back
        dma_hwb *dma = new dma_hwb(*this, first_pos, cnt,
                (first_idx + cnt) % len);
        dma->dma_addr = hwb_addr;

456
#ifdef DEBUG_LAN
457
        log << " hwb=" << *((uint32_t *) dma->data) << logger::endl;
458
#endif
459
460
461
462
463
464
465
        runner->issue_dma(*dma);
    }
}

bool lan_queue_tx::trigger_tx_packet()
{
    size_t n = ready_segments.size();
Antoine Kaufmann's avatar
Antoine Kaufmann committed
466
467
468
469
470
471
472
473
474
    size_t d_skip = 0, dcnt;
    bool eop = false;
    uint64_t d1;
    uint32_t iipt, l4t, pkt_len, total_len = 0, data_limit;
    bool tso = false;
    uint32_t tso_mss = 0, tso_paylen = 0;
    uint16_t maclen = 0, iplen = 0, l4len = 0;

    // abort if no queued up descriptors
475
476
477
    if (n == 0)
        return false;

Antoine Kaufmann's avatar
Antoine Kaufmann committed
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
#ifdef DEBUG_LAN
    log << "trigger_tx_packet(n=" << n << ", firstidx=" <<
        ready_segments.at(0)->index << ")" << logger::endl;
    log << "  tso_off=" << tso_off << " tso_len=" << tso_len << logger::endl;
#endif


    // check if we have a context descriptor first
    tx_desc_ctx *rd = ready_segments.at(0);
    uint8_t dtype = (rd->d->cmd_type_offset_bsz & I40E_TXD_QW1_DTYPE_MASK) >>
        I40E_TXD_QW1_DTYPE_SHIFT;
    if (dtype == I40E_TX_DESC_DTYPE_CONTEXT) {
        struct i40e_tx_context_desc *ctxd =
            reinterpret_cast<struct i40e_tx_context_desc *> (rd->d);
        d1 = ctxd->type_cmd_tso_mss;

        uint16_t cmd = ((d1 & I40E_TXD_CTX_QW1_CMD_MASK) >>
                I40E_TXD_CTX_QW1_CMD_SHIFT);
        tso = !!(cmd & I40E_TX_CTX_DESC_TSO);
        tso_mss = (d1 & I40E_TXD_CTX_QW1_MSS_MASK) >>
            I40E_TXD_CTX_QW1_MSS_SHIFT;

#ifdef DEBUG_LAN
        log << "  tso=" << tso << " mss=" << tso_mss << logger::endl;
#endif

        d_skip = 1;
    }
506

Antoine Kaufmann's avatar
Antoine Kaufmann committed
507
508
509
    // find EOP descriptor
    for (dcnt = d_skip; dcnt < n && !eop; dcnt++) {
        tx_desc_ctx *rd = ready_segments.at(dcnt);
510
        d1 = rd->d->cmd_type_offset_bsz;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
511

512
#ifdef DEBUG_LAN
513
514
        log << " data fetched didx=" << rd->index << " d1=" <<
            d1 << logger::endl;
515
#endif
516

Antoine Kaufmann's avatar
Antoine Kaufmann committed
517
518
519
520
        dtype = (d1 & I40E_TXD_QW1_DTYPE_MASK) >> I40E_TXD_QW1_DTYPE_SHIFT;
        if (dtype != I40E_TX_DESC_DTYPE_DATA) {
            log << "trigger tx desc is not a data descriptor idx=" << rd->index
                << " d1=" << d1 << logger::endl;
521
522
            abort();
        }
523

524
525
526
527
528
        uint16_t cmd = (d1 & I40E_TXD_QW1_CMD_MASK) >> I40E_TXD_QW1_CMD_SHIFT;
        eop = (cmd & I40E_TX_DESC_CMD_EOP);
        iipt = cmd & (I40E_TX_DESC_CMD_IIPT_MASK);
        l4t = (cmd & I40E_TX_DESC_CMD_L4T_EOFT_MASK);

Antoine Kaufmann's avatar
Antoine Kaufmann committed
529
530
531
532
533
534
535
536
537
        if (eop) {
            uint32_t off = (d1 & I40E_TXD_QW1_OFFSET_MASK) >> I40E_TXD_QW1_OFFSET_SHIFT;
            maclen = ((off & I40E_TXD_QW1_MACLEN_MASK) >>
                I40E_TX_DESC_LENGTH_MACLEN_SHIFT) * 2;
            iplen = ((off & I40E_TXD_QW1_IPLEN_MASK) >>
                I40E_TX_DESC_LENGTH_IPLEN_SHIFT) * 4;
            l4len = ((off & I40E_TXD_QW1_L4LEN_MASK) >>
                I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT) * 4;
        }
538

Antoine Kaufmann's avatar
Antoine Kaufmann committed
539
540
        pkt_len = (d1 & I40E_TXD_QW1_TX_BUF_SZ_MASK) >>
            I40E_TXD_QW1_TX_BUF_SZ_SHIFT;
541
        total_len += pkt_len;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
542
543
544
545

#ifdef DEBUG_LAN
        log << "    eop=" << eop << " len=" << pkt_len << logger::endl;
#endif
546
547
    }

Antoine Kaufmann's avatar
Antoine Kaufmann committed
548
    // Unit not completely fetched yet
549
550
551
    if (!eop)
        return false;

Antoine Kaufmann's avatar
Antoine Kaufmann committed
552
553
554
555
556
    if (tso) {
        if (tso_off == 0)
            data_limit = maclen + iplen + l4len + tso_mss;
        else
            data_limit = tso_off + tso_mss;
557

Antoine Kaufmann's avatar
Antoine Kaufmann committed
558
559
560
561
562
563
564
565
566
567
        if (data_limit > total_len) {
            data_limit = total_len;
        }
    } else {
        if (total_len > MTU) {
            log << "    packet is longer (" << total_len << ") than MTU (" <<
                MTU << ")" << logger::endl;
            abort();
        }
        data_limit = total_len;
568
    }
Antoine Kaufmann's avatar
Antoine Kaufmann committed
569

570
#ifdef DEBUG_LAN
571
    log << "    iipt=" << iipt << " l4t=" << l4t <<
Antoine Kaufmann's avatar
Antoine Kaufmann committed
572
573
574
575
        " maclen=" << maclen << " iplen=" << iplen << " l4len=" << l4len <<
        " total_len=" << total_len << " data_limit=" << data_limit <<
        logger::endl;

576
577
578
#else
    (void) iipt;
#endif
579
580


Antoine Kaufmann's avatar
Antoine Kaufmann committed
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
    // copy data for this segment
    uint32_t off = 0;
    for (dcnt = d_skip; dcnt < n && off < data_limit; dcnt++) {
        tx_desc_ctx *rd = ready_segments.at(dcnt);
        d1 = rd->d->cmd_type_offset_bsz;
        uint16_t pkt_len = (d1 & I40E_TXD_QW1_TX_BUF_SZ_MASK) >>
            I40E_TXD_QW1_TX_BUF_SZ_SHIFT;

        if (off <= tso_off && off + pkt_len > tso_off) {
            uint32_t start = tso_off;
            uint32_t end = off + pkt_len;
            if (end > data_limit)
                end = data_limit;

#ifdef DEBUG_LAN
        log << "    copying data from off=" << off << " idx=" << rd->index <<
            " start=" << start << " end=" << end << " tso_len=" << tso_len <<
            logger::endl;
#endif

            memcpy(pktbuf + tso_len, (uint8_t *) rd->data + (start - off),
                    end - start);
            tso_off = end;
            tso_len += end - start;
        }

        off += pkt_len;
    }

    assert(tso_len <= MTU);

    if (!tso) {
#ifdef DEBUG_LAN
        log << "    normal non-tso packet" << logger::endl;
#endif

        if (l4t == I40E_TX_DESC_CMD_L4T_EOFT_TCP) {
            uint16_t tcp_off = maclen + iplen;
            xsum_tcp(pktbuf + tcp_off, tso_len - tcp_off);
        }

        runner->eth_send(pktbuf, tso_len);
    } else {
#ifdef DEBUG_LAN
        log << "    tso packet off=" << tso_off << " len=" << tso_len <<
            logger::endl;
#endif

        // TSO gets hairier
        uint16_t hdrlen = maclen + iplen + l4len;

        // calculate payload size
        tso_paylen = tso_len - hdrlen;
        if (tso_paylen > tso_mss)
            tso_paylen = tso_mss;

        xsum_tcpip_tso(pktbuf + maclen, iplen, l4len, tso_paylen);

        runner->eth_send(pktbuf, tso_len);

        tso_postupdate_header(pktbuf + maclen, iplen, l4len, tso_paylen);

        // not done yet with this TSO unit
        if (tso && tso_off < total_len) {
            tso_len = hdrlen;
            return true;
        }
    }

#ifdef DEBUG_LAN
        log << "    unit done" << logger::endl;
#endif
653
654
655
656
657
    while (dcnt-- > 0) {
        ready_segments.front()->processed();
        ready_segments.pop_front();
    }

Antoine Kaufmann's avatar
Antoine Kaufmann committed
658
659
660
    tso_len = 0;
    tso_off = 0;

661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
    return true;
}

void lan_queue_tx::trigger_tx()
{
    while (trigger_tx_packet());
}

lan_queue_tx::tx_desc_ctx::tx_desc_ctx(lan_queue_tx &queue_)
    : desc_ctx(queue_), tq(queue_)
{
    d = reinterpret_cast<struct i40e_tx_desc *>(desc);
}

void lan_queue_tx::tx_desc_ctx::prepare()
{
    uint64_t d1 = d->cmd_type_offset_bsz;

679
#ifdef DEBUG_LAN
680
681
    queue.log << " desc fetched didx=" << index << " d1=" <<
        d1 << logger::endl;
682
#endif
683

684
    uint8_t dtype = (d1 & I40E_TXD_QW1_DTYPE_MASK) >> I40E_TXD_QW1_DTYPE_SHIFT;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
685
686
687
688
    if (dtype == I40E_TX_DESC_DTYPE_DATA) {
        uint16_t len = (d1 & I40E_TXD_QW1_TX_BUF_SZ_MASK) >>
            I40E_TXD_QW1_TX_BUF_SZ_SHIFT;

689
#ifdef DEBUG_LAN
690
691
        queue.log << "  bufaddr=" << d->buffer_addr <<
            " len=" << len << logger::endl;
692
#endif
Antoine Kaufmann's avatar
Antoine Kaufmann committed
693

694
        data_fetch(d->buffer_addr, len);
Antoine Kaufmann's avatar
Antoine Kaufmann committed
695
    } else if (dtype == I40E_TX_DESC_DTYPE_CONTEXT) {
Antoine Kaufmann's avatar
Antoine Kaufmann committed
696
#ifdef DEBUG_LAN
Antoine Kaufmann's avatar
Antoine Kaufmann committed
697
        struct i40e_tx_context_desc *ctxd =
698
            reinterpret_cast<struct i40e_tx_context_desc *> (d);
699
700
        queue.log << "  context descriptor: tp=" << ctxd->tunneling_params <<
            " l2t=" << ctxd->l2tag2 << " tctm=" << ctxd->type_cmd_tso_mss << logger::endl;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
701
#endif
Antoine Kaufmann's avatar
Antoine Kaufmann committed
702

Antoine Kaufmann's avatar
Antoine Kaufmann committed
703
        prepared();
Antoine Kaufmann's avatar
Antoine Kaufmann committed
704
    } else {
705
        queue.log << "txq: only support context & data descriptors" << logger::endl;
706
707
        abort();
    }
708
709
710
711
712
713
714
715
716

}

void lan_queue_tx::tx_desc_ctx::process()
{
    tq.ready_segments.push_back(this);
    tq.trigger_tx();
}

717
718
719
720
721
722
723
void lan_queue_tx::tx_desc_ctx::processed()
{
    d->cmd_type_offset_bsz = I40E_TX_DESC_DTYPE_DESC_DONE <<
        I40E_TXD_QW1_DTYPE_SHIFT;
    desc_ctx::processed();
}

724
725
726
lan_queue_tx::dma_hwb::dma_hwb(lan_queue_tx &queue_, uint32_t pos_,
        uint32_t cnt_, uint32_t nh_)
    : queue(queue_), pos(pos_), cnt(cnt_), next_head(nh_)
727
728
729
730
731
732
733
734
735
736
737
738
{
    data = &next_head;
    len = 4;
    write = true;
}

lan_queue_tx::dma_hwb::~dma_hwb()
{
}

void lan_queue_tx::dma_hwb::done()
{
739
#ifdef DEBUG_LAN
740
    queue.log << " tx head written back" << logger::endl;
741
#endif
742
    queue.writeback_done(pos, cnt);
743
    queue.trigger();
744
745
    delete this;
}