i40e_lan.cc 20.7 KB
Newer Older
Antoine Kaufmann's avatar
Antoine Kaufmann committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
/*
 * Copyright 2021 Max Planck Institute for Software Systems, and
 * National University of Singapore
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

25
#include <arpa/inet.h>
26
27
#include <stdlib.h>
#include <string.h>
28

29
30
31
#include <cassert>
#include <iostream>

Antoine Kaufmann's avatar
Antoine Kaufmann committed
32
#include "sims/nic/i40e_bm/headers.h"
33
34
#include "sims/nic/i40e_bm/i40e_base_wrapper.h"
#include "sims/nic/i40e_bm/i40e_bm.h"
35

36
namespace i40e {
37
38
39
40

extern nicbm::Runner *runner;

lan::lan(i40e_bm &dev_, size_t num_qs_)
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
    : dev(dev_), log("lan"), rss_kc(dev_.regs.pfqf_hkey), num_qs(num_qs_) {
  rxqs = new lan_queue_rx *[num_qs];
  txqs = new lan_queue_tx *[num_qs];

  for (size_t i = 0; i < num_qs; i++) {
    rxqs[i] =
        new lan_queue_rx(*this, dev.regs.qrx_tail[i], i, dev.regs.qrx_ena[i],
                         dev.regs.glhmc_lanrxbase[0], dev.regs.qint_rqctl[i]);
    txqs[i] =
        new lan_queue_tx(*this, dev.regs.qtx_tail[i], i, dev.regs.qtx_ena[i],
                         dev.regs.glhmc_lantxbase[0], dev.regs.qint_tqctl[i]);
  }
}

void lan::reset() {
  rss_kc.set_dirty();
  for (size_t i = 0; i < num_qs; i++) {
    rxqs[i]->reset();
    txqs[i]->reset();
  }
}

void lan::qena_updated(uint16_t idx, bool rx) {
  uint32_t &reg = (rx ? dev.regs.qrx_ena[idx] : dev.regs.qtx_ena[idx]);
65
#ifdef DEBUG_LAN
66
67
  log << " qena updated idx=" << idx << " rx=" << rx << " reg=" << reg
      << logger::endl;
68
#endif
69
70
  lan_queue_base &q = (rx ? static_cast<lan_queue_base &>(*rxqs[idx])
                          : static_cast<lan_queue_base &>(*txqs[idx]));
71

72
73
74
75
76
  if ((reg & I40E_QRX_ENA_QENA_REQ_MASK) && !q.is_enabled()) {
    q.enable();
  } else if (!(reg & I40E_QRX_ENA_QENA_REQ_MASK) && q.is_enabled()) {
    q.disable();
  }
77
78
}

79
void lan::tail_updated(uint16_t idx, bool rx) {
80
#ifdef DEBUG_LAN
81
  log << " tail updated idx=" << idx << " rx=" << rx << logger::endl;
82
#endif
83

84
85
  lan_queue_base &q = (rx ? static_cast<lan_queue_base &>(*rxqs[idx])
                          : static_cast<lan_queue_base &>(*txqs[idx]));
86

87
88
  if (q.is_enabled())
    q.reg_updated();
89
90
}

91
92
void lan::rss_key_updated() {
  rss_kc.set_dirty();
Antoine Kaufmann's avatar
Antoine Kaufmann committed
93
94
95
}

bool lan::rss_steering(const void *data, size_t len, uint16_t &queue,
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
                       uint32_t &hash) {
  hash = 0;

  const headers::pkt_tcp *tcp =
      reinterpret_cast<const headers::pkt_tcp *>(data);
  const headers::pkt_udp *udp =
      reinterpret_cast<const headers::pkt_udp *>(data);

  // should actually determine packet type and mask with enabled packet types
  // TODO(antoinek): ipv6
  if (tcp->eth.type == htons(ETH_TYPE_IP) && tcp->ip.proto == IP_PROTO_TCP) {
    hash = rss_kc.hash_ipv4(ntohl(tcp->ip.src), ntohl(tcp->ip.dest),
                            ntohs(tcp->tcp.src), ntohs(tcp->tcp.dest));
  } else if (udp->eth.type == htons(ETH_TYPE_IP) &&
             udp->ip.proto == IP_PROTO_UDP) {
    hash = rss_kc.hash_ipv4(ntohl(udp->ip.src), ntohl(udp->ip.dest),
                            ntohs(udp->udp.src), ntohs(udp->udp.dest));
  } else if (udp->eth.type == htons(ETH_TYPE_IP)) {
    hash = rss_kc.hash_ipv4(ntohl(udp->ip.src), ntohl(udp->ip.dest), 0, 0);
  } else {
    return false;
  }

  uint16_t luts =
      (!(dev.regs.pfqf_ctl_0 & I40E_PFQF_CTL_0_HASHLUTSIZE_MASK) ? 128 : 512);
  uint16_t idx = hash % luts;
  queue = (dev.regs.pfqf_hlut[idx / 4] >> (8 * (idx % 4))) & 0x3f;
123
#ifdef DEBUG_LAN
124
  log << "  q=" << queue << " h=" << hash << " i=" << idx << logger::endl;
125
#endif
126
  return true;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
127
128
}

129
void lan::packet_received(const void *data, size_t len) {
130
#ifdef DEBUG_LAN
131
  log << " packet received len=" << len << logger::endl;
132
#endif
133

134
135
136
137
  uint32_t hash = 0;
  uint16_t queue = 0;
  rss_steering(data, len, queue, hash);
  rxqs[queue]->packet_received(data, len, hash);
138
139
}

140
lan_queue_base::lan_queue_base(lan &lanmgr_, const std::string &qtype,
141
142
143
                               uint32_t &reg_tail_, size_t idx_,
                               uint32_t &reg_ena_, uint32_t &fpm_basereg_,
                               uint32_t &reg_intqctl_, uint16_t ctx_size_)
144
    : queue_base(qtype + std::to_string(idx_), reg_dummy_head, reg_tail_),
145
146
147
148
149
150
151
152
      lanmgr(lanmgr_),
      enabling(false),
      idx(idx_),
      reg_ena(reg_ena_),
      fpm_basereg(fpm_basereg_),
      reg_intqctl(reg_intqctl_),
      ctx_size(ctx_size_) {
  ctx = new uint8_t[ctx_size_];
153
154
}

155
156
157
void lan_queue_base::reset() {
  enabling = false;
  queue_base::reset();
158
159
}

160
161
162
void lan_queue_base::enable() {
  if (enabling || enabled)
    return;
163

164
#ifdef DEBUG_LAN
165
  log << " lan enabling queue " << idx << logger::endl;
166
#endif
167
  enabling = true;
168

169
  qctx_fetch *qf = new qctx_fetch(*this);
170
171
  qf->write_ = false;
  qf->dma_addr_ = ((fpm_basereg & I40E_GLHMC_LANTXBASE_FPMLANTXBASE_MASK) >>
172
173
                   I40E_GLHMC_LANTXBASE_FPMLANTXBASE_SHIFT) *
                  512;
174
175
176
  qf->dma_addr_ += ctx_size * idx;
  qf->len_ = ctx_size;
  qf->data_ = ctx;
177

178
  lanmgr.dev.hmc.issue_mem_op(*qf);
179
180
}

181
void lan_queue_base::ctx_fetched() {
182
#ifdef DEBUG_LAN
183
  log << " lan ctx fetched " << idx << logger::endl;
184
#endif
185

186
  initialize();
187

188
189
190
  enabling = false;
  enabled = true;
  reg_ena |= I40E_QRX_ENA_QENA_STAT_MASK;
191

192
  reg_updated();
193
194
}

195
void lan_queue_base::disable() {
196
#ifdef DEBUG_LAN
197
  log << " lan disabling queue " << idx << logger::endl;
198
#endif
199
200
201
  enabled = false;
  // TODO(antoinek): write back
  reg_ena &= ~I40E_QRX_ENA_QENA_STAT_MASK;
202
203
}

204
205
206
void lan_queue_base::interrupt() {
  uint32_t qctl = reg_intqctl;
  uint32_t gctl = lanmgr.dev.regs.pfint_dyn_ctl0;
207
#ifdef DEBUG_LAN
208
  log << " interrupt qctl=" << qctl << " gctl=" << gctl << logger::endl;
209
#endif
210

211
212
213
214
  uint16_t msix_idx = (qctl & I40E_QINT_TQCTL_MSIX_INDX_MASK) >>
                      I40E_QINT_TQCTL_MSIX_INDX_SHIFT;
  uint8_t msix0_idx = (qctl & I40E_QINT_TQCTL_MSIX0_INDX_MASK) >>
                      I40E_QINT_TQCTL_MSIX0_INDX_SHIFT;
215

216
217
218
  bool cause_ena = !!(qctl & I40E_QINT_TQCTL_CAUSE_ENA_MASK) &&
                   !!(gctl & I40E_PFINT_DYN_CTL0_INTENA_MASK);
  if (!cause_ena) {
219
#ifdef DEBUG_LAN
220
    log << " interrupt cause disabled" << logger::endl;
221
#endif
222
223
    return;
  }
224

225
  if (msix_idx == 0) {
226
#ifdef DEBUG_LAN
227
    log << "   setting int0.qidx=" << msix0_idx << logger::endl;
228
#endif
229
230
231
232
    lanmgr.dev.regs.pfint_icr0 |=
        I40E_PFINT_ICR0_INTEVENT_MASK |
        (1 << (I40E_PFINT_ICR0_QUEUE_0_SHIFT + msix0_idx));
  }
233

234
235
  uint8_t itr =
      (qctl & I40E_QINT_TQCTL_ITR_INDX_MASK) >> I40E_QINT_TQCTL_ITR_INDX_SHIFT;
236
  lanmgr.dev.SignalInterrupt(msix_idx, itr);
237
238
}

239
lan_queue_base::qctx_fetch::qctx_fetch(lan_queue_base &lq_) : lq(lq_) {
240
241
}

242
243
244
void lan_queue_base::qctx_fetch::done() {
  lq.ctx_fetched();
  delete this;
245
246
247
}

lan_queue_rx::lan_queue_rx(lan &lanmgr_, uint32_t &reg_tail_, size_t idx_,
248
249
                           uint32_t &reg_ena_, uint32_t &reg_fpmbase_,
                           uint32_t &reg_intqctl_)
250
    : lan_queue_base(lanmgr_, "rxq", reg_tail_, idx_, reg_ena_, reg_fpmbase_,
251
252
253
254
                     reg_intqctl_, 32) {
  // use larger value for initialization
  desc_len = 32;
  ctxs_init();
255
256
}

257
258
259
void lan_queue_rx::reset() {
  dcache.clear();
  queue_base::reset();
260
261
}

262
void lan_queue_rx::initialize() {
263
#ifdef DEBUG_LAN
264
  log << " initialize()" << logger::endl;
265
#endif
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
  uint8_t *ctx_p = reinterpret_cast<uint8_t *>(ctx);

  uint16_t *head_p = reinterpret_cast<uint16_t *>(ctx_p + 0);
  uint64_t *base_p = reinterpret_cast<uint64_t *>(ctx_p + 4);
  uint16_t *qlen_p = reinterpret_cast<uint16_t *>(ctx_p + 11);
  uint16_t *dbsz_p = reinterpret_cast<uint16_t *>(ctx_p + 12);
  uint16_t *hbsz_p = reinterpret_cast<uint16_t *>(ctx_p + 13);
  uint32_t *rxmax_p = reinterpret_cast<uint32_t *>(ctx_p + 21);

  reg_dummy_head = (*head_p) & ((1 << 13) - 1);

  base = ((*base_p) & ((1ULL << 57) - 1)) * 128;
  len = (*qlen_p >> 1) & ((1 << 13) - 1);

  dbuff_size = (((*dbsz_p) >> 6) & ((1 << 7) - 1)) * 128;
  hbuff_size = (((*hbsz_p) >> 5) & ((1 << 5) - 1)) * 64;
  uint8_t dtype = ((*hbsz_p) >> 10) & ((1 << 2) - 1);
  bool longdesc = !!(((*hbsz_p) >> 12) & 0x1);
  desc_len = (longdesc ? 32 : 16);
  crc_strip = !!(((*hbsz_p) >> 13) & 0x1);
  rxmax = (((*rxmax_p) >> 6) & ((1 << 14) - 1)) * 128;

  if (!longdesc) {
    log << "lan_queue_rx::initialize: currently only 32B descs "
           " supported"
        << logger::endl;
    abort();
  }
  if (dtype != 0) {
    log << "lan_queue_rx::initialize: no header split supported"
        << logger::endl;
    abort();
  }
299

300
#ifdef DEBUG_LAN
301
302
303
304
  log << "  head=" << reg_dummy_head << " base=" << base << " len=" << len
      << " dbsz=" << dbuff_size << " hbsz=" << hbuff_size
      << " dtype=" << (unsigned)dtype << " longdesc=" << longdesc
      << " crcstrip=" << crc_strip << " rxmax=" << rxmax << logger::endl;
305
#endif
306
307
}

308
309
queue_base::desc_ctx &lan_queue_rx::desc_ctx_create() {
  return *new rx_desc_ctx(*this);
310
311
}

312
313
314
void lan_queue_rx::packet_received(const void *data, size_t pktlen,
                                   uint32_t h) {
  size_t num_descs = (pktlen + dbuff_size - 1) / dbuff_size;
315

316
317
  if (!enabled)
    return;
318

319
  if (dcache.size() < num_descs) {
320
#ifdef DEBUG_LAN
321
322
    log << " not enough rx descs (" << num_descs << ", dropping packet"
        << logger::endl;
323
#endif
324
325
    return;
  }
326

327
328
  for (size_t i = 0; i < num_descs; i++) {
    rx_desc_ctx &ctx = *dcache.front();
329

330
#ifdef DEBUG_LAN
331
332
    log << " packet part=" << i << " received didx=" << ctx.index
        << " cnt=" << dcache.size() << logger::endl;
333
#endif
334
335
336
337
338
339
340
341
    dcache.pop_front();

    const uint8_t *buf = (const uint8_t *)data + (dbuff_size * i);
    if (i == num_descs - 1) {
      // last packet
      ctx.packet_received(buf, pktlen - dbuff_size * i, true);
    } else {
      ctx.packet_received(buf, dbuff_size, false);
342
    }
343
  }
344
345
}

346
lan_queue_rx::rx_desc_ctx::rx_desc_ctx(lan_queue_rx &queue_)
347
    : desc_ctx(queue_), rq(queue_) {
348
349
}

350
351
void lan_queue_rx::rx_desc_ctx::data_written(uint64_t addr, size_t len) {
  processed();
352
}
353

354
355
void lan_queue_rx::rx_desc_ctx::process() {
  rq.dcache.push_back(this);
356
}
357

358
359
360
361
void lan_queue_rx::rx_desc_ctx::packet_received(const void *data, size_t pktlen,
                                                bool last) {
  union i40e_32byte_rx_desc *rxd =
      reinterpret_cast<union i40e_32byte_rx_desc *>(desc);
362

363
  uint64_t addr = rxd->read.pkt_addr;
364

365
366
367
  memset(rxd, 0, sizeof(*rxd));
  rxd->wb.qword1.status_error_len |= (1 << I40E_RX_DESC_STATUS_DD_SHIFT);
  rxd->wb.qword1.status_error_len |= (pktlen << I40E_RXD_QW1_LENGTH_PBUF_SHIFT);
368

369
370
371
372
373
374
  if (last) {
    rxd->wb.qword1.status_error_len |= (1 << I40E_RX_DESC_STATUS_EOF_SHIFT);
    // TODO(antoinek): only if checksums are correct
    rxd->wb.qword1.status_error_len |= (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT);
  }
  data_write(addr, pktlen, data);
375
376
}

377
lan_queue_tx::lan_queue_tx(lan &lanmgr_, uint32_t &reg_tail_, size_t idx_,
378
379
                           uint32_t &reg_ena_, uint32_t &reg_fpmbase_,
                           uint32_t &reg_intqctl)
380
    : lan_queue_base(lanmgr_, "txq", reg_tail_, idx_, reg_ena_, reg_fpmbase_,
381
382
383
                     reg_intqctl, 128) {
  desc_len = 16;
  ctxs_init();
384
385
}

386
387
388
389
390
void lan_queue_tx::reset() {
  tso_off = 0;
  tso_len = 0;
  ready_segments.clear();
  queue_base::reset();
Antoine Kaufmann's avatar
Antoine Kaufmann committed
391
392
}

393
void lan_queue_tx::initialize() {
394
#ifdef DEBUG_LAN
395
  log << " initialize()" << logger::endl;
396
#endif
397
  uint8_t *ctx_p = reinterpret_cast<uint8_t *>(ctx);
398

399
400
401
402
  uint16_t *head_p = reinterpret_cast<uint16_t *>(ctx_p + 0);
  uint64_t *base_p = reinterpret_cast<uint64_t *>(ctx_p + 4);
  uint16_t *hwb_qlen_p = reinterpret_cast<uint16_t *>(ctx_p + 20);
  uint64_t *hwb_addr_p = reinterpret_cast<uint64_t *>(ctx_p + 24);
403

404
  reg_dummy_head = (*head_p) & ((1 << 13) - 1);
405

406
407
  base = ((*base_p) & ((1ULL << 57) - 1)) * 128;
  len = ((*hwb_qlen_p) >> 1) & ((1 << 13) - 1);
408

409
410
  hwb = !!(*hwb_qlen_p & (1 << 0));
  hwb_addr = *hwb_addr_p;
411

412
#ifdef DEBUG_LAN
413
414
  log << "  head=" << reg_dummy_head << " base=" << base << " len=" << len
      << " hwb=" << hwb << " hwb_addr=" << hwb_addr << logger::endl;
415
#endif
416
}
417

418
419
queue_base::desc_ctx &lan_queue_tx::desc_ctx_create() {
  return *new tx_desc_ctx(*this);
420
}
421

422
void lan_queue_tx::do_writeback(uint32_t first_idx, uint32_t first_pos,
423
424
425
426
427
428
429
                                uint32_t cnt) {
  if (!hwb) {
    // if head index writeback is disabled we need to write descriptor back
    lan_queue_base::do_writeback(first_idx, first_pos, cnt);
  } else {
    // else we just need to write the index back
    dma_hwb *dma = new dma_hwb(*this, first_pos, cnt, (first_idx + cnt) % len);
430
    dma->dma_addr_ = hwb_addr;
431

432
#ifdef DEBUG_LAN
433
    log << " hwb=" << *((uint32_t *)dma->data) << logger::endl;
434
#endif
435
    runner->IssueDma(*dma);
436
  }
437
438
}

439
440
441
442
443
444
445
446
447
bool lan_queue_tx::trigger_tx_packet() {
  size_t n = ready_segments.size();
  size_t d_skip = 0, dcnt;
  bool eop = false;
  uint64_t d1;
  uint32_t iipt, l4t, pkt_len, total_len = 0, data_limit;
  bool tso = false;
  uint32_t tso_mss = 0, tso_paylen = 0;
  uint16_t maclen = 0, iplen = 0, l4len = 0;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
448

449
450
451
  // abort if no queued up descriptors
  if (n == 0)
    return false;
452

Antoine Kaufmann's avatar
Antoine Kaufmann committed
453
#ifdef DEBUG_LAN
454
455
456
  log << "trigger_tx_packet(n=" << n
      << ", firstidx=" << ready_segments.at(0)->index << ")" << logger::endl;
  log << "  tso_off=" << tso_off << " tso_len=" << tso_len << logger::endl;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
457
458
#endif

459
460
461
462
463
464
465
466
  // check if we have a context descriptor first
  tx_desc_ctx *rd = ready_segments.at(0);
  uint8_t dtype = (rd->d->cmd_type_offset_bsz & I40E_TXD_QW1_DTYPE_MASK) >>
                  I40E_TXD_QW1_DTYPE_SHIFT;
  if (dtype == I40E_TX_DESC_DTYPE_CONTEXT) {
    struct i40e_tx_context_desc *ctxd =
        reinterpret_cast<struct i40e_tx_context_desc *>(rd->d);
    d1 = ctxd->type_cmd_tso_mss;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
467

468
469
470
471
    uint16_t cmd =
        ((d1 & I40E_TXD_CTX_QW1_CMD_MASK) >> I40E_TXD_CTX_QW1_CMD_SHIFT);
    tso = !!(cmd & I40E_TX_CTX_DESC_TSO);
    tso_mss = (d1 & I40E_TXD_CTX_QW1_MSS_MASK) >> I40E_TXD_CTX_QW1_MSS_SHIFT;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
472
473

#ifdef DEBUG_LAN
474
    log << "  tso=" << tso << " mss=" << tso_mss << logger::endl;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
475
476
#endif

477
478
    d_skip = 1;
  }
479

480
481
482
483
  // find EOP descriptor
  for (dcnt = d_skip; dcnt < n && !eop; dcnt++) {
    tx_desc_ctx *rd = ready_segments.at(dcnt);
    d1 = rd->d->cmd_type_offset_bsz;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
484

485
#ifdef DEBUG_LAN
486
    log << " data fetched didx=" << rd->index << " d1=" << d1 << logger::endl;
487
#endif
488

489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
    dtype = (d1 & I40E_TXD_QW1_DTYPE_MASK) >> I40E_TXD_QW1_DTYPE_SHIFT;
    if (dtype != I40E_TX_DESC_DTYPE_DATA) {
      log << "trigger tx desc is not a data descriptor idx=" << rd->index
          << " d1=" << d1 << logger::endl;
      abort();
    }

    uint16_t cmd = (d1 & I40E_TXD_QW1_CMD_MASK) >> I40E_TXD_QW1_CMD_SHIFT;
    eop = (cmd & I40E_TX_DESC_CMD_EOP);
    iipt = cmd & (I40E_TX_DESC_CMD_IIPT_MASK);
    l4t = (cmd & I40E_TX_DESC_CMD_L4T_EOFT_MASK);

    if (eop) {
      uint32_t off =
          (d1 & I40E_TXD_QW1_OFFSET_MASK) >> I40E_TXD_QW1_OFFSET_SHIFT;
      maclen = ((off & I40E_TXD_QW1_MACLEN_MASK) >>
                I40E_TX_DESC_LENGTH_MACLEN_SHIFT) *
               2;
      iplen =
          ((off & I40E_TXD_QW1_IPLEN_MASK) >> I40E_TX_DESC_LENGTH_IPLEN_SHIFT) *
          4;
      l4len = ((off & I40E_TXD_QW1_L4LEN_MASK) >>
               I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT) *
              4;
    }

    pkt_len =
        (d1 & I40E_TXD_QW1_TX_BUF_SZ_MASK) >> I40E_TXD_QW1_TX_BUF_SZ_SHIFT;
    total_len += pkt_len;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
518
519

#ifdef DEBUG_LAN
520
    log << "    eop=" << eop << " len=" << pkt_len << logger::endl;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
521
#endif
522
  }
523

524
525
526
  // Unit not completely fetched yet
  if (!eop)
    return false;
527

528
529
530
531
532
  if (tso) {
    if (tso_off == 0)
      data_limit = maclen + iplen + l4len + tso_mss;
    else
      data_limit = tso_off + tso_mss;
533

534
535
536
537
538
539
540
541
    if (data_limit > total_len) {
      data_limit = total_len;
    }
  } else {
    if (total_len > MTU) {
      log << "    packet is longer (" << total_len << ") than MTU (" << MTU
          << ")" << logger::endl;
      abort();
542
    }
543
544
    data_limit = total_len;
  }
Antoine Kaufmann's avatar
Antoine Kaufmann committed
545

546
#ifdef DEBUG_LAN
547
548
549
  log << "    iipt=" << iipt << " l4t=" << l4t << " maclen=" << maclen
      << " iplen=" << iplen << " l4len=" << l4len << " total_len=" << total_len
      << " data_limit=" << data_limit << logger::endl;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
550

551
#else
552
  (void)iipt;
553
#endif
554

555
556
557
558
559
560
561
  // copy data for this segment
  uint32_t off = 0;
  for (dcnt = d_skip; dcnt < n && off < data_limit; dcnt++) {
    tx_desc_ctx *rd = ready_segments.at(dcnt);
    d1 = rd->d->cmd_type_offset_bsz;
    uint16_t pkt_len =
        (d1 & I40E_TXD_QW1_TX_BUF_SZ_MASK) >> I40E_TXD_QW1_TX_BUF_SZ_SHIFT;
562

563
564
565
566
567
    if (off <= tso_off && off + pkt_len > tso_off) {
      uint32_t start = tso_off;
      uint32_t end = off + pkt_len;
      if (end > data_limit)
        end = data_limit;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
568
569

#ifdef DEBUG_LAN
570
571
572
      log << "    copying data from off=" << off << " idx=" << rd->index
          << " start=" << start << " end=" << end << " tso_len=" << tso_len
          << logger::endl;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
573
574
#endif

575
576
577
578
      memcpy(pktbuf + tso_len, (uint8_t *)rd->data + (start - off),
             end - start);
      tso_off = end;
      tso_len += end - start;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
579
580
    }

581
582
583
584
    off += pkt_len;
  }

  assert(tso_len <= MTU);
Antoine Kaufmann's avatar
Antoine Kaufmann committed
585

586
  if (!tso) {
Antoine Kaufmann's avatar
Antoine Kaufmann committed
587
#ifdef DEBUG_LAN
588
    log << "    normal non-tso packet" << logger::endl;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
589
590
#endif

591
592
593
    if (l4t == I40E_TX_DESC_CMD_L4T_EOFT_TCP) {
      uint16_t tcp_off = maclen + iplen;
      xsum_tcp(pktbuf + tcp_off, tso_len - tcp_off);
594
595
596
    } else if (l4t == I40E_TX_DESC_CMD_L4T_EOFT_UDP) {
      uint16_t udp_off = maclen + iplen;
      xsum_udp(pktbuf + udp_off, tso_len - udp_off);
597
    }
Antoine Kaufmann's avatar
Antoine Kaufmann committed
598

599
    runner->EthSend(pktbuf, tso_len);
600
  } else {
Antoine Kaufmann's avatar
Antoine Kaufmann committed
601
#ifdef DEBUG_LAN
602
603
    log << "    tso packet off=" << tso_off << " len=" << tso_len
        << logger::endl;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
604
605
#endif

606
607
    // TSO gets hairier
    uint16_t hdrlen = maclen + iplen + l4len;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
608

609
610
611
612
    // calculate payload size
    tso_paylen = tso_len - hdrlen;
    if (tso_paylen > tso_mss)
      tso_paylen = tso_mss;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
613

614
    xsum_tcpip_tso(pktbuf + maclen, iplen, l4len, tso_paylen);
Antoine Kaufmann's avatar
Antoine Kaufmann committed
615

616
    runner->EthSend(pktbuf, tso_len);
Antoine Kaufmann's avatar
Antoine Kaufmann committed
617

618
    tso_postupdate_header(pktbuf + maclen, iplen, l4len, tso_paylen);
Antoine Kaufmann's avatar
Antoine Kaufmann committed
619

620
621
622
623
    // not done yet with this TSO unit
    if (tso && tso_off < total_len) {
      tso_len = hdrlen;
      return true;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
624
    }
625
  }
Antoine Kaufmann's avatar
Antoine Kaufmann committed
626
627

#ifdef DEBUG_LAN
628
  log << "    unit done" << logger::endl;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
629
#endif
630
631
632
633
  while (dcnt-- > 0) {
    ready_segments.front()->processed();
    ready_segments.pop_front();
  }
634

635
636
  tso_len = 0;
  tso_off = 0;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
637

638
  return true;
639
640
}

641
642
643
void lan_queue_tx::trigger_tx() {
  while (trigger_tx_packet()) {
  }
644
645
646
}

lan_queue_tx::tx_desc_ctx::tx_desc_ctx(lan_queue_tx &queue_)
647
648
    : desc_ctx(queue_), tq(queue_) {
  d = reinterpret_cast<struct i40e_tx_desc *>(desc);
649
650
}

651
652
void lan_queue_tx::tx_desc_ctx::prepare() {
  uint64_t d1 = d->cmd_type_offset_bsz;
653

654
#ifdef DEBUG_LAN
655
  queue.log << " desc fetched didx=" << index << " d1=" << d1 << logger::endl;
656
#endif
657

658
659
660
661
  uint8_t dtype = (d1 & I40E_TXD_QW1_DTYPE_MASK) >> I40E_TXD_QW1_DTYPE_SHIFT;
  if (dtype == I40E_TX_DESC_DTYPE_DATA) {
    uint16_t len =
        (d1 & I40E_TXD_QW1_TX_BUF_SZ_MASK) >> I40E_TXD_QW1_TX_BUF_SZ_SHIFT;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
662

663
#ifdef DEBUG_LAN
664
665
    queue.log << "  bufaddr=" << d->buffer_addr << " len=" << len
              << logger::endl;
666
#endif
Antoine Kaufmann's avatar
Antoine Kaufmann committed
667

668
669
    data_fetch(d->buffer_addr, len);
  } else if (dtype == I40E_TX_DESC_DTYPE_CONTEXT) {
Antoine Kaufmann's avatar
Antoine Kaufmann committed
670
#ifdef DEBUG_LAN
671
672
673
674
675
    struct i40e_tx_context_desc *ctxd =
        reinterpret_cast<struct i40e_tx_context_desc *>(d);
    queue.log << "  context descriptor: tp=" << ctxd->tunneling_params
              << " l2t=" << ctxd->l2tag2 << " tctm=" << ctxd->type_cmd_tso_mss
              << logger::endl;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
676
#endif
Antoine Kaufmann's avatar
Antoine Kaufmann committed
677

678
679
680
681
682
    prepared();
  } else {
    queue.log << "txq: only support context & data descriptors" << logger::endl;
    abort();
  }
683
684
}

685
686
687
void lan_queue_tx::tx_desc_ctx::process() {
  tq.ready_segments.push_back(this);
  tq.trigger_tx();
688
689
}

690
691
692
693
void lan_queue_tx::tx_desc_ctx::processed() {
  d->cmd_type_offset_bsz = I40E_TX_DESC_DTYPE_DESC_DONE
                           << I40E_TXD_QW1_DTYPE_SHIFT;
  desc_ctx::processed();
694
695
}

696
lan_queue_tx::dma_hwb::dma_hwb(lan_queue_tx &queue_, uint32_t pos_,
697
698
                               uint32_t cnt_, uint32_t nh_)
    : queue(queue_), pos(pos_), cnt(cnt_), next_head(nh_) {
699
700
701
  data_ = &next_head;
  len_ = 4;
  write_ = true;
702
703
}

704
lan_queue_tx::dma_hwb::~dma_hwb() {
705
706
}

707
void lan_queue_tx::dma_hwb::done() {
708
#ifdef DEBUG_LAN
709
  queue.log << " tx head written back" << logger::endl;
710
#endif
711
712
713
  queue.writeback_done(pos, cnt);
  queue.trigger();
  delete this;
714
}
715
}  // namespace i40e