i40e_queues.cc 11.8 KB
Newer Older
Antoine Kaufmann's avatar
Antoine Kaufmann committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
/*
 * Copyright 2021 Max Planck Institute for Software Systems, and
 * National University of Singapore
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

Antoine Kaufmann's avatar
Antoine Kaufmann committed
25
26
#include <stdlib.h>
#include <string.h>
27
28

#include <algorithm>
Antoine Kaufmann's avatar
Antoine Kaufmann committed
29
30
31
#include <cassert>
#include <iostream>

Antoine Kaufmann's avatar
Antoine Kaufmann committed
32
#include "sims/nic/i40e_bm/i40e_base_wrapper.h"
33
#include "sims/nic/i40e_bm/i40e_bm.h"
Antoine Kaufmann's avatar
Antoine Kaufmann committed
34

35
namespace i40e {
Antoine Kaufmann's avatar
Antoine Kaufmann committed
36

37
queue_base::queue_base(const std::string &qname_, uint32_t &reg_head_,
38
                       uint32_t &reg_tail_, i40e_bm &dev_)
39
    : qname(qname_),
40
41
      log(qname_, dev_.runner_),
      dev(dev_),
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
      active_first_pos(0),
      active_first_idx(0),
      active_cnt(0),
      base(0),
      len(0),
      reg_head(reg_head_),
      reg_tail(reg_tail_),
      enabled(false),
      desc_len(0) {
  for (size_t i = 0; i < MAX_ACTIVE_DESCS; i++) {
    desc_ctxs[i] = nullptr;
  }
}

void queue_base::ctxs_init() {
  for (size_t i = 0; i < MAX_ACTIVE_DESCS; i++) {
    desc_ctxs[i] = &desc_ctx_create();
  }
}

void queue_base::trigger_fetch() {
  if (!enabled)
    return;

  // calculate how many we can fetch
  uint32_t next_idx = (active_first_idx + active_cnt) % len;
  uint32_t desc_avail = (reg_tail - next_idx) % len;
  uint32_t fetch_cnt = desc_avail;
  fetch_cnt = std::min(fetch_cnt, MAX_ACTIVE_DESCS - active_cnt);
  if (max_active_capacity() <= active_cnt)
    fetch_cnt = std::min(fetch_cnt, max_active_capacity() - active_cnt);
  fetch_cnt = std::min(fetch_cnt, max_fetch_capacity());

  if (next_idx + fetch_cnt > len)
    fetch_cnt = len - next_idx;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
77

78
#ifdef DEBUG_QUEUES
79
80
  log << "fetching avail=" << desc_avail << " cnt=" << fetch_cnt
      << " idx=" << next_idx << logger::endl;
81
#endif
82

83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
  // abort if nothign to fetch
  if (fetch_cnt == 0)
    return;

  // mark descriptor contexts as fetching
  uint32_t first_pos = (active_first_pos + active_cnt) % MAX_ACTIVE_DESCS;
  for (uint32_t i = 0; i < fetch_cnt; i++) {
    desc_ctx &ctx = *desc_ctxs[(first_pos + i) % MAX_ACTIVE_DESCS];
    assert(ctx.state == desc_ctx::DESC_EMPTY);

    ctx.state = desc_ctx::DESC_FETCHING;
    ctx.index = (next_idx + i) % len;
  }
  active_cnt += fetch_cnt;

  // prepare & issue dma
  dma_fetch *dma = new dma_fetch(*this, desc_len * fetch_cnt);
100
101
  dma->write_ = false;
  dma->dma_addr_ = base + next_idx * desc_len;
102
  dma->pos = first_pos;
103
#ifdef DEBUG_QUEUES
104
  log << "    dma = " << dma << logger::endl;
105
#endif
106
  dev.runner_->IssueDma(*dma);
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
}

void queue_base::trigger_process() {
  if (!enabled)
    return;

  // first skip over descriptors that are already done processing
  uint32_t i;
  for (i = 0; i < active_cnt; i++)
    if (desc_ctxs[(active_first_pos + i) % MAX_ACTIVE_DESCS]->state <=
        desc_ctx::DESC_PREPARED)
      break;

  // then run all prepared contexts
  uint32_t j;
  for (j = 0; i + j < active_cnt; j++) {
    desc_ctx &ctx = *desc_ctxs[(active_first_pos + i + j) % MAX_ACTIVE_DESCS];
    if (ctx.state != desc_ctx::DESC_PREPARED)
      break;

    ctx.state = desc_ctx::DESC_PROCESSING;
128
#ifdef DEBUG_QUEUES
129
    log << "processing desc " << ctx.index << logger::endl;
130
#endif
131
132
    ctx.process();
  }
133
134
}

135
136
137
void queue_base::trigger_writeback() {
  if (!enabled)
    return;
138

139
140
141
142
143
144
  // from first pos count number of processed descriptors
  uint32_t avail;
  for (avail = 0; avail < active_cnt; avail++)
    if (desc_ctxs[(active_first_pos + avail) % MAX_ACTIVE_DESCS]->state !=
        desc_ctx::DESC_PROCESSED)
      break;
145

146
147
148
  uint32_t cnt = std::min(avail, max_writeback_capacity());
  if (active_first_idx + cnt > len)
    cnt = len - active_first_idx;
149

150
#ifdef DEBUG_QUEUES
151
152
  log << "writing back avail=" << avail << " cnt=" << cnt
      << " idx=" << active_first_idx << logger::endl;
153
#endif
154

155
156
  if (cnt == 0)
    return;
157

158
159
160
161
162
  // mark these descriptors as writing back
  for (uint32_t i = 0; i < cnt; i++) {
    desc_ctx &ctx = *desc_ctxs[(active_first_pos + i) % MAX_ACTIVE_DESCS];
    ctx.state = desc_ctx::DESC_WRITING_BACK;
  }
163

164
  do_writeback(active_first_idx, active_first_pos, cnt);
165
166
}

167
void queue_base::trigger() {
168
169
170
171
172
  trigger_fetch();
  trigger_process();
  trigger_writeback();
}

173
void queue_base::reset() {
174
#ifdef DEBUG_QUEUES
175
  log << "reset" << logger::endl;
176
177
#endif

178
179
180
181
  enabled = false;
  active_first_pos = 0;
  active_first_idx = 0;
  active_cnt = 0;
182

183
184
185
  for (size_t i = 0; i < MAX_ACTIVE_DESCS; i++) {
    desc_ctxs[i]->state = desc_ctx::DESC_EMPTY;
  }
186
187
}

188
void queue_base::reg_updated() {
189
#ifdef DEBUG_QUEUES
190
191
  log << "reg_updated: tail=" << reg_tail << " enabled=" << (int)enabled
      << logger::endl;
192
#endif
193
194
  if (!enabled)
    return;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
195

196
  trigger();
Antoine Kaufmann's avatar
Antoine Kaufmann committed
197
198
}

199
200
bool queue_base::is_enabled() {
  return enabled;
201
202
}

203
204
uint32_t queue_base::max_fetch_capacity() {
  return UINT32_MAX;
205
206
}

207
208
uint32_t queue_base::max_active_capacity() {
  return UINT32_MAX;
209
210
}

211
212
uint32_t queue_base::max_writeback_capacity() {
  return UINT32_MAX;
213
214
}

215
void queue_base::interrupt() {
216
217
218
}

void queue_base::do_writeback(uint32_t first_idx, uint32_t first_pos,
219
220
                              uint32_t cnt) {
  dma_wb *dma = new dma_wb(*this, desc_len * cnt);
221
222
  dma->write_ = true;
  dma->dma_addr_ = base + first_idx * desc_len;
223
224
  dma->pos = first_pos;

225
  uint8_t *buf = reinterpret_cast<uint8_t *>(dma->data_);
226
227
228
229
230
231
  for (uint32_t i = 0; i < cnt; i++) {
    desc_ctx &ctx = *desc_ctxs[(first_pos + i) % MAX_ACTIVE_DESCS];
    assert(ctx.state == desc_ctx::DESC_WRITING_BACK);
    memcpy(buf + i * desc_len, ctx.desc, desc_len);
  }

232
  dev.runner_->IssueDma(*dma);
233
234
235
236
237
238
239
240
241
242
243
244
}

void queue_base::writeback_done(uint32_t first_pos, uint32_t cnt) {
  if (!enabled)
    return;

  // first mark descriptors as written back
  for (uint32_t i = 0; i < cnt; i++) {
    desc_ctx &ctx = *desc_ctxs[(first_pos + i) % MAX_ACTIVE_DESCS];
    assert(ctx.state == desc_ctx::DESC_WRITING_BACK);
    ctx.state = desc_ctx::DESC_WRITTEN_BACK;
  }
245

246
#ifdef DEBUG_QUEUES
247
248
249
  log << "written back afi=" << active_first_idx << " afp=" << active_first_pos
      << " acnt=" << active_cnt << " pos=" << first_pos << " cnt=" << cnt
      << logger::endl;
250
#endif
251

252
253
254
255
256
257
258
259
260
261
262
  // then start at the beginning and check how many are written back and then
  // free those
  uint32_t bump_cnt = 0;
  for (bump_cnt = 0; bump_cnt < active_cnt; bump_cnt++) {
    desc_ctx &ctx =
        *desc_ctxs[(active_first_pos + bump_cnt) % MAX_ACTIVE_DESCS];
    if (ctx.state != desc_ctx::DESC_WRITTEN_BACK)
      break;

    ctx.state = desc_ctx::DESC_EMPTY;
  }
263
#ifdef DEBUG_QUEUES
264
  log << "   bump_cnt=" << bump_cnt << logger::endl;
265
#endif
266

267
268
269
  active_first_pos = (active_first_pos + bump_cnt) % MAX_ACTIVE_DESCS;
  active_first_idx = (active_first_idx + bump_cnt) % len;
  active_cnt -= bump_cnt;
270

271
272
  reg_head = active_first_idx;
  interrupt();
273
}
274

275
queue_base::desc_ctx::desc_ctx(queue_base &queue_)
276
277
278
279
280
281
282
    : queue(queue_),
      state(DESC_EMPTY),
      index(0),
      data(nullptr),
      data_len(0),
      data_capacity(0) {
  desc = new uint8_t[queue_.desc_len];
283
284
}

285
286
287
288
queue_base::desc_ctx::~desc_ctx() {
  delete[]((uint8_t *)desc);
  if (data_capacity > 0)
    delete[]((uint8_t *)data);
289
290
}

291
292
void queue_base::desc_ctx::prepare() {
  prepared();
293
294
}

295
void queue_base::desc_ctx::prepared() {
296
#ifdef DEBUG_QUEUES
297
  queue.log << "prepared desc " << index << logger::endl;
298
#endif
299
300
  assert(state == DESC_PREPARING);
  state = DESC_PREPARED;
301
302
}

303
void queue_base::desc_ctx::processed() {
304
#ifdef DEBUG_QUEUES
305
  queue.log << "processed desc " << index << logger::endl;
306
#endif
307
308
  assert(state == DESC_PROCESSING);
  state = DESC_PROCESSED;
309
310
}

311
#define MAX_DMA_SIZE ((size_t)9024)
312

313
314
void queue_base::desc_ctx::data_fetch(uint64_t addr, size_t data_len) {
  if (data_capacity < data_len) {
315
#ifdef DEBUG_QUEUES
316
    queue.log << "data_fetch allocating" << logger::endl;
317
#endif
318
319
    if (data_capacity != 0)
      delete[]((uint8_t *)data);
320

321
322
323
    data = new uint8_t[data_len];
    data_capacity = data_len;
  }
324

325
326
327
328
  dma_data_fetch *dma =
      new dma_data_fetch(*this, std::min(data_len, MAX_DMA_SIZE), data);
  dma->part_offset = 0;
  dma->total_len = data_len;
329
330
  dma->write_ = false;
  dma->dma_addr_ = addr;
331

332
#ifdef DEBUG_QUEUES
333
334
335
  queue.log << "fetching data idx=" << index << " addr=" << addr
            << " len=" << data_len << logger::endl;
  queue.log << "  dma = " << dma << " data=" << data << logger::endl;
336
#endif
337
  queue.dev.runner_->IssueDma(*dma);
338
339
}

340
341
void queue_base::desc_ctx::data_fetched(uint64_t addr, size_t len) {
  prepared();
342
343
344
}

void queue_base::desc_ctx::data_write(uint64_t addr, size_t data_len,
345
                                      const void *buf) {
346
#ifdef DEBUG_QUEUES
347
348
  queue.log << "data_write(addr=" << addr << " datalen=" << data_len << ")"
            << logger::endl;
349
#endif
350
  dma_data_wb *data_dma = new dma_data_wb(*this, data_len);
351
352
353
  data_dma->write_ = true;
  data_dma->dma_addr_ = addr;
  memcpy(data_dma->data_, buf, data_len);
354

355
  queue.dev.runner_->IssueDma(*data_dma);
356
357
}

358
void queue_base::desc_ctx::data_written(uint64_t addr, size_t len) {
359
#ifdef DEBUG_QUEUES
360
361
  queue.log << "data_written(addr=" << addr << " datalen=" << len << ")"
            << logger::endl;
362
#endif
363
  processed();
Antoine Kaufmann's avatar
Antoine Kaufmann committed
364
365
}

366
queue_base::dma_fetch::dma_fetch(queue_base &queue_, size_t len)
367
    : queue(queue_) {
368
369
  data_ = new char[len];
  len_ = len;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
370
371
}

372
queue_base::dma_fetch::~dma_fetch() {
373
  delete[]((char *)data_);
Antoine Kaufmann's avatar
Antoine Kaufmann committed
374
375
}

376
void queue_base::dma_fetch::done() {
377
378
  uint8_t *buf = reinterpret_cast<uint8_t *>(data_);
  for (uint32_t i = 0; i < len_ / queue.desc_len; i++) {
379
380
    desc_ctx &ctx = *queue.desc_ctxs[(pos + i) % queue.MAX_ACTIVE_DESCS];
    memcpy(ctx.desc, buf + queue.desc_len * i, queue.desc_len);
381

382
#ifdef DEBUG_QUEUES
383
    queue.log << "preparing desc " << ctx.index << logger::endl;
384
#endif
385
386
387
388
389
    ctx.state = desc_ctx::DESC_PREPARING;
    ctx.prepare();
  }
  queue.trigger();
  delete this;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
390
391
}

392
queue_base::dma_data_fetch::dma_data_fetch(desc_ctx &ctx_, size_t len,
393
394
                                           void *buffer)
    : ctx(ctx_) {
395
396
  data_ = buffer;
  len_ = len;
397
398
}

399
queue_base::dma_data_fetch::~dma_data_fetch() {
400
401
}

402
void queue_base::dma_data_fetch::done() {
403
404
405
  part_offset += len_;
  dma_addr_ += len_;
  data_ = (uint8_t *)data_ + len_;
406

407
  if (part_offset < total_len) {
408
#ifdef DEBUG_QUEUES
409
    ctx.queue.log << "  dma_fetch: next part of multi part dma" << logger::endl;
410
#endif
411
    len_ = std::min(total_len - part_offset, MAX_DMA_SIZE);
412
    ctx.queue.dev.runner_->IssueDma(*this);
413
414
    return;
  }
415
  ctx.data_fetched(dma_addr_ - part_offset, total_len);
416
417
  ctx.queue.trigger();
  delete this;
418
}
Antoine Kaufmann's avatar
Antoine Kaufmann committed
419

420
421
422
queue_base::dma_wb::dma_wb(queue_base &queue_, size_t len) : queue(queue_) {
  data_ = new char[len];
  len_ = len;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
423
424
}

425
queue_base::dma_wb::~dma_wb() {
426
  delete[]((char *)data_);
Antoine Kaufmann's avatar
Antoine Kaufmann committed
427
428
}

429
void queue_base::dma_wb::done() {
430
  queue.writeback_done(pos, len_ / queue.desc_len);
431
432
  queue.trigger();
  delete this;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
433
434
}

435
436
437
queue_base::dma_data_wb::dma_data_wb(desc_ctx &ctx_, size_t len) : ctx(ctx_) {
  data_ = new char[len];
  len_ = len;
438
439
}

440
queue_base::dma_data_wb::~dma_data_wb() {
441
  delete[]((char *)data_);
442
443
}

444
void queue_base::dma_data_wb::done() {
445
  ctx.data_written(dma_addr_, len_);
446
447
  ctx.queue.trigger();
  delete this;
448
}
449
}  // namespace i40e