i40e_queues.cc 10.8 KB
Newer Older
Antoine Kaufmann's avatar
Antoine Kaufmann committed
1
2
3
4
#include <stdlib.h>
#include <string.h>
#include <cassert>
#include <iostream>
5
#include <algorithm>
Antoine Kaufmann's avatar
Antoine Kaufmann committed
6
7
8
9
10
11
12
13
14

#include "i40e_bm.h"

#include "i40e_base_wrapper.h"

using namespace i40e;

extern nicbm::Runner *runner;

15
16
queue_base::queue_base(const std::string &qname_, uint32_t &reg_head_,
        uint32_t &reg_tail_)
17
    : qname(qname_), log(qname_), active_first_pos(0), active_first_idx(0), active_cnt(0),
18
19
    base(0), len(0), reg_head(reg_head_), reg_tail(reg_tail_),
    enabled(false), desc_len(0)
Antoine Kaufmann's avatar
Antoine Kaufmann committed
20
{
21
22
23
24
25
26
27
28
29
30
    for (size_t i = 0; i < MAX_ACTIVE_DESCS; i++) {
        desc_ctxs[i] = nullptr;
    }
}

void queue_base::ctxs_init()
{
    for (size_t i = 0; i < MAX_ACTIVE_DESCS; i++) {
        desc_ctxs[i] = &desc_ctx_create();
    }
Antoine Kaufmann's avatar
Antoine Kaufmann committed
31
32
33
34
}

void queue_base::trigger_fetch()
{
35
    if (!enabled)
Antoine Kaufmann's avatar
Antoine Kaufmann committed
36
37
        return;

38
39
40
41
42
43
44
45
    // calculate how many we can fetch
    uint32_t next_idx = (active_first_idx + active_cnt) % len;
    uint32_t desc_avail = (reg_tail - next_idx) % len;
    uint32_t fetch_cnt = desc_avail;
    fetch_cnt = std::min(fetch_cnt, MAX_ACTIVE_DESCS - active_cnt);
    if (max_active_capacity() <= active_cnt)
        fetch_cnt = std::min(fetch_cnt, max_active_capacity() - active_cnt);
    fetch_cnt = std::min(fetch_cnt, max_fetch_capacity());
Antoine Kaufmann's avatar
Antoine Kaufmann committed
46

47
48
    if (next_idx + fetch_cnt > len)
        fetch_cnt = len - next_idx;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
49

50
#ifdef DEBUG_QUEUES
51
52
    log << "fetching avail=" << desc_avail << " cnt=" << fetch_cnt << " idx=" <<
        next_idx << logger::endl;
53
#endif
54

55
56
57
58
59
60
61
62
63
64
65
66
67
68
    // abort if nothign to fetch
    if (fetch_cnt == 0)
        return;

    // mark descriptor contexts as fetching
    uint32_t first_pos = (active_first_pos + active_cnt) % MAX_ACTIVE_DESCS;
    for (uint32_t i = 0; i < fetch_cnt; i++) {
        desc_ctx &ctx = *desc_ctxs[(first_pos + i) % MAX_ACTIVE_DESCS];
        assert(ctx.state == desc_ctx::DESC_EMPTY);

        ctx.state = desc_ctx::DESC_FETCHING;
        ctx.index = (next_idx + i) % len;
    }
    active_cnt += fetch_cnt;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
69

70
71
72
73
74
    // prepare & issue dma
    dma_fetch *dma = new dma_fetch(*this, desc_len * fetch_cnt);
    dma->write = false;
    dma->dma_addr = base + next_idx * desc_len;
    dma->pos = first_pos;
75
#ifdef DEBUG_QUEUES
76
    log << "    dma = " << dma << logger::endl;
77
#endif
Antoine Kaufmann's avatar
Antoine Kaufmann committed
78
79
80
    runner->issue_dma(*dma);
}

81
void queue_base::trigger_process()
82
{
83
84
    if (!enabled)
        return;
85

86
87
88
89
    // first skip over descriptors that are already done processing
    uint32_t i;
    for (i = 0; i < active_cnt; i++)
        if (desc_ctxs[(active_first_pos + i) % MAX_ACTIVE_DESCS]->state
90
                <= desc_ctx::DESC_PREPARED)
91
92
93
94
95
96
97
98
99
100
101
            break;

    // then run all prepared contexts
    uint32_t j;
    for (j = 0; i + j < active_cnt; j++) {
        desc_ctx &ctx = *desc_ctxs[(active_first_pos + i + j)
            % MAX_ACTIVE_DESCS];
        if (ctx.state != desc_ctx::DESC_PREPARED)
            break;

        ctx.state = desc_ctx::DESC_PROCESSING;
102
#ifdef DEBUG_QUEUES
103
        log << "processing desc " << ctx.index << logger::endl;
104
#endif
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
        ctx.process();
    }
}

void queue_base::trigger_writeback()
{
    if (!enabled)
        return;

    // from first pos count number of processed descriptors
    uint32_t avail;
    for (avail = 0; avail < active_cnt; avail++)
        if (desc_ctxs[(active_first_pos + avail) % MAX_ACTIVE_DESCS]->state
                != desc_ctx::DESC_PROCESSED)
            break;

    uint32_t cnt = std::min(avail, max_writeback_capacity());
122
123
    if (active_first_idx + cnt > len)
        cnt = len - active_first_idx;
124

125
#ifdef DEBUG_QUEUES
126
127
    log << "writing back avail=" << avail << " cnt=" << cnt <<
        " idx=" << active_first_idx << logger::endl;
128
#endif
129
130
131
132
133
134
135
136
137
138
139

    if (cnt == 0)
        return;

    // mark these descriptors as writing back
    for (uint32_t i = 0; i < cnt; i++) {
        desc_ctx &ctx = *desc_ctxs[(active_first_pos + i) % MAX_ACTIVE_DESCS];
        ctx.state = desc_ctx::DESC_WRITING_BACK;
    }

    do_writeback(active_first_idx, active_first_pos, cnt);
140
141
}

142
143
144
145
146
147
148
void queue_base::trigger()
{
  trigger_fetch();
  trigger_process();
  trigger_writeback();
}

149
150
void queue_base::reset()
{
151
#ifdef DEBUG_QUEUES
152
    log << "reset" << logger::endl;
153
154
#endif

155
    enabled = false;
156
157
158
159
160
161
162
    active_first_pos = 0;
    active_first_idx = 0;
    active_cnt = 0;

    for (size_t i = 0; i < MAX_ACTIVE_DESCS; i++) {
        desc_ctxs[i]->state = desc_ctx::DESC_EMPTY;
    }
163
164
}

Antoine Kaufmann's avatar
Antoine Kaufmann committed
165
166
167
168
169
void queue_base::reg_updated()
{
    if (!enabled)
        return;

170
    trigger();
Antoine Kaufmann's avatar
Antoine Kaufmann committed
171
172
}

173
174
175
176
177
bool queue_base::is_enabled()
{
    return enabled;
}

178
179
180
181
182
183
uint32_t queue_base::max_fetch_capacity()
{
    return UINT32_MAX;
}

uint32_t queue_base::max_active_capacity()
Antoine Kaufmann's avatar
Antoine Kaufmann committed
184
{
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
    return UINT32_MAX;
}

uint32_t queue_base::max_writeback_capacity()
{
    return UINT32_MAX;
}

void queue_base::interrupt()
{
}

void queue_base::do_writeback(uint32_t first_idx, uint32_t first_pos,
        uint32_t cnt)
{
    dma_wb *dma = new dma_wb(*this, desc_len * cnt);
Antoine Kaufmann's avatar
Antoine Kaufmann committed
201
    dma->write = true;
202
203
204
205
206
207
208
209
210
    dma->dma_addr = base + first_idx * desc_len;
    dma->pos = first_pos;

    uint8_t *buf = reinterpret_cast<uint8_t *> (dma->data);
    for (uint32_t i = 0; i < cnt; i++) {
        desc_ctx &ctx = *desc_ctxs[(first_pos + i) % MAX_ACTIVE_DESCS];
        assert(ctx.state == desc_ctx::DESC_WRITING_BACK);
        memcpy(buf + i * desc_len, ctx.desc, desc_len);
    }
Antoine Kaufmann's avatar
Antoine Kaufmann committed
211
212
213
214

    runner->issue_dma(*dma);
}

215
void queue_base::writeback_done(uint32_t first_pos, uint32_t cnt)
216
{
217
218
    if (!enabled)
        return;
219

220
221
222
223
224
225
226
    // first mark descriptors as written back
    for (uint32_t i = 0; i < cnt; i++) {
        desc_ctx &ctx = *desc_ctxs[(first_pos + i) % MAX_ACTIVE_DESCS];
        assert(ctx.state == desc_ctx::DESC_WRITING_BACK);
        ctx.state = desc_ctx::DESC_WRITTEN_BACK;
    }

227
#ifdef DEBUG_QUEUES
228
229
230
    log << "written back afi=" << active_first_idx << " afp=" <<
        active_first_pos << " acnt=" << active_cnt << " pos=" <<
        first_pos << " cnt=" << cnt << logger::endl;
231
#endif
232
233
234
235
236
237
238
239
240
241
242
243

    // then start at the beginning and check how many are written back and then
    // free those
    uint32_t bump_cnt = 0;
    for (bump_cnt = 0; bump_cnt < active_cnt; bump_cnt++) {
        desc_ctx &ctx = *desc_ctxs[(active_first_pos + bump_cnt) %
            MAX_ACTIVE_DESCS];
        if (ctx.state != desc_ctx::DESC_WRITTEN_BACK)
            break;

        ctx.state = desc_ctx::DESC_EMPTY;
    }
244
#ifdef DEBUG_QUEUES
245
    log << "   bump_cnt=" << bump_cnt << logger::endl;
246
#endif
247
248
249
250
251
252
253
254

    active_first_pos = (active_first_pos + bump_cnt) % MAX_ACTIVE_DESCS;
    active_first_idx = (active_first_idx + bump_cnt) % len;
    active_cnt -= bump_cnt;

    reg_head = active_first_idx;
    interrupt();
}
255

256
257
258
259
260
queue_base::desc_ctx::desc_ctx(queue_base &queue_)
    : queue(queue_), state(DESC_EMPTY), index(0), data(nullptr), data_len(0),
    data_capacity(0)
{
    desc = new uint8_t[queue_.desc_len];
261
262
}

263
queue_base::desc_ctx::~desc_ctx()
264
{
265
266
267
    delete[] ((uint8_t *) desc);
    if (data_capacity > 0)
        delete[] ((uint8_t *) data);
268
269
}

270
void queue_base::desc_ctx::prepare()
271
{
272
    prepared();
273
274
}

275
void queue_base::desc_ctx::prepared()
276
{
277
#ifdef DEBUG_QUEUES
278
    queue.log << "prepared desc " << index << logger::endl;
279
#endif
280
281
    assert(state == DESC_PREPARING);
    state = DESC_PREPARED;
282
283
}

284
void queue_base::desc_ctx::processed()
Antoine Kaufmann's avatar
Antoine Kaufmann committed
285
{
286
#ifdef DEBUG_QUEUES
287
    queue.log << "processed desc " << index << logger::endl;
288
#endif
289
290
291
292
    assert(state == DESC_PROCESSING);
    state = DESC_PROCESSED;
}

293
294
#define MAX_DMA_SIZE ((size_t) 0x1000)

295
296
297
void queue_base::desc_ctx::data_fetch(uint64_t addr, size_t data_len)
{
    if (data_capacity < data_len) {
298
#ifdef DEBUG_QUEUES
299
        queue.log << "data_fetch allocating" << logger::endl;
300
#endif
301
302
303
304
305
306
307
        if (data_capacity != 0)
            delete[] ((uint8_t *) data);

        data = new uint8_t[data_len];
        data_capacity = data_len;
    }

308
309
310
311
    dma_data_fetch *dma = new dma_data_fetch(*this, std::min(data_len,
                MAX_DMA_SIZE), data);
    dma->part_offset = 0;
    dma->total_len = data_len;
312
313
314
    dma->write = false;
    dma->dma_addr = addr;

315
#ifdef DEBUG_QUEUES
316
317
318
    queue.log << "fetching data idx=" << index << " addr=" << addr << " len=" <<
        data_len << logger::endl;
    queue.log << "  dma = " << dma << " data=" << data << logger::endl;
319
#endif
320
    runner->issue_dma(*dma);
321

322
323
324
325
326
327
328
329
330
331
}

void queue_base::desc_ctx::data_fetched(uint64_t addr, size_t len)
{
    prepared();
}

void queue_base::desc_ctx::data_write(uint64_t addr, size_t data_len,
        const void *buf)
{
332
#ifdef DEBUG_QUEUES
333
334
    queue.log << "data_write(addr=" << addr << " datalen=" << data_len << ")" <<
        logger::endl;
335
#endif
336
337
338
339
340
341
342
343
344
345
    dma_data_wb *data_dma = new dma_data_wb(*this, data_len);
    data_dma->write = true;
    data_dma->dma_addr = addr;
    memcpy(data_dma->data, buf, data_len);

    runner->issue_dma(*data_dma);
}

void queue_base::desc_ctx::data_written(uint64_t addr, size_t len)
{
346
#ifdef DEBUG_QUEUES
347
348
    queue.log << "data_written(addr=" << addr << " datalen=" << len << ")" <<
        logger::endl;
349
#endif
350
    processed();
Antoine Kaufmann's avatar
Antoine Kaufmann committed
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
}

queue_base::dma_fetch::dma_fetch(queue_base &queue_, size_t len_)
    : queue(queue_)
{
    data = new char[len_];
    len = len_;
}

queue_base::dma_fetch::~dma_fetch()
{
    delete[] ((char *) data);
}

void queue_base::dma_fetch::done()
{
367
368
369
370
371
    uint8_t *buf = reinterpret_cast <uint8_t *> (data);
    for (uint32_t i = 0; i < len / queue.desc_len; i++) {
        desc_ctx &ctx = *queue.desc_ctxs[(pos + i) % queue.MAX_ACTIVE_DESCS];
        memcpy(ctx.desc, buf + queue.desc_len * i, queue.desc_len);

372
#ifdef DEBUG_QUEUES
373
        queue.log << "preparing desc " << ctx.index << logger::endl;
374
#endif
375
376
377
        ctx.state = desc_ctx::DESC_PREPARING;
        ctx.prepare();
    }
378
    queue.trigger();
Antoine Kaufmann's avatar
Antoine Kaufmann committed
379
380
381
    delete this;
}

382
383
384
queue_base::dma_data_fetch::dma_data_fetch(desc_ctx &ctx_, size_t len_,
        void *buffer)
    : ctx(ctx_)
385
{
386
    data = buffer;
387
388
389
390
391
392
393
394
395
    len = len_;
}

queue_base::dma_data_fetch::~dma_data_fetch()
{
}

void queue_base::dma_data_fetch::done()
{
396
397
398
399
400
401
402
403
404
405
406
407
408
    part_offset += len;
    dma_addr += len;
    data = (uint8_t *) data + len;

    if (part_offset < total_len) {
#ifdef DEBUG_QUEUES
        ctx.queue.log << "  dma_fetch: next part of multi part dma" << logger::endl;
#endif
        len = std::min(total_len - part_offset, MAX_DMA_SIZE);
        runner->issue_dma(*this);
        return;
    }
    ctx.data_fetched(dma_addr - part_offset, total_len);
409
    ctx.queue.trigger();
410
411
    delete this;
}
Antoine Kaufmann's avatar
Antoine Kaufmann committed
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426

queue_base::dma_wb::dma_wb(queue_base &queue_, size_t len_)
    : queue(queue_)
{
    data = new char[len_];
    len = len_;
}

queue_base::dma_wb::~dma_wb()
{
    delete[] ((char *) data);
}

void queue_base::dma_wb::done()
{
427
    queue.writeback_done(pos, len / queue.desc_len);
428
    queue.trigger();
Antoine Kaufmann's avatar
Antoine Kaufmann committed
429
430
431
432
    delete this;
}


433
434
queue_base::dma_data_wb::dma_data_wb(desc_ctx &ctx_, size_t len_)
    : ctx(ctx_)
435
436
437
438
439
440
441
442
443
444
445
446
{
    data = new char[len_];
    len = len_;
}

queue_base::dma_data_wb::~dma_data_wb()
{
    delete[] ((char *) data);
}

void queue_base::dma_data_wb::done()
{
447
    ctx.data_written(dma_addr, len);
448
    ctx.queue.trigger();
449
450
    delete this;
}