"test/retiarii_test/cgo_mnasnet/test.py" did not exist on "6aaca5f7ad2f0f169cc9439407a2137c44a25de9"
i40e_queues.cc 12 KB
Newer Older
Antoine Kaufmann's avatar
Antoine Kaufmann committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
/*
 * Copyright 2021 Max Planck Institute for Software Systems, and
 * National University of Singapore
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

Antoine Kaufmann's avatar
Antoine Kaufmann committed
25
26
27
28
#include <stdlib.h>
#include <string.h>
#include <cassert>
#include <iostream>
29
#include <algorithm>
Antoine Kaufmann's avatar
Antoine Kaufmann committed
30
31
32
33
34
35
36
37
38

#include "i40e_bm.h"

#include "i40e_base_wrapper.h"

using namespace i40e;

extern nicbm::Runner *runner;

39
40
queue_base::queue_base(const std::string &qname_, uint32_t &reg_head_,
        uint32_t &reg_tail_)
41
    : qname(qname_), log(qname_), active_first_pos(0), active_first_idx(0), active_cnt(0),
42
43
    base(0), len(0), reg_head(reg_head_), reg_tail(reg_tail_),
    enabled(false), desc_len(0)
Antoine Kaufmann's avatar
Antoine Kaufmann committed
44
{
45
46
47
48
49
50
51
52
53
54
    for (size_t i = 0; i < MAX_ACTIVE_DESCS; i++) {
        desc_ctxs[i] = nullptr;
    }
}

void queue_base::ctxs_init()
{
    for (size_t i = 0; i < MAX_ACTIVE_DESCS; i++) {
        desc_ctxs[i] = &desc_ctx_create();
    }
Antoine Kaufmann's avatar
Antoine Kaufmann committed
55
56
57
58
}

void queue_base::trigger_fetch()
{
59
    if (!enabled)
Antoine Kaufmann's avatar
Antoine Kaufmann committed
60
61
        return;

62
63
64
65
66
67
68
69
    // calculate how many we can fetch
    uint32_t next_idx = (active_first_idx + active_cnt) % len;
    uint32_t desc_avail = (reg_tail - next_idx) % len;
    uint32_t fetch_cnt = desc_avail;
    fetch_cnt = std::min(fetch_cnt, MAX_ACTIVE_DESCS - active_cnt);
    if (max_active_capacity() <= active_cnt)
        fetch_cnt = std::min(fetch_cnt, max_active_capacity() - active_cnt);
    fetch_cnt = std::min(fetch_cnt, max_fetch_capacity());
Antoine Kaufmann's avatar
Antoine Kaufmann committed
70

71
72
    if (next_idx + fetch_cnt > len)
        fetch_cnt = len - next_idx;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
73

74
#ifdef DEBUG_QUEUES
75
76
    log << "fetching avail=" << desc_avail << " cnt=" << fetch_cnt << " idx=" <<
        next_idx << logger::endl;
77
#endif
78

79
80
81
82
83
84
85
86
87
88
89
90
91
92
    // abort if nothign to fetch
    if (fetch_cnt == 0)
        return;

    // mark descriptor contexts as fetching
    uint32_t first_pos = (active_first_pos + active_cnt) % MAX_ACTIVE_DESCS;
    for (uint32_t i = 0; i < fetch_cnt; i++) {
        desc_ctx &ctx = *desc_ctxs[(first_pos + i) % MAX_ACTIVE_DESCS];
        assert(ctx.state == desc_ctx::DESC_EMPTY);

        ctx.state = desc_ctx::DESC_FETCHING;
        ctx.index = (next_idx + i) % len;
    }
    active_cnt += fetch_cnt;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
93

94
95
96
97
98
    // prepare & issue dma
    dma_fetch *dma = new dma_fetch(*this, desc_len * fetch_cnt);
    dma->write = false;
    dma->dma_addr = base + next_idx * desc_len;
    dma->pos = first_pos;
99
#ifdef DEBUG_QUEUES
100
    log << "    dma = " << dma << logger::endl;
101
#endif
Antoine Kaufmann's avatar
Antoine Kaufmann committed
102
103
104
    runner->issue_dma(*dma);
}

105
void queue_base::trigger_process()
106
{
107
108
    if (!enabled)
        return;
109

110
111
112
113
    // first skip over descriptors that are already done processing
    uint32_t i;
    for (i = 0; i < active_cnt; i++)
        if (desc_ctxs[(active_first_pos + i) % MAX_ACTIVE_DESCS]->state
114
                <= desc_ctx::DESC_PREPARED)
115
116
117
118
119
120
121
122
123
124
125
            break;

    // then run all prepared contexts
    uint32_t j;
    for (j = 0; i + j < active_cnt; j++) {
        desc_ctx &ctx = *desc_ctxs[(active_first_pos + i + j)
            % MAX_ACTIVE_DESCS];
        if (ctx.state != desc_ctx::DESC_PREPARED)
            break;

        ctx.state = desc_ctx::DESC_PROCESSING;
126
#ifdef DEBUG_QUEUES
127
        log << "processing desc " << ctx.index << logger::endl;
128
#endif
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
        ctx.process();
    }
}

void queue_base::trigger_writeback()
{
    if (!enabled)
        return;

    // from first pos count number of processed descriptors
    uint32_t avail;
    for (avail = 0; avail < active_cnt; avail++)
        if (desc_ctxs[(active_first_pos + avail) % MAX_ACTIVE_DESCS]->state
                != desc_ctx::DESC_PROCESSED)
            break;

    uint32_t cnt = std::min(avail, max_writeback_capacity());
146
147
    if (active_first_idx + cnt > len)
        cnt = len - active_first_idx;
148

149
#ifdef DEBUG_QUEUES
150
151
    log << "writing back avail=" << avail << " cnt=" << cnt <<
        " idx=" << active_first_idx << logger::endl;
152
#endif
153
154
155
156
157
158
159
160
161
162
163

    if (cnt == 0)
        return;

    // mark these descriptors as writing back
    for (uint32_t i = 0; i < cnt; i++) {
        desc_ctx &ctx = *desc_ctxs[(active_first_pos + i) % MAX_ACTIVE_DESCS];
        ctx.state = desc_ctx::DESC_WRITING_BACK;
    }

    do_writeback(active_first_idx, active_first_pos, cnt);
164
165
}

166
167
168
169
170
171
172
void queue_base::trigger()
{
  trigger_fetch();
  trigger_process();
  trigger_writeback();
}

173
174
void queue_base::reset()
{
175
#ifdef DEBUG_QUEUES
176
    log << "reset" << logger::endl;
177
178
#endif

179
    enabled = false;
180
181
182
183
184
185
186
    active_first_pos = 0;
    active_first_idx = 0;
    active_cnt = 0;

    for (size_t i = 0; i < MAX_ACTIVE_DESCS; i++) {
        desc_ctxs[i]->state = desc_ctx::DESC_EMPTY;
    }
187
188
}

Antoine Kaufmann's avatar
Antoine Kaufmann committed
189
190
void queue_base::reg_updated()
{
191
192
193
194
#ifdef DEBUG_QUEUES
    log << "reg_updated: tail=" << reg_tail << " enabled=" << (int) enabled <<
        logger::endl;
#endif
Antoine Kaufmann's avatar
Antoine Kaufmann committed
195
196
197
    if (!enabled)
        return;

198
    trigger();
Antoine Kaufmann's avatar
Antoine Kaufmann committed
199
200
}

201
202
203
204
205
bool queue_base::is_enabled()
{
    return enabled;
}

206
207
208
209
210
211
uint32_t queue_base::max_fetch_capacity()
{
    return UINT32_MAX;
}

uint32_t queue_base::max_active_capacity()
Antoine Kaufmann's avatar
Antoine Kaufmann committed
212
{
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
    return UINT32_MAX;
}

uint32_t queue_base::max_writeback_capacity()
{
    return UINT32_MAX;
}

void queue_base::interrupt()
{
}

void queue_base::do_writeback(uint32_t first_idx, uint32_t first_pos,
        uint32_t cnt)
{
    dma_wb *dma = new dma_wb(*this, desc_len * cnt);
Antoine Kaufmann's avatar
Antoine Kaufmann committed
229
    dma->write = true;
230
231
232
233
234
235
236
237
238
    dma->dma_addr = base + first_idx * desc_len;
    dma->pos = first_pos;

    uint8_t *buf = reinterpret_cast<uint8_t *> (dma->data);
    for (uint32_t i = 0; i < cnt; i++) {
        desc_ctx &ctx = *desc_ctxs[(first_pos + i) % MAX_ACTIVE_DESCS];
        assert(ctx.state == desc_ctx::DESC_WRITING_BACK);
        memcpy(buf + i * desc_len, ctx.desc, desc_len);
    }
Antoine Kaufmann's avatar
Antoine Kaufmann committed
239
240
241
242

    runner->issue_dma(*dma);
}

243
void queue_base::writeback_done(uint32_t first_pos, uint32_t cnt)
244
{
245
246
    if (!enabled)
        return;
247

248
249
250
251
252
253
254
    // first mark descriptors as written back
    for (uint32_t i = 0; i < cnt; i++) {
        desc_ctx &ctx = *desc_ctxs[(first_pos + i) % MAX_ACTIVE_DESCS];
        assert(ctx.state == desc_ctx::DESC_WRITING_BACK);
        ctx.state = desc_ctx::DESC_WRITTEN_BACK;
    }

255
#ifdef DEBUG_QUEUES
256
257
258
    log << "written back afi=" << active_first_idx << " afp=" <<
        active_first_pos << " acnt=" << active_cnt << " pos=" <<
        first_pos << " cnt=" << cnt << logger::endl;
259
#endif
260
261
262
263
264
265
266
267
268
269
270
271

    // then start at the beginning and check how many are written back and then
    // free those
    uint32_t bump_cnt = 0;
    for (bump_cnt = 0; bump_cnt < active_cnt; bump_cnt++) {
        desc_ctx &ctx = *desc_ctxs[(active_first_pos + bump_cnt) %
            MAX_ACTIVE_DESCS];
        if (ctx.state != desc_ctx::DESC_WRITTEN_BACK)
            break;

        ctx.state = desc_ctx::DESC_EMPTY;
    }
272
#ifdef DEBUG_QUEUES
273
    log << "   bump_cnt=" << bump_cnt << logger::endl;
274
#endif
275
276
277
278
279
280
281
282

    active_first_pos = (active_first_pos + bump_cnt) % MAX_ACTIVE_DESCS;
    active_first_idx = (active_first_idx + bump_cnt) % len;
    active_cnt -= bump_cnt;

    reg_head = active_first_idx;
    interrupt();
}
283

284
285
286
287
288
queue_base::desc_ctx::desc_ctx(queue_base &queue_)
    : queue(queue_), state(DESC_EMPTY), index(0), data(nullptr), data_len(0),
    data_capacity(0)
{
    desc = new uint8_t[queue_.desc_len];
289
290
}

291
queue_base::desc_ctx::~desc_ctx()
292
{
293
294
295
    delete[] ((uint8_t *) desc);
    if (data_capacity > 0)
        delete[] ((uint8_t *) data);
296
297
}

298
void queue_base::desc_ctx::prepare()
299
{
300
    prepared();
301
302
}

303
void queue_base::desc_ctx::prepared()
304
{
305
#ifdef DEBUG_QUEUES
306
    queue.log << "prepared desc " << index << logger::endl;
307
#endif
308
309
    assert(state == DESC_PREPARING);
    state = DESC_PREPARED;
310
311
}

312
void queue_base::desc_ctx::processed()
Antoine Kaufmann's avatar
Antoine Kaufmann committed
313
{
314
#ifdef DEBUG_QUEUES
315
    queue.log << "processed desc " << index << logger::endl;
316
#endif
317
318
319
320
    assert(state == DESC_PROCESSING);
    state = DESC_PROCESSED;
}

321
#define MAX_DMA_SIZE ((size_t) 9024)
322

323
324
325
void queue_base::desc_ctx::data_fetch(uint64_t addr, size_t data_len)
{
    if (data_capacity < data_len) {
326
#ifdef DEBUG_QUEUES
327
        queue.log << "data_fetch allocating" << logger::endl;
328
#endif
329
330
331
332
333
334
335
        if (data_capacity != 0)
            delete[] ((uint8_t *) data);

        data = new uint8_t[data_len];
        data_capacity = data_len;
    }

336
337
338
339
    dma_data_fetch *dma = new dma_data_fetch(*this, std::min(data_len,
                MAX_DMA_SIZE), data);
    dma->part_offset = 0;
    dma->total_len = data_len;
340
341
342
    dma->write = false;
    dma->dma_addr = addr;

343
#ifdef DEBUG_QUEUES
344
345
346
    queue.log << "fetching data idx=" << index << " addr=" << addr << " len=" <<
        data_len << logger::endl;
    queue.log << "  dma = " << dma << " data=" << data << logger::endl;
347
#endif
348
    runner->issue_dma(*dma);
349

350
351
352
353
354
355
356
357
358
359
}

void queue_base::desc_ctx::data_fetched(uint64_t addr, size_t len)
{
    prepared();
}

void queue_base::desc_ctx::data_write(uint64_t addr, size_t data_len,
        const void *buf)
{
360
#ifdef DEBUG_QUEUES
361
362
    queue.log << "data_write(addr=" << addr << " datalen=" << data_len << ")" <<
        logger::endl;
363
#endif
364
365
366
367
368
369
370
371
372
373
    dma_data_wb *data_dma = new dma_data_wb(*this, data_len);
    data_dma->write = true;
    data_dma->dma_addr = addr;
    memcpy(data_dma->data, buf, data_len);

    runner->issue_dma(*data_dma);
}

void queue_base::desc_ctx::data_written(uint64_t addr, size_t len)
{
374
#ifdef DEBUG_QUEUES
375
376
    queue.log << "data_written(addr=" << addr << " datalen=" << len << ")" <<
        logger::endl;
377
#endif
378
    processed();
Antoine Kaufmann's avatar
Antoine Kaufmann committed
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
}

queue_base::dma_fetch::dma_fetch(queue_base &queue_, size_t len_)
    : queue(queue_)
{
    data = new char[len_];
    len = len_;
}

queue_base::dma_fetch::~dma_fetch()
{
    delete[] ((char *) data);
}

void queue_base::dma_fetch::done()
{
395
396
397
398
399
    uint8_t *buf = reinterpret_cast <uint8_t *> (data);
    for (uint32_t i = 0; i < len / queue.desc_len; i++) {
        desc_ctx &ctx = *queue.desc_ctxs[(pos + i) % queue.MAX_ACTIVE_DESCS];
        memcpy(ctx.desc, buf + queue.desc_len * i, queue.desc_len);

400
#ifdef DEBUG_QUEUES
401
        queue.log << "preparing desc " << ctx.index << logger::endl;
402
#endif
403
404
405
        ctx.state = desc_ctx::DESC_PREPARING;
        ctx.prepare();
    }
406
    queue.trigger();
Antoine Kaufmann's avatar
Antoine Kaufmann committed
407
408
409
    delete this;
}

410
411
412
queue_base::dma_data_fetch::dma_data_fetch(desc_ctx &ctx_, size_t len_,
        void *buffer)
    : ctx(ctx_)
413
{
414
    data = buffer;
415
416
417
418
419
420
421
422
423
    len = len_;
}

queue_base::dma_data_fetch::~dma_data_fetch()
{
}

void queue_base::dma_data_fetch::done()
{
424
425
426
427
428
429
430
431
432
433
434
435
436
    part_offset += len;
    dma_addr += len;
    data = (uint8_t *) data + len;

    if (part_offset < total_len) {
#ifdef DEBUG_QUEUES
        ctx.queue.log << "  dma_fetch: next part of multi part dma" << logger::endl;
#endif
        len = std::min(total_len - part_offset, MAX_DMA_SIZE);
        runner->issue_dma(*this);
        return;
    }
    ctx.data_fetched(dma_addr - part_offset, total_len);
437
    ctx.queue.trigger();
438
439
    delete this;
}
Antoine Kaufmann's avatar
Antoine Kaufmann committed
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454

queue_base::dma_wb::dma_wb(queue_base &queue_, size_t len_)
    : queue(queue_)
{
    data = new char[len_];
    len = len_;
}

queue_base::dma_wb::~dma_wb()
{
    delete[] ((char *) data);
}

void queue_base::dma_wb::done()
{
455
    queue.writeback_done(pos, len / queue.desc_len);
456
    queue.trigger();
Antoine Kaufmann's avatar
Antoine Kaufmann committed
457
458
459
460
    delete this;
}


461
462
queue_base::dma_data_wb::dma_data_wb(desc_ctx &ctx_, size_t len_)
    : ctx(ctx_)
463
464
465
466
467
468
469
470
471
472
473
474
{
    data = new char[len_];
    len = len_;
}

queue_base::dma_data_wb::~dma_data_wb()
{
    delete[] ((char *) data);
}

void queue_base::dma_data_wb::done()
{
475
    ctx.data_written(dma_addr, len);
476
    ctx.queue.trigger();
477
478
    delete this;
}