i40e_queues.cc 10.4 KB
Newer Older
Antoine Kaufmann's avatar
Antoine Kaufmann committed
1
2
3
4
5
6
7
8
9
10
11
12
13
#include <stdlib.h>
#include <string.h>
#include <cassert>
#include <iostream>

#include "i40e_bm.h"

#include "i40e_base_wrapper.h"

using namespace i40e;

extern nicbm::Runner *runner;

14
15
16
queue_base::queue_base(const std::string &qname_, uint32_t &reg_head_,
        uint32_t &reg_tail_)
    : qname(qname_), active_first_pos(0), active_first_idx(0), active_cnt(0),
17
18
    base(0), len(0), reg_head(reg_head_), reg_tail(reg_tail_),
    enabled(false), desc_len(0)
Antoine Kaufmann's avatar
Antoine Kaufmann committed
19
{
20
21
22
23
24
25
26
27
28
29
    for (size_t i = 0; i < MAX_ACTIVE_DESCS; i++) {
        desc_ctxs[i] = nullptr;
    }
}

void queue_base::ctxs_init()
{
    for (size_t i = 0; i < MAX_ACTIVE_DESCS; i++) {
        desc_ctxs[i] = &desc_ctx_create();
    }
Antoine Kaufmann's avatar
Antoine Kaufmann committed
30
31
32
33
}

void queue_base::trigger_fetch()
{
34
    if (!enabled)
Antoine Kaufmann's avatar
Antoine Kaufmann committed
35
36
        return;

37
38
39
40
41
42
43
44
    // calculate how many we can fetch
    uint32_t next_idx = (active_first_idx + active_cnt) % len;
    uint32_t desc_avail = (reg_tail - next_idx) % len;
    uint32_t fetch_cnt = desc_avail;
    fetch_cnt = std::min(fetch_cnt, MAX_ACTIVE_DESCS - active_cnt);
    if (max_active_capacity() <= active_cnt)
        fetch_cnt = std::min(fetch_cnt, max_active_capacity() - active_cnt);
    fetch_cnt = std::min(fetch_cnt, max_fetch_capacity());
Antoine Kaufmann's avatar
Antoine Kaufmann committed
45

46
47
    if (next_idx + fetch_cnt > len)
        fetch_cnt = len - next_idx;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
48

49
50
#ifdef DEBUG_QUEUES
    std::cerr << qname << ": fetching avail=" << desc_avail <<
51
        " cnt=" << fetch_cnt << " idx=" << next_idx << std::endl;
52
#endif
53

54
55
56
57
58
59
60
61
62
63
64
65
66
67
    // abort if nothign to fetch
    if (fetch_cnt == 0)
        return;

    // mark descriptor contexts as fetching
    uint32_t first_pos = (active_first_pos + active_cnt) % MAX_ACTIVE_DESCS;
    for (uint32_t i = 0; i < fetch_cnt; i++) {
        desc_ctx &ctx = *desc_ctxs[(first_pos + i) % MAX_ACTIVE_DESCS];
        assert(ctx.state == desc_ctx::DESC_EMPTY);

        ctx.state = desc_ctx::DESC_FETCHING;
        ctx.index = (next_idx + i) % len;
    }
    active_cnt += fetch_cnt;
Antoine Kaufmann's avatar
Antoine Kaufmann committed
68

69
70
71
72
73
    // prepare & issue dma
    dma_fetch *dma = new dma_fetch(*this, desc_len * fetch_cnt);
    dma->write = false;
    dma->dma_addr = base + next_idx * desc_len;
    dma->pos = first_pos;
74
75
76
#ifdef DEBUG_QUEUES
    std::cerr << qname << ":    dma = " << dma << std::endl;
#endif
Antoine Kaufmann's avatar
Antoine Kaufmann committed
77
78
79
    runner->issue_dma(*dma);
}

80
void queue_base::trigger_process()
81
{
82
83
    if (!enabled)
        return;
84

85
86
87
88
    // first skip over descriptors that are already done processing
    uint32_t i;
    for (i = 0; i < active_cnt; i++)
        if (desc_ctxs[(active_first_pos + i) % MAX_ACTIVE_DESCS]->state
89
                <= desc_ctx::DESC_PREPARED)
90
91
92
93
94
95
96
97
98
99
100
            break;

    // then run all prepared contexts
    uint32_t j;
    for (j = 0; i + j < active_cnt; j++) {
        desc_ctx &ctx = *desc_ctxs[(active_first_pos + i + j)
            % MAX_ACTIVE_DESCS];
        if (ctx.state != desc_ctx::DESC_PREPARED)
            break;

        ctx.state = desc_ctx::DESC_PROCESSING;
101
102
103
#ifdef DEBUG_QUEUES
        std::cerr << qname << ": processing desc " << ctx.index << std::endl;
#endif
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
        ctx.process();
    }
}

void queue_base::trigger_writeback()
{
    if (!enabled)
        return;

    // from first pos count number of processed descriptors
    uint32_t avail;
    for (avail = 0; avail < active_cnt; avail++)
        if (desc_ctxs[(active_first_pos + avail) % MAX_ACTIVE_DESCS]->state
                != desc_ctx::DESC_PROCESSED)
            break;

    uint32_t cnt = std::min(avail, max_writeback_capacity());
121
122
    if (active_first_idx + cnt > len)
        cnt = len - active_first_idx;
123

124
125
126
127
#ifdef DEBUG_QUEUES
    std::cerr << qname << ": writing back avail=" << avail << " cnt=" << cnt <<
        " idx=" << active_first_idx << std::endl;
#endif
128
129
130
131
132
133
134
135
136
137
138

    if (cnt == 0)
        return;

    // mark these descriptors as writing back
    for (uint32_t i = 0; i < cnt; i++) {
        desc_ctx &ctx = *desc_ctxs[(active_first_pos + i) % MAX_ACTIVE_DESCS];
        ctx.state = desc_ctx::DESC_WRITING_BACK;
    }

    do_writeback(active_first_idx, active_first_pos, cnt);
139
140
}

141
142
void queue_base::reset()
{
143
144
145
146
#ifdef DEBUG_QUEUES
    std::cerr << qname << ": reset" << std::endl;
#endif

147
    enabled = false;
148
149
150
151
152
153
154
    active_first_pos = 0;
    active_first_idx = 0;
    active_cnt = 0;

    for (size_t i = 0; i < MAX_ACTIVE_DESCS; i++) {
        desc_ctxs[i]->state = desc_ctx::DESC_EMPTY;
    }
155
156
}

Antoine Kaufmann's avatar
Antoine Kaufmann committed
157
158
159
160
161
162
163
164
void queue_base::reg_updated()
{
    if (!enabled)
        return;

    trigger_fetch();
}

165
166
167
168
169
bool queue_base::is_enabled()
{
    return enabled;
}

170
171
172
173
174
175
uint32_t queue_base::max_fetch_capacity()
{
    return UINT32_MAX;
}

uint32_t queue_base::max_active_capacity()
Antoine Kaufmann's avatar
Antoine Kaufmann committed
176
{
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
    return UINT32_MAX;
}

uint32_t queue_base::max_writeback_capacity()
{
    return UINT32_MAX;
}

void queue_base::interrupt()
{
}

void queue_base::do_writeback(uint32_t first_idx, uint32_t first_pos,
        uint32_t cnt)
{
    dma_wb *dma = new dma_wb(*this, desc_len * cnt);
Antoine Kaufmann's avatar
Antoine Kaufmann committed
193
    dma->write = true;
194
195
196
197
198
199
200
201
202
    dma->dma_addr = base + first_idx * desc_len;
    dma->pos = first_pos;

    uint8_t *buf = reinterpret_cast<uint8_t *> (dma->data);
    for (uint32_t i = 0; i < cnt; i++) {
        desc_ctx &ctx = *desc_ctxs[(first_pos + i) % MAX_ACTIVE_DESCS];
        assert(ctx.state == desc_ctx::DESC_WRITING_BACK);
        memcpy(buf + i * desc_len, ctx.desc, desc_len);
    }
Antoine Kaufmann's avatar
Antoine Kaufmann committed
203
204
205
206

    runner->issue_dma(*dma);
}

207
void queue_base::writeback_done(uint32_t first_pos, uint32_t cnt)
208
{
209
210
    if (!enabled)
        return;
211

212
213
214
215
216
217
218
    // first mark descriptors as written back
    for (uint32_t i = 0; i < cnt; i++) {
        desc_ctx &ctx = *desc_ctxs[(first_pos + i) % MAX_ACTIVE_DESCS];
        assert(ctx.state == desc_ctx::DESC_WRITING_BACK);
        ctx.state = desc_ctx::DESC_WRITTEN_BACK;
    }

219
220
221
222
223
#ifdef DEBUG_QUEUES
    std::cerr << qname << ": written back afi=" << active_first_idx <<
        " afp=" << active_first_pos << " acnt=" << active_cnt << " pos=" <<
        first_pos << " cnt=" << cnt << std::endl;
#endif
224
225
226
227
228
229
230
231
232
233
234
235

    // then start at the beginning and check how many are written back and then
    // free those
    uint32_t bump_cnt = 0;
    for (bump_cnt = 0; bump_cnt < active_cnt; bump_cnt++) {
        desc_ctx &ctx = *desc_ctxs[(active_first_pos + bump_cnt) %
            MAX_ACTIVE_DESCS];
        if (ctx.state != desc_ctx::DESC_WRITTEN_BACK)
            break;

        ctx.state = desc_ctx::DESC_EMPTY;
    }
236
237
238
#ifdef DEBUG_QUEUES
    std::cerr << qname << ":    bump_cnt=" << bump_cnt << std::endl;
#endif
239
240
241
242
243
244
245

    active_first_pos = (active_first_pos + bump_cnt) % MAX_ACTIVE_DESCS;
    active_first_idx = (active_first_idx + bump_cnt) % len;
    active_cnt -= bump_cnt;

    reg_head = active_first_idx;
    interrupt();
246
247
248

    trigger_fetch();
    trigger_writeback();
249
}
250

251
252
253
254
255
queue_base::desc_ctx::desc_ctx(queue_base &queue_)
    : queue(queue_), state(DESC_EMPTY), index(0), data(nullptr), data_len(0),
    data_capacity(0)
{
    desc = new uint8_t[queue_.desc_len];
256
257
}

258
queue_base::desc_ctx::~desc_ctx()
259
{
260
261
262
    delete[] ((uint8_t *) desc);
    if (data_capacity > 0)
        delete[] ((uint8_t *) data);
263
264
}

265
void queue_base::desc_ctx::prepare()
266
{
267
    prepared();
268
269
}

270
void queue_base::desc_ctx::prepared()
271
{
272
273
274
#ifdef DEBUG_QUEUES
    std::cerr << queue.qname << ": prepared desc " << index << std::endl;
#endif
275
276
277
    assert(state == DESC_PREPARING);
    state = DESC_PREPARED;
    queue.trigger_process();
278
279
}

280
void queue_base::desc_ctx::processed()
Antoine Kaufmann's avatar
Antoine Kaufmann committed
281
{
282
283
284
#ifdef DEBUG_QUEUES
    std::cerr << queue.qname << ": processed desc " << index << std::endl;
#endif
285
286
287
288
289
290
291
292
    assert(state == DESC_PROCESSING);
    state = DESC_PROCESSED;
    queue.trigger_writeback();
}

void queue_base::desc_ctx::data_fetch(uint64_t addr, size_t data_len)
{
    if (data_capacity < data_len) {
293
294
295
#ifdef DEBUG_QUEUES
        std::cerr << queue.qname << ": data_fetch allocating" << std::endl;
#endif
296
297
298
299
300
301
302
303
304
305
306
        if (data_capacity != 0)
            delete[] ((uint8_t *) data);

        data = new uint8_t[data_len];
        data_capacity = data_len;
    }

    dma_data_fetch *dma = new dma_data_fetch(*this, data_len, data);
    dma->write = false;
    dma->dma_addr = addr;

307
308
309
310
311
312
#ifdef DEBUG_QUEUES
    std::cerr << queue.qname << ": fetching data idx=" << index << " addr=" <<
        addr << " len=" << data_len << std::endl;
    std::cerr << queue.qname << ": dma = " << dma << " data=" << data <<
        std::endl;
#endif
313
    runner->issue_dma(*dma);
314

315
316
317
318
319
320
321
322
323
324
}

void queue_base::desc_ctx::data_fetched(uint64_t addr, size_t len)
{
    prepared();
}

void queue_base::desc_ctx::data_write(uint64_t addr, size_t data_len,
        const void *buf)
{
325
326
327
328
#ifdef DEBUG_QUEUES
    std::cerr << queue.qname << ": data_write(addr=" << addr << " datalen=" <<
        data_len << ")" << std::endl;
#endif
329
330
331
332
333
334
335
336
337
338
    dma_data_wb *data_dma = new dma_data_wb(*this, data_len);
    data_dma->write = true;
    data_dma->dma_addr = addr;
    memcpy(data_dma->data, buf, data_len);

    runner->issue_dma(*data_dma);
}

void queue_base::desc_ctx::data_written(uint64_t addr, size_t len)
{
339
340
341
342
#ifdef DEBUG_QUEUES
    std::cerr << queue.qname << ": data_written(addr=" << addr << " datalen=" <<
        len << ")" << std::endl;
#endif
343
    processed();
Antoine Kaufmann's avatar
Antoine Kaufmann committed
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
}

queue_base::dma_fetch::dma_fetch(queue_base &queue_, size_t len_)
    : queue(queue_)
{
    data = new char[len_];
    len = len_;
}

queue_base::dma_fetch::~dma_fetch()
{
    delete[] ((char *) data);
}

void queue_base::dma_fetch::done()
{
360
361
362
363
364
    uint8_t *buf = reinterpret_cast <uint8_t *> (data);
    for (uint32_t i = 0; i < len / queue.desc_len; i++) {
        desc_ctx &ctx = *queue.desc_ctxs[(pos + i) % queue.MAX_ACTIVE_DESCS];
        memcpy(ctx.desc, buf + queue.desc_len * i, queue.desc_len);

365
366
367
368
#ifdef DEBUG_QUEUES
        std::cerr << ctx.queue.qname << ": preparing desc " << ctx.index <<
            std::endl;
#endif
369
370
371
        ctx.state = desc_ctx::DESC_PREPARING;
        ctx.prepare();
    }
Antoine Kaufmann's avatar
Antoine Kaufmann committed
372
373
374
    delete this;
}

375
376
377
queue_base::dma_data_fetch::dma_data_fetch(desc_ctx &ctx_, size_t len_,
        void *buffer)
    : ctx(ctx_)
378
{
379
    data = buffer;
380
381
382
383
384
385
386
387
388
    len = len_;
}

queue_base::dma_data_fetch::~dma_data_fetch()
{
}

void queue_base::dma_data_fetch::done()
{
389
    ctx.data_fetched(dma_addr, len);
390
391
    delete this;
}
Antoine Kaufmann's avatar
Antoine Kaufmann committed
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406

queue_base::dma_wb::dma_wb(queue_base &queue_, size_t len_)
    : queue(queue_)
{
    data = new char[len_];
    len = len_;
}

queue_base::dma_wb::~dma_wb()
{
    delete[] ((char *) data);
}

void queue_base::dma_wb::done()
{
407
    queue.writeback_done(pos, len / queue.desc_len);
Antoine Kaufmann's avatar
Antoine Kaufmann committed
408
409
410
411
    delete this;
}


412
413
queue_base::dma_data_wb::dma_data_wb(desc_ctx &ctx_, size_t len_)
    : ctx(ctx_)
414
415
416
417
418
419
420
421
422
423
424
425
{
    data = new char[len_];
    len = len_;
}

queue_base::dma_data_wb::~dma_data_wb()
{
    delete[] ((char *) data);
}

void queue_base::dma_data_wb::done()
{
426
    ctx.data_written(dma_addr, len);
427
428
    delete this;
}