DPDK 24.11.0-rc2
Loading...
Searching...
No Matches
rte_graph_worker_common.h
Go to the documentation of this file.
1/* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2020 Marvell International Ltd.
3 */
4
5#ifndef _RTE_GRAPH_WORKER_COMMON_H_
6#define _RTE_GRAPH_WORKER_COMMON_H_
7
15#include <assert.h>
16#include <stdalign.h>
17#include <stddef.h>
18
19#include <rte_common.h>
20#include <rte_cycles.h>
21#include <rte_prefetch.h>
22#include <rte_memcpy.h>
23#include <rte_memory.h>
24
25#include "rte_graph.h"
26
27#ifdef __cplusplus
28extern "C" {
29#endif
30
32/* When adding a new graph model entry, update rte_graph_model_is_valid() implementation. */
33#define RTE_GRAPH_MODEL_RTC 0
34#define RTE_GRAPH_MODEL_MCORE_DISPATCH 1
36#define RTE_GRAPH_MODEL_DEFAULT RTE_GRAPH_MODEL_RTC
43SLIST_HEAD(rte_graph_rq_head, rte_graph);
44
50struct __rte_cache_aligned rte_graph {
51 /* Fast path area. */
52 uint32_t tail;
53 uint32_t head;
54 uint32_t cir_mask;
55 rte_node_t nb_nodes;
56 rte_graph_off_t *cir_start;
57 rte_graph_off_t nodes_start;
58 uint8_t model;
59 uint8_t reserved1;
60 uint16_t reserved2;
61 union {
62 /* Fast schedule area for mcore dispatch model */
63 struct {
64 alignas(RTE_CACHE_LINE_SIZE) struct rte_graph_rq_head *rq;
65 /* The run-queue */
66 struct rte_graph_rq_head rq_head; /* The head for run-queue list */
67
68 unsigned int lcore_id;
69 struct rte_ring *wq;
70 struct rte_mempool *mp;
71 } dispatch;
72 };
73 SLIST_ENTRY(rte_graph) next; /* The next for rte_graph list */
74 /* End of Fast path area.*/
75 rte_graph_t id;
76 int socket;
77 char name[RTE_GRAPH_NAMESIZE];
78 bool pcap_enable;
80 uint64_t nb_pkt_captured;
82 uint64_t nb_pkt_to_capture;
83 char pcap_filename[RTE_GRAPH_PCAP_FILE_SZ];
84 uint64_t fence;
85};
86
92struct __rte_cache_aligned rte_node {
93 /* Slow path area */
94 uint64_t fence;
95 rte_graph_off_t next;
96 rte_node_t id;
97 rte_node_t parent_id;
98 rte_edge_t nb_edges;
99 uint32_t realloc_count;
101 char parent[RTE_NODE_NAMESIZE];
102 char name[RTE_NODE_NAMESIZE];
105 rte_node_process_t original_process;
106
107 union {
108 /* Fast schedule area for mcore dispatch model */
109 struct {
110 unsigned int lcore_id;
111 uint64_t total_sched_objs;
112 uint64_t total_sched_fail;
113 } dispatch;
114 };
115 rte_graph_off_t xstat_off;
116 /* Fast path area */
117 __extension__ struct __rte_cache_aligned {
118#define RTE_NODE_CTX_SZ 16
119 union {
120 uint8_t ctx[RTE_NODE_CTX_SZ];
121 __extension__ struct {
122 void *ctx_ptr;
123 void *ctx_ptr2;
124 };
125 };
126 uint16_t size;
127 uint16_t idx;
128 rte_graph_off_t off;
129 uint64_t total_cycles;
130 uint64_t total_calls;
131 uint64_t total_objs;
132 union {
133 void **objs;
134 uint64_t objs_u64;
135 };
136 union {
137 rte_node_process_t process;
138 uint64_t process_u64;
139 };
140 alignas(RTE_CACHE_LINE_MIN_SIZE) struct rte_node *nodes[];
141 };
142};
143
144static_assert(offsetof(struct rte_node, nodes) - offsetof(struct rte_node, ctx)
145 == RTE_CACHE_LINE_MIN_SIZE, "rte_node fast path area must fit in 64 bytes");
146
159void __rte_node_stream_alloc(struct rte_graph *graph, struct rte_node *node);
160
175void __rte_node_stream_alloc_size(struct rte_graph *graph,
176 struct rte_node *node, uint16_t req_size);
177
178/* Fast path helper functions */
179
190static __rte_always_inline void
191__rte_node_process(struct rte_graph *graph, struct rte_node *node)
192{
193 uint64_t start;
194 uint16_t rc;
195 void **objs;
196
197 RTE_ASSERT(node->fence == RTE_GRAPH_FENCE);
198 objs = node->objs;
199 rte_prefetch0(objs);
200
202 start = rte_rdtsc();
203 rc = node->process(graph, node, objs, node->idx);
204 node->total_cycles += rte_rdtsc() - start;
205 node->total_calls++;
206 node->total_objs += rc;
207 } else {
208 node->process(graph, node, objs, node->idx);
209 }
210 node->idx = 0;
211}
212
223static __rte_always_inline void
224__rte_node_enqueue_tail_update(struct rte_graph *graph, struct rte_node *node)
225{
226 uint32_t tail;
227
228 tail = graph->tail;
229 graph->cir_start[tail++] = node->off;
230 graph->tail = tail & graph->cir_mask;
231}
232
250static __rte_always_inline void
251__rte_node_enqueue_prologue(struct rte_graph *graph, struct rte_node *node,
252 const uint16_t idx, const uint16_t space)
253{
254
255 /* Add to the pending stream list if the node is new */
256 if (idx == 0)
257 __rte_node_enqueue_tail_update(graph, node);
258
259 if (unlikely(node->size < (idx + space)))
260 __rte_node_stream_alloc_size(graph, node, node->size + space);
261}
262
276static __rte_always_inline struct rte_node *
277__rte_node_next_node_get(struct rte_node *node, rte_edge_t next)
278{
279 RTE_ASSERT(next < node->nb_edges);
280 RTE_ASSERT(node->fence == RTE_GRAPH_FENCE);
281 node = node->nodes[next];
282 RTE_ASSERT(node->fence == RTE_GRAPH_FENCE);
283
284 return node;
285}
286
302static inline void
303rte_node_enqueue(struct rte_graph *graph, struct rte_node *node,
304 rte_edge_t next, void **objs, uint16_t nb_objs)
305{
306 node = __rte_node_next_node_get(node, next);
307 const uint16_t idx = node->idx;
308
309 __rte_node_enqueue_prologue(graph, node, idx, nb_objs);
310
311 rte_memcpy(&node->objs[idx], objs, nb_objs * sizeof(void *));
312 node->idx = idx + nb_objs;
313}
314
328static inline void
329rte_node_enqueue_x1(struct rte_graph *graph, struct rte_node *node,
330 rte_edge_t next, void *obj)
331{
332 node = __rte_node_next_node_get(node, next);
333 uint16_t idx = node->idx;
334
335 __rte_node_enqueue_prologue(graph, node, idx, 1);
336
337 node->objs[idx++] = obj;
338 node->idx = idx;
339}
340
357static inline void
358rte_node_enqueue_x2(struct rte_graph *graph, struct rte_node *node,
359 rte_edge_t next, void *obj0, void *obj1)
360{
361 node = __rte_node_next_node_get(node, next);
362 uint16_t idx = node->idx;
363
364 __rte_node_enqueue_prologue(graph, node, idx, 2);
365
366 node->objs[idx++] = obj0;
367 node->objs[idx++] = obj1;
368 node->idx = idx;
369}
370
391static inline void
392rte_node_enqueue_x4(struct rte_graph *graph, struct rte_node *node,
393 rte_edge_t next, void *obj0, void *obj1, void *obj2,
394 void *obj3)
395{
396 node = __rte_node_next_node_get(node, next);
397 uint16_t idx = node->idx;
398
399 __rte_node_enqueue_prologue(graph, node, idx, 4);
400
401 node->objs[idx++] = obj0;
402 node->objs[idx++] = obj1;
403 node->objs[idx++] = obj2;
404 node->objs[idx++] = obj3;
405 node->idx = idx;
406}
407
424static inline void
425rte_node_enqueue_next(struct rte_graph *graph, struct rte_node *node,
426 rte_edge_t *nexts, void **objs, uint16_t nb_objs)
427{
428 uint16_t i;
429
430 for (i = 0; i < nb_objs; i++)
431 rte_node_enqueue_x1(graph, node, nexts[i], objs[i]);
432}
433
453static inline void **
454rte_node_next_stream_get(struct rte_graph *graph, struct rte_node *node,
455 rte_edge_t next, uint16_t nb_objs)
456{
457 node = __rte_node_next_node_get(node, next);
458 const uint16_t idx = node->idx;
459 uint16_t free_space = node->size - idx;
460
461 if (unlikely(free_space < nb_objs))
462 __rte_node_stream_alloc_size(graph, node, node->size + nb_objs);
463
464 return &node->objs[idx];
465}
466
483static inline void
484rte_node_next_stream_put(struct rte_graph *graph, struct rte_node *node,
485 rte_edge_t next, uint16_t idx)
486{
487 if (unlikely(!idx))
488 return;
489
490 node = __rte_node_next_node_get(node, next);
491 if (node->idx == 0)
492 __rte_node_enqueue_tail_update(graph, node);
493
494 node->idx += idx;
495}
496
511static inline void
512rte_node_next_stream_move(struct rte_graph *graph, struct rte_node *src,
513 rte_edge_t next)
514{
515 struct rte_node *dst = __rte_node_next_node_get(src, next);
516
517 /* Let swap the pointers if dst don't have valid objs */
518 if (likely(dst->idx == 0)) {
519 void **dobjs = dst->objs;
520 uint16_t dsz = dst->size;
521 dst->objs = src->objs;
522 dst->size = src->size;
523 src->objs = dobjs;
524 src->size = dsz;
525 dst->idx = src->idx;
526 __rte_node_enqueue_tail_update(graph, dst);
527 } else { /* Move the objects from src node to dst node */
528 rte_node_enqueue(graph, src, next, src->objs, src->idx);
529 }
530}
531
541bool
543
554int rte_graph_worker_model_set(uint8_t model);
555
568uint8_t rte_graph_worker_model_get(struct rte_graph *graph);
569
583uint8_t rte_graph_worker_model_no_check_get(struct rte_graph *graph)
584{
585 return graph->model;
586}
587
600__rte_experimental
601static inline void
602rte_node_xstat_increment(struct rte_node *node, uint16_t xstat_id, uint64_t value)
603{
605 uint64_t *xstat = (uint64_t *)RTE_PTR_ADD(node, node->xstat_off);
606 xstat[xstat_id] += value;
607 }
608}
609
610#ifdef __cplusplus
611}
612#endif
613
614#endif /* _RTE_GRAPH_WORKER_COIMMON_H_ */
#define likely(x)
#define unlikely(x)
#define RTE_CACHE_LINE_MIN_SIZE
Definition rte_common.h:624
#define offsetof(TYPE, MEMBER)
Definition rte_common.h:724
#define RTE_PTR_ADD(ptr, x)
Definition rte_common.h:469
#define __rte_cache_aligned
Definition rte_common.h:627
#define __rte_always_inline
Definition rte_common.h:413
uint16_t(* rte_node_process_t)(struct rte_graph *graph, struct rte_node *node, void **objs, uint16_t nb_objs)
Definition rte_graph.h:94
uint32_t rte_node_t
Definition rte_graph.h:41
#define RTE_GRAPH_FENCE
Definition rte_graph.h:38
uint16_t rte_edge_t
Definition rte_graph.h:42
#define RTE_NODE_NAMESIZE
Definition rte_graph.h:31
#define RTE_GRAPH_PCAP_FILE_SZ
Definition rte_graph.h:33
uint32_t rte_graph_off_t
Definition rte_graph.h:40
uint16_t rte_graph_t
Definition rte_graph.h:43
#define RTE_GRAPH_NAMESIZE
Definition rte_graph.h:30
static __rte_always_inline int rte_graph_has_stats_feature(void)
Definition rte_graph.h:701
static __rte_always_inline uint8_t rte_graph_worker_model_no_check_get(struct rte_graph *graph)
static void rte_node_enqueue_x4(struct rte_graph *graph, struct rte_node *node, rte_edge_t next, void *obj0, void *obj1, void *obj2, void *obj3)
bool rte_graph_model_is_valid(uint8_t model)
static void rte_node_next_stream_put(struct rte_graph *graph, struct rte_node *node, rte_edge_t next, uint16_t idx)
static void rte_node_enqueue(struct rte_graph *graph, struct rte_node *node, rte_edge_t next, void **objs, uint16_t nb_objs)
static void rte_node_next_stream_move(struct rte_graph *graph, struct rte_node *src, rte_edge_t next)
int rte_graph_worker_model_set(uint8_t model)
static void rte_node_enqueue_next(struct rte_graph *graph, struct rte_node *node, rte_edge_t *nexts, void **objs, uint16_t nb_objs)
static void rte_node_enqueue_x1(struct rte_graph *graph, struct rte_node *node, rte_edge_t next, void *obj)
static __rte_experimental void rte_node_xstat_increment(struct rte_node *node, uint16_t xstat_id, uint64_t value)
static void ** rte_node_next_stream_get(struct rte_graph *graph, struct rte_node *node, rte_edge_t next, uint16_t nb_objs)
static void rte_node_enqueue_x2(struct rte_graph *graph, struct rte_node *node, rte_edge_t next, void *obj0, void *obj1)
uint8_t rte_graph_worker_model_get(struct rte_graph *graph)
static void * rte_memcpy(void *dst, const void *src, size_t n)
static void rte_prefetch0(const volatile void *p)