5#ifndef _RTE_GRAPH_WORKER_COMMON_H_
6#define _RTE_GRAPH_WORKER_COMMON_H_
33#define RTE_GRAPH_MODEL_RTC 0
34#define RTE_GRAPH_MODEL_MCORE_DISPATCH 1
36#define RTE_GRAPH_MODEL_DEFAULT RTE_GRAPH_MODEL_RTC
43SLIST_HEAD(rte_graph_rq_head, rte_graph);
64 alignas(RTE_CACHE_LINE_SIZE)
struct rte_graph_rq_head *rq;
66 struct rte_graph_rq_head rq_head;
68 unsigned int lcore_id;
73 SLIST_ENTRY(rte_graph) next;
80 uint64_t nb_pkt_captured;
82 uint64_t nb_pkt_to_capture;
99 uint32_t realloc_count;
110 unsigned int lcore_id;
111 uint64_t total_sched_objs;
112 uint64_t total_sched_fail;
118#define RTE_NODE_CTX_SZ 16
120 uint8_t ctx[RTE_NODE_CTX_SZ];
121 __extension__
struct {
129 uint64_t total_cycles;
130 uint64_t total_calls;
138 uint64_t process_u64;
144static_assert(
offsetof(
struct rte_node, nodes) -
offsetof(
struct rte_node, ctx)
159void __rte_node_stream_alloc(
struct rte_graph *graph,
struct rte_node *node);
175void __rte_node_stream_alloc_size(
struct rte_graph *graph,
176 struct rte_node *node, uint16_t req_size);
191__rte_node_process(
struct rte_graph *graph,
struct rte_node *node)
203 rc = node->process(graph, node, objs, node->idx);
204 node->total_cycles += rte_rdtsc() - start;
206 node->total_objs += rc;
208 node->process(graph, node, objs, node->idx);
224__rte_node_enqueue_tail_update(
struct rte_graph *graph,
struct rte_node *node)
229 graph->cir_start[tail++] = node->off;
230 graph->tail = tail & graph->cir_mask;
251__rte_node_enqueue_prologue(
struct rte_graph *graph,
struct rte_node *node,
252 const uint16_t idx,
const uint16_t space)
257 __rte_node_enqueue_tail_update(graph, node);
259 if (
unlikely(node->size < (idx + space)))
260 __rte_node_stream_alloc_size(graph, node, node->size + space);
277__rte_node_next_node_get(
struct rte_node *node,
rte_edge_t next)
279 RTE_ASSERT(next < node->nb_edges);
281 node = node->nodes[next];
304 rte_edge_t next,
void **objs, uint16_t nb_objs)
306 node = __rte_node_next_node_get(node, next);
307 const uint16_t idx = node->idx;
309 __rte_node_enqueue_prologue(graph, node, idx, nb_objs);
311 rte_memcpy(&node->objs[idx], objs, nb_objs *
sizeof(
void *));
312 node->idx = idx + nb_objs;
332 node = __rte_node_next_node_get(node, next);
333 uint16_t idx = node->idx;
335 __rte_node_enqueue_prologue(graph, node, idx, 1);
337 node->objs[idx++] = obj;
361 node = __rte_node_next_node_get(node, next);
362 uint16_t idx = node->idx;
364 __rte_node_enqueue_prologue(graph, node, idx, 2);
366 node->objs[idx++] = obj0;
367 node->objs[idx++] = obj1;
393 rte_edge_t next,
void *obj0,
void *obj1,
void *obj2,
396 node = __rte_node_next_node_get(node, next);
397 uint16_t idx = node->idx;
399 __rte_node_enqueue_prologue(graph, node, idx, 4);
401 node->objs[idx++] = obj0;
402 node->objs[idx++] = obj1;
403 node->objs[idx++] = obj2;
404 node->objs[idx++] = obj3;
426 rte_edge_t *nexts,
void **objs, uint16_t nb_objs)
430 for (i = 0; i < nb_objs; i++)
457 node = __rte_node_next_node_get(node, next);
458 const uint16_t idx = node->idx;
459 uint16_t free_space = node->size - idx;
462 __rte_node_stream_alloc_size(graph, node, node->size + nb_objs);
464 return &node->objs[idx];
490 node = __rte_node_next_node_get(node, next);
492 __rte_node_enqueue_tail_update(graph, node);
515 struct rte_node *dst = __rte_node_next_node_get(src, next);
518 if (
likely(dst->idx == 0)) {
519 void **dobjs = dst->objs;
520 uint16_t dsz = dst->size;
521 dst->objs = src->objs;
522 dst->size = src->size;
526 __rte_node_enqueue_tail_update(graph, dst);
605 uint64_t *xstat = (uint64_t *)
RTE_PTR_ADD(node, node->xstat_off);
606 xstat[xstat_id] += value;
#define RTE_CACHE_LINE_MIN_SIZE
#define offsetof(TYPE, MEMBER)
#define RTE_PTR_ADD(ptr, x)
#define __rte_cache_aligned
#define __rte_always_inline
uint16_t(* rte_node_process_t)(struct rte_graph *graph, struct rte_node *node, void **objs, uint16_t nb_objs)
#define RTE_NODE_NAMESIZE
#define RTE_GRAPH_PCAP_FILE_SZ
#define RTE_GRAPH_NAMESIZE
static __rte_always_inline int rte_graph_has_stats_feature(void)
static __rte_always_inline uint8_t rte_graph_worker_model_no_check_get(struct rte_graph *graph)
static void rte_node_enqueue_x4(struct rte_graph *graph, struct rte_node *node, rte_edge_t next, void *obj0, void *obj1, void *obj2, void *obj3)
bool rte_graph_model_is_valid(uint8_t model)
static void rte_node_next_stream_put(struct rte_graph *graph, struct rte_node *node, rte_edge_t next, uint16_t idx)
static void rte_node_enqueue(struct rte_graph *graph, struct rte_node *node, rte_edge_t next, void **objs, uint16_t nb_objs)
static void rte_node_next_stream_move(struct rte_graph *graph, struct rte_node *src, rte_edge_t next)
int rte_graph_worker_model_set(uint8_t model)
static void rte_node_enqueue_next(struct rte_graph *graph, struct rte_node *node, rte_edge_t *nexts, void **objs, uint16_t nb_objs)
static void rte_node_enqueue_x1(struct rte_graph *graph, struct rte_node *node, rte_edge_t next, void *obj)
static __rte_experimental void rte_node_xstat_increment(struct rte_node *node, uint16_t xstat_id, uint64_t value)
static void ** rte_node_next_stream_get(struct rte_graph *graph, struct rte_node *node, rte_edge_t next, uint16_t nb_objs)
static void rte_node_enqueue_x2(struct rte_graph *graph, struct rte_node *node, rte_edge_t next, void *obj0, void *obj1)
uint8_t rte_graph_worker_model_get(struct rte_graph *graph)
static void * rte_memcpy(void *dst, const void *src, size_t n)
static void rte_prefetch0(const volatile void *p)