blob: c5e6a288036d011065bdc2352d5f4f4de3039890 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
|
/* Copyright (c) 2017, ARM Limited. All rights reserved.
*
* Copyright (c) 2017-2018, Linaro Limited
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
#ifndef ODP_SCHEDULE_SCALABLE_H
#define ODP_SCHEDULE_SCALABLE_H
#include <odp/api/align.h>
#include <odp/api/schedule.h>
#include <odp/api/ticketlock.h>
#include <odp_schedule_scalable_config.h>
#include <odp_schedule_scalable_ordered.h>
#include <odp_llqueue.h>
/*
* ODP_SCHED_PRIO_HIGHEST/NORMAL/LOWEST/DEFAULT are compile time
* constants, but not ODP_SCHED_PRIO_NUM. The current API for this
* is odp_schedule_num_prio(). The other schedulers also define
* this internally as NUM_PRIO.
*
* One additional priority level for idle pktin queues.
* This is only for internal use and not visible to the user.
*/
#define ODP_SCHED_PRIO_PKTIN 8
#define ODP_SCHED_PRIO_NUM 9
typedef struct ODP_ALIGNED_CACHE {
union {
struct {
struct llqueue llq;
uint32_t prio;
};
char line[ODP_CACHE_LINE_SIZE];
};
} sched_queue_t;
#define TICKET_INVALID (uint16_t)(~0U)
typedef struct ODP_ALIGNED(sizeof(uint64_t)) {
int32_t numevts;
uint16_t wrr_budget;
uint8_t cur_ticket;
uint8_t nxt_ticket;
} qschedstate_t;
typedef uint32_t ringidx_t;
#ifdef CONFIG_SPLIT_PRODCONS
#define SPLIT_PC ODP_ALIGNED_CACHE
#else
#define SPLIT_PC
#endif
#define ODP_NO_SCHED_QUEUE (ODP_SCHED_SYNC_ORDERED + 1)
typedef struct ODP_ALIGNED_CACHE {
struct llnode node;
sched_queue_t *schedq;
#ifdef CONFIG_QSCHST_LOCK
odp_ticketlock_t qschlock;
#endif
qschedstate_t qschst;
uint8_t pop_deficit;
uint8_t qschst_type;
uint8_t pktio_idx;
uint8_t rx_queue;
uint16_t xoffset;
uint8_t sched_prio;
ringidx_t prod_read SPLIT_PC;
ringidx_t prod_write;
ringidx_t prod_mask;
odp_buffer_hdr_t **prod_ring;
ringidx_t cons_write SPLIT_PC;
ringidx_t cons_read;
reorder_window_t *rwin;
void *user_ctx;
#ifdef CONFIG_SPLIT_PRODCONS
odp_buffer_hdr_t **cons_ring;
ringidx_t cons_mask;
uint16_t cons_type;
#else
#define cons_mask prod_mask
#define cons_ring prod_ring
#define cons_type qschst_type
#endif
odp_schedule_group_t sched_grp;
uint32_t loop_check[CONFIG_NUM_CPU_IDS];
} sched_elem_t;
/* Number of scheduling groups */
#define MAX_SCHED_GROUP (sizeof(sched_group_mask_t) * CHAR_BIT)
typedef bitset_t sched_group_mask_t;
typedef struct {
/* Threads currently associated with the sched group */
bitset_t thr_actual[ODP_SCHED_PRIO_NUM] ODP_ALIGNED_CACHE;
bitset_t thr_wanted;
/* Used to spread queues over schedq's */
uint32_t xcount[ODP_SCHED_PRIO_NUM];
/* Number of schedq's per prio */
uint32_t xfactor;
char name[ODP_SCHED_GROUP_NAME_LEN];
/* ODP_SCHED_PRIO_NUM * xfactor. Must be last. */
sched_queue_t schedq[1] ODP_ALIGNED_CACHE;
} sched_group_t;
/* Number of reorder contexts per thread */
#define TS_RVEC_SIZE 16
typedef struct ODP_ALIGNED_CACHE {
/* Atomic queue currently being processed or NULL */
sched_elem_t *atomq;
/* Schedq the currently processed queue was popped from */
sched_queue_t *src_schedq;
/* Current reorder context or NULL */
reorder_context_t *rctx;
uint8_t pause;
uint8_t out_of_order;
uint8_t tidx;
uint8_t pad;
uint32_t dequeued; /* Number of events dequeued from atomic queue */
uint16_t ticket; /* Ticket for atomic queue or TICKET_INVALID */
uint16_t num_schedq;
uint16_t sg_sem; /* Set when sg_wanted is modified by other thread */
#define SCHEDQ_PER_THREAD (MAX_SCHED_GROUP * ODP_SCHED_PRIO_NUM)
sched_queue_t *schedq_list[SCHEDQ_PER_THREAD];
/* Current sched_group membership */
sched_group_mask_t sg_actual[ODP_SCHED_PRIO_NUM];
/* Future sched_group membership. */
sched_group_mask_t sg_wanted[ODP_SCHED_PRIO_NUM];
bitset_t priv_rvec_free;
/* Bitset of free entries in rvec[] */
bitset_t rvec_free ODP_ALIGNED_CACHE;
/* Reordering contexts to allocate from */
reorder_context_t rvec[TS_RVEC_SIZE] ODP_ALIGNED_CACHE;
uint32_t loop_cnt; /*Counter to check pktio ingress queue dead loop */
} sched_scalable_thread_state_t;
void _odp_sched_update_enq(sched_elem_t *q, uint32_t actual);
void _odp_sched_update_enq_sp(sched_elem_t *q, uint32_t actual);
sched_queue_t *_odp_sched_queue_add(odp_schedule_group_t grp, uint32_t prio);
void _odp_sched_queue_rem(odp_schedule_group_t grp, uint32_t prio);
#endif /* ODP_SCHEDULE_SCALABLE_H */
|
