]>
Commit | Line | Data |
---|---|---|
9f95a23c | 1 | /* SPDX-License-Identifier: BSD-3-Clause |
7c673cae | 2 | * |
9f95a23c | 3 | * Copyright (c) 2010-2015 Intel Corporation |
7c673cae FG |
4 | * Copyright (c) 2007,2008 Kip Macy kmacy@freebsd.org |
5 | * All rights reserved. | |
9f95a23c TL |
6 | * Derived from FreeBSD's bufring.h |
7 | * Used as BSD-3 Licensed with permission from Kip Macy. | |
8 | */ | |
7c673cae FG |
9 | |
10 | #include <stdio.h> | |
11 | #include <stdarg.h> | |
12 | #include <string.h> | |
13 | #include <stdint.h> | |
14 | #include <inttypes.h> | |
15 | #include <errno.h> | |
16 | #include <sys/queue.h> | |
17 | ||
18 | #include <rte_common.h> | |
19 | #include <rte_log.h> | |
20 | #include <rte_memory.h> | |
21 | #include <rte_memzone.h> | |
22 | #include <rte_malloc.h> | |
23 | #include <rte_launch.h> | |
24 | #include <rte_eal.h> | |
25 | #include <rte_eal_memconfig.h> | |
26 | #include <rte_atomic.h> | |
27 | #include <rte_per_lcore.h> | |
28 | #include <rte_lcore.h> | |
29 | #include <rte_branch_prediction.h> | |
30 | #include <rte_errno.h> | |
31 | #include <rte_string_fns.h> | |
32 | #include <rte_spinlock.h> | |
33 | ||
34 | #include "rte_ring.h" | |
35 | ||
36 | TAILQ_HEAD(rte_ring_list, rte_tailq_entry); | |
37 | ||
38 | static struct rte_tailq_elem rte_ring_tailq = { | |
39 | .name = RTE_TAILQ_RING_NAME, | |
40 | }; | |
41 | EAL_REGISTER_TAILQ(rte_ring_tailq) | |
42 | ||
43 | /* true if x is a power of 2 */ | |
44 | #define POWEROF2(x) ((((x)-1) & (x)) == 0) | |
45 | ||
46 | /* return the size of memory occupied by a ring */ | |
47 | ssize_t | |
48 | rte_ring_get_memsize(unsigned count) | |
49 | { | |
50 | ssize_t sz; | |
51 | ||
52 | /* count must be a power of 2 */ | |
53 | if ((!POWEROF2(count)) || (count > RTE_RING_SZ_MASK )) { | |
54 | RTE_LOG(ERR, RING, | |
55 | "Requested size is invalid, must be power of 2, and " | |
56 | "do not exceed the size limit %u\n", RTE_RING_SZ_MASK); | |
57 | return -EINVAL; | |
58 | } | |
59 | ||
60 | sz = sizeof(struct rte_ring) + count * sizeof(void *); | |
61 | sz = RTE_ALIGN(sz, RTE_CACHE_LINE_SIZE); | |
62 | return sz; | |
63 | } | |
64 | ||
65 | int | |
66 | rte_ring_init(struct rte_ring *r, const char *name, unsigned count, | |
67 | unsigned flags) | |
68 | { | |
69 | int ret; | |
70 | ||
71 | /* compilation-time checks */ | |
72 | RTE_BUILD_BUG_ON((sizeof(struct rte_ring) & | |
73 | RTE_CACHE_LINE_MASK) != 0); | |
7c673cae FG |
74 | RTE_BUILD_BUG_ON((offsetof(struct rte_ring, cons) & |
75 | RTE_CACHE_LINE_MASK) != 0); | |
7c673cae FG |
76 | RTE_BUILD_BUG_ON((offsetof(struct rte_ring, prod) & |
77 | RTE_CACHE_LINE_MASK) != 0); | |
7c673cae FG |
78 | |
79 | /* init the ring structure */ | |
80 | memset(r, 0, sizeof(*r)); | |
9f95a23c | 81 | ret = strlcpy(r->name, name, sizeof(r->name)); |
7c673cae FG |
82 | if (ret < 0 || ret >= (int)sizeof(r->name)) |
83 | return -ENAMETOOLONG; | |
84 | r->flags = flags; | |
11fdf7f2 TL |
85 | r->prod.single = (flags & RING_F_SP_ENQ) ? __IS_SP : __IS_MP; |
86 | r->cons.single = (flags & RING_F_SC_DEQ) ? __IS_SC : __IS_MC; | |
9f95a23c TL |
87 | |
88 | if (flags & RING_F_EXACT_SZ) { | |
89 | r->size = rte_align32pow2(count + 1); | |
90 | r->mask = r->size - 1; | |
91 | r->capacity = count; | |
92 | } else { | |
93 | if ((!POWEROF2(count)) || (count > RTE_RING_SZ_MASK)) { | |
94 | RTE_LOG(ERR, RING, | |
95 | "Requested size is invalid, must be power of 2, and not exceed the size limit %u\n", | |
96 | RTE_RING_SZ_MASK); | |
97 | return -EINVAL; | |
98 | } | |
99 | r->size = count; | |
100 | r->mask = count - 1; | |
101 | r->capacity = r->mask; | |
102 | } | |
7c673cae FG |
103 | r->prod.head = r->cons.head = 0; |
104 | r->prod.tail = r->cons.tail = 0; | |
105 | ||
106 | return 0; | |
107 | } | |
108 | ||
109 | /* create the ring */ | |
110 | struct rte_ring * | |
111 | rte_ring_create(const char *name, unsigned count, int socket_id, | |
112 | unsigned flags) | |
113 | { | |
114 | char mz_name[RTE_MEMZONE_NAMESIZE]; | |
115 | struct rte_ring *r; | |
116 | struct rte_tailq_entry *te; | |
117 | const struct rte_memzone *mz; | |
118 | ssize_t ring_size; | |
119 | int mz_flags = 0; | |
120 | struct rte_ring_list* ring_list = NULL; | |
9f95a23c | 121 | const unsigned int requested_count = count; |
7c673cae FG |
122 | int ret; |
123 | ||
124 | ring_list = RTE_TAILQ_CAST(rte_ring_tailq.head, rte_ring_list); | |
125 | ||
9f95a23c TL |
126 | /* for an exact size ring, round up from count to a power of two */ |
127 | if (flags & RING_F_EXACT_SZ) | |
128 | count = rte_align32pow2(count + 1); | |
129 | ||
7c673cae FG |
130 | ring_size = rte_ring_get_memsize(count); |
131 | if (ring_size < 0) { | |
132 | rte_errno = ring_size; | |
133 | return NULL; | |
134 | } | |
135 | ||
136 | ret = snprintf(mz_name, sizeof(mz_name), "%s%s", | |
137 | RTE_RING_MZ_PREFIX, name); | |
138 | if (ret < 0 || ret >= (int)sizeof(mz_name)) { | |
139 | rte_errno = ENAMETOOLONG; | |
140 | return NULL; | |
141 | } | |
142 | ||
143 | te = rte_zmalloc("RING_TAILQ_ENTRY", sizeof(*te), 0); | |
144 | if (te == NULL) { | |
145 | RTE_LOG(ERR, RING, "Cannot reserve memory for tailq\n"); | |
146 | rte_errno = ENOMEM; | |
147 | return NULL; | |
148 | } | |
149 | ||
150 | rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK); | |
151 | ||
152 | /* reserve a memory zone for this ring. If we can't get rte_config or | |
153 | * we are secondary process, the memzone_reserve function will set | |
154 | * rte_errno for us appropriately - hence no check in this this function */ | |
9f95a23c TL |
155 | mz = rte_memzone_reserve_aligned(mz_name, ring_size, socket_id, |
156 | mz_flags, __alignof__(*r)); | |
7c673cae FG |
157 | if (mz != NULL) { |
158 | r = mz->addr; | |
159 | /* no need to check return value here, we already checked the | |
160 | * arguments above */ | |
9f95a23c | 161 | rte_ring_init(r, name, requested_count, flags); |
7c673cae FG |
162 | |
163 | te->data = (void *) r; | |
164 | r->memzone = mz; | |
165 | ||
166 | TAILQ_INSERT_TAIL(ring_list, te, next); | |
167 | } else { | |
168 | r = NULL; | |
169 | RTE_LOG(ERR, RING, "Cannot reserve memory\n"); | |
170 | rte_free(te); | |
171 | } | |
172 | rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK); | |
173 | ||
174 | return r; | |
175 | } | |
176 | ||
177 | /* free the ring */ | |
178 | void | |
179 | rte_ring_free(struct rte_ring *r) | |
180 | { | |
181 | struct rte_ring_list *ring_list = NULL; | |
182 | struct rte_tailq_entry *te; | |
183 | ||
184 | if (r == NULL) | |
185 | return; | |
186 | ||
187 | /* | |
188 | * Ring was not created with rte_ring_create, | |
189 | * therefore, there is no memzone to free. | |
190 | */ | |
191 | if (r->memzone == NULL) { | |
9f95a23c TL |
192 | RTE_LOG(ERR, RING, |
193 | "Cannot free ring, not created with rte_ring_create()\n"); | |
7c673cae FG |
194 | return; |
195 | } | |
196 | ||
197 | if (rte_memzone_free(r->memzone) != 0) { | |
198 | RTE_LOG(ERR, RING, "Cannot free memory\n"); | |
199 | return; | |
200 | } | |
201 | ||
202 | ring_list = RTE_TAILQ_CAST(rte_ring_tailq.head, rte_ring_list); | |
203 | rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK); | |
204 | ||
205 | /* find out tailq entry */ | |
206 | TAILQ_FOREACH(te, ring_list, next) { | |
207 | if (te->data == (void *) r) | |
208 | break; | |
209 | } | |
210 | ||
211 | if (te == NULL) { | |
212 | rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK); | |
213 | return; | |
214 | } | |
215 | ||
216 | TAILQ_REMOVE(ring_list, te, next); | |
217 | ||
218 | rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK); | |
219 | ||
220 | rte_free(te); | |
221 | } | |
222 | ||
7c673cae FG |
223 | /* dump the status of the ring on the console */ |
224 | void | |
225 | rte_ring_dump(FILE *f, const struct rte_ring *r) | |
226 | { | |
7c673cae FG |
227 | fprintf(f, "ring <%s>@%p\n", r->name, r); |
228 | fprintf(f, " flags=%x\n", r->flags); | |
11fdf7f2 | 229 | fprintf(f, " size=%"PRIu32"\n", r->size); |
9f95a23c | 230 | fprintf(f, " capacity=%"PRIu32"\n", r->capacity); |
7c673cae FG |
231 | fprintf(f, " ct=%"PRIu32"\n", r->cons.tail); |
232 | fprintf(f, " ch=%"PRIu32"\n", r->cons.head); | |
233 | fprintf(f, " pt=%"PRIu32"\n", r->prod.tail); | |
234 | fprintf(f, " ph=%"PRIu32"\n", r->prod.head); | |
235 | fprintf(f, " used=%u\n", rte_ring_count(r)); | |
236 | fprintf(f, " avail=%u\n", rte_ring_free_count(r)); | |
7c673cae FG |
237 | } |
238 | ||
239 | /* dump the status of all rings on the console */ | |
240 | void | |
241 | rte_ring_list_dump(FILE *f) | |
242 | { | |
243 | const struct rte_tailq_entry *te; | |
244 | struct rte_ring_list *ring_list; | |
245 | ||
246 | ring_list = RTE_TAILQ_CAST(rte_ring_tailq.head, rte_ring_list); | |
247 | ||
248 | rte_rwlock_read_lock(RTE_EAL_TAILQ_RWLOCK); | |
249 | ||
250 | TAILQ_FOREACH(te, ring_list, next) { | |
251 | rte_ring_dump(f, (struct rte_ring *) te->data); | |
252 | } | |
253 | ||
254 | rte_rwlock_read_unlock(RTE_EAL_TAILQ_RWLOCK); | |
255 | } | |
256 | ||
257 | /* search a ring from its name */ | |
258 | struct rte_ring * | |
259 | rte_ring_lookup(const char *name) | |
260 | { | |
261 | struct rte_tailq_entry *te; | |
262 | struct rte_ring *r = NULL; | |
263 | struct rte_ring_list *ring_list; | |
264 | ||
265 | ring_list = RTE_TAILQ_CAST(rte_ring_tailq.head, rte_ring_list); | |
266 | ||
267 | rte_rwlock_read_lock(RTE_EAL_TAILQ_RWLOCK); | |
268 | ||
269 | TAILQ_FOREACH(te, ring_list, next) { | |
270 | r = (struct rte_ring *) te->data; | |
271 | if (strncmp(name, r->name, RTE_RING_NAMESIZE) == 0) | |
272 | break; | |
273 | } | |
274 | ||
275 | rte_rwlock_read_unlock(RTE_EAL_TAILQ_RWLOCK); | |
276 | ||
277 | if (te == NULL) { | |
278 | rte_errno = ENOENT; | |
279 | return NULL; | |
280 | } | |
281 | ||
282 | return r; | |
283 | } |