]>
Commit | Line | Data |
---|---|---|
11fdf7f2 TL |
1 | /* SPDX-License-Identifier: BSD-3-Clause |
2 | * Copyright(c) 2017 Intel Corporation | |
3 | */ | |
4 | ||
5 | #include <stdio.h> | |
6 | #include <unistd.h> | |
7 | #include <inttypes.h> | |
8 | #include <limits.h> | |
9 | #include <string.h> | |
11fdf7f2 TL |
10 | |
11 | #include <rte_compat.h> | |
12 | #include <rte_service.h> | |
f67539c2 | 13 | #include <rte_service_component.h> |
11fdf7f2 TL |
14 | |
15 | #include <rte_eal.h> | |
16 | #include <rte_lcore.h> | |
17 | #include <rte_common.h> | |
18 | #include <rte_debug.h> | |
19 | #include <rte_cycles.h> | |
20 | #include <rte_atomic.h> | |
21 | #include <rte_memory.h> | |
22 | #include <rte_malloc.h> | |
f67539c2 TL |
23 | #include <rte_spinlock.h> |
24 | ||
25 | #include "eal_private.h" | |
11fdf7f2 TL |
26 | |
27 | #define RTE_SERVICE_NUM_MAX 64 | |
28 | ||
29 | #define SERVICE_F_REGISTERED (1 << 0) | |
30 | #define SERVICE_F_STATS_ENABLED (1 << 1) | |
31 | #define SERVICE_F_START_CHECK (1 << 2) | |
32 | ||
33 | /* runstates for services and lcores, denoting if they are active or not */ | |
34 | #define RUNSTATE_STOPPED 0 | |
35 | #define RUNSTATE_RUNNING 1 | |
36 | ||
37 | /* internal representation of a service */ | |
38 | struct rte_service_spec_impl { | |
39 | /* public part of the struct */ | |
40 | struct rte_service_spec spec; | |
41 | ||
f67539c2 | 42 | /* spin lock that when set indicates a service core is currently |
11fdf7f2 TL |
43 | * running this service callback. When not set, a core may take the |
44 | * lock and then run the service callback. | |
45 | */ | |
f67539c2 | 46 | rte_spinlock_t execute_lock; |
11fdf7f2 TL |
47 | |
48 | /* API set/get-able variables */ | |
49 | int8_t app_runstate; | |
50 | int8_t comp_runstate; | |
51 | uint8_t internal_flags; | |
52 | ||
53 | /* per service statistics */ | |
f67539c2 TL |
54 | /* Indicates how many cores the service is mapped to run on. |
55 | * It does not indicate the number of cores the service is running | |
56 | * on currently. | |
57 | */ | |
58 | uint32_t num_mapped_cores; | |
11fdf7f2 TL |
59 | uint64_t calls; |
60 | uint64_t cycles_spent; | |
11fdf7f2 TL |
61 | } __rte_cache_aligned; |
62 | ||
63 | /* the internal values of a service core */ | |
64 | struct core_state { | |
65 | /* map of services IDs are run on this core */ | |
66 | uint64_t service_mask; | |
67 | uint8_t runstate; /* running or stopped */ | |
68 | uint8_t is_service_core; /* set if core is currently a service core */ | |
f67539c2 | 69 | uint8_t service_active_on_lcore[RTE_SERVICE_NUM_MAX]; |
11fdf7f2 TL |
70 | uint64_t loops; |
71 | uint64_t calls_per_service[RTE_SERVICE_NUM_MAX]; | |
72 | } __rte_cache_aligned; | |
73 | ||
74 | static uint32_t rte_service_count; | |
75 | static struct rte_service_spec_impl *rte_services; | |
76 | static struct core_state *lcore_states; | |
77 | static uint32_t rte_service_library_initialized; | |
78 | ||
f67539c2 TL |
79 | int32_t |
80 | rte_service_init(void) | |
11fdf7f2 TL |
81 | { |
82 | if (rte_service_library_initialized) { | |
f67539c2 TL |
83 | RTE_LOG(NOTICE, EAL, |
84 | "service library init() called, init flag %d\n", | |
11fdf7f2 TL |
85 | rte_service_library_initialized); |
86 | return -EALREADY; | |
87 | } | |
88 | ||
89 | rte_services = rte_calloc("rte_services", RTE_SERVICE_NUM_MAX, | |
90 | sizeof(struct rte_service_spec_impl), | |
91 | RTE_CACHE_LINE_SIZE); | |
92 | if (!rte_services) { | |
f67539c2 | 93 | RTE_LOG(ERR, EAL, "error allocating rte services array\n"); |
11fdf7f2 TL |
94 | goto fail_mem; |
95 | } | |
96 | ||
97 | lcore_states = rte_calloc("rte_service_core_states", RTE_MAX_LCORE, | |
98 | sizeof(struct core_state), RTE_CACHE_LINE_SIZE); | |
99 | if (!lcore_states) { | |
f67539c2 | 100 | RTE_LOG(ERR, EAL, "error allocating core states array\n"); |
11fdf7f2 TL |
101 | goto fail_mem; |
102 | } | |
103 | ||
104 | int i; | |
105 | int count = 0; | |
106 | struct rte_config *cfg = rte_eal_get_configuration(); | |
107 | for (i = 0; i < RTE_MAX_LCORE; i++) { | |
108 | if (lcore_config[i].core_role == ROLE_SERVICE) { | |
109 | if ((unsigned int)i == cfg->master_lcore) | |
110 | continue; | |
111 | rte_service_lcore_add(i); | |
112 | count++; | |
113 | } | |
114 | } | |
115 | ||
116 | rte_service_library_initialized = 1; | |
117 | return 0; | |
118 | fail_mem: | |
f67539c2 TL |
119 | rte_free(rte_services); |
120 | rte_free(lcore_states); | |
11fdf7f2 TL |
121 | return -ENOMEM; |
122 | } | |
123 | ||
124 | void | |
125 | rte_service_finalize(void) | |
126 | { | |
127 | if (!rte_service_library_initialized) | |
128 | return; | |
129 | ||
f67539c2 TL |
130 | rte_service_lcore_reset_all(); |
131 | rte_eal_mp_wait_lcore(); | |
11fdf7f2 | 132 | |
f67539c2 TL |
133 | rte_free(rte_services); |
134 | rte_free(lcore_states); | |
11fdf7f2 TL |
135 | |
136 | rte_service_library_initialized = 0; | |
137 | } | |
138 | ||
139 | /* returns 1 if service is registered and has not been unregistered | |
140 | * Returns 0 if service never registered, or has been unregistered | |
141 | */ | |
142 | static inline int | |
143 | service_valid(uint32_t id) | |
144 | { | |
145 | return !!(rte_services[id].internal_flags & SERVICE_F_REGISTERED); | |
146 | } | |
147 | ||
f67539c2 TL |
148 | static struct rte_service_spec_impl * |
149 | service_get(uint32_t id) | |
150 | { | |
151 | return &rte_services[id]; | |
152 | } | |
153 | ||
11fdf7f2 TL |
154 | /* validate ID and retrieve service pointer, or return error value */ |
155 | #define SERVICE_VALID_GET_OR_ERR_RET(id, service, retval) do { \ | |
156 | if (id >= RTE_SERVICE_NUM_MAX || !service_valid(id)) \ | |
157 | return retval; \ | |
158 | service = &rte_services[id]; \ | |
159 | } while (0) | |
160 | ||
161 | /* returns 1 if statistics should be collected for service | |
162 | * Returns 0 if statistics should not be collected for service | |
163 | */ | |
164 | static inline int | |
165 | service_stats_enabled(struct rte_service_spec_impl *impl) | |
166 | { | |
167 | return !!(impl->internal_flags & SERVICE_F_STATS_ENABLED); | |
168 | } | |
169 | ||
170 | static inline int | |
171 | service_mt_safe(struct rte_service_spec_impl *s) | |
172 | { | |
173 | return !!(s->spec.capabilities & RTE_SERVICE_CAP_MT_SAFE); | |
174 | } | |
175 | ||
176 | int32_t | |
177 | rte_service_set_stats_enable(uint32_t id, int32_t enabled) | |
178 | { | |
179 | struct rte_service_spec_impl *s; | |
180 | SERVICE_VALID_GET_OR_ERR_RET(id, s, 0); | |
181 | ||
182 | if (enabled) | |
183 | s->internal_flags |= SERVICE_F_STATS_ENABLED; | |
184 | else | |
185 | s->internal_flags &= ~(SERVICE_F_STATS_ENABLED); | |
186 | ||
187 | return 0; | |
188 | } | |
189 | ||
190 | int32_t | |
191 | rte_service_set_runstate_mapped_check(uint32_t id, int32_t enabled) | |
192 | { | |
193 | struct rte_service_spec_impl *s; | |
194 | SERVICE_VALID_GET_OR_ERR_RET(id, s, 0); | |
195 | ||
196 | if (enabled) | |
197 | s->internal_flags |= SERVICE_F_START_CHECK; | |
198 | else | |
199 | s->internal_flags &= ~(SERVICE_F_START_CHECK); | |
200 | ||
201 | return 0; | |
202 | } | |
203 | ||
204 | uint32_t | |
205 | rte_service_get_count(void) | |
206 | { | |
207 | return rte_service_count; | |
208 | } | |
209 | ||
210 | int32_t | |
211 | rte_service_get_by_name(const char *name, uint32_t *service_id) | |
212 | { | |
213 | if (!service_id) | |
214 | return -EINVAL; | |
215 | ||
216 | int i; | |
217 | for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) { | |
218 | if (service_valid(i) && | |
219 | strcmp(name, rte_services[i].spec.name) == 0) { | |
220 | *service_id = i; | |
221 | return 0; | |
222 | } | |
223 | } | |
224 | ||
225 | return -ENODEV; | |
226 | } | |
227 | ||
228 | const char * | |
229 | rte_service_get_name(uint32_t id) | |
230 | { | |
231 | struct rte_service_spec_impl *s; | |
232 | SERVICE_VALID_GET_OR_ERR_RET(id, s, 0); | |
233 | return s->spec.name; | |
234 | } | |
235 | ||
236 | int32_t | |
237 | rte_service_probe_capability(uint32_t id, uint32_t capability) | |
238 | { | |
239 | struct rte_service_spec_impl *s; | |
240 | SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL); | |
241 | return !!(s->spec.capabilities & capability); | |
242 | } | |
243 | ||
244 | int32_t | |
245 | rte_service_component_register(const struct rte_service_spec *spec, | |
246 | uint32_t *id_ptr) | |
247 | { | |
248 | uint32_t i; | |
249 | int32_t free_slot = -1; | |
250 | ||
251 | if (spec->callback == NULL || strlen(spec->name) == 0) | |
252 | return -EINVAL; | |
253 | ||
254 | for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) { | |
255 | if (!service_valid(i)) { | |
256 | free_slot = i; | |
257 | break; | |
258 | } | |
259 | } | |
260 | ||
261 | if ((free_slot < 0) || (i == RTE_SERVICE_NUM_MAX)) | |
262 | return -ENOSPC; | |
263 | ||
264 | struct rte_service_spec_impl *s = &rte_services[free_slot]; | |
265 | s->spec = *spec; | |
266 | s->internal_flags |= SERVICE_F_REGISTERED | SERVICE_F_START_CHECK; | |
267 | ||
11fdf7f2 TL |
268 | rte_service_count++; |
269 | ||
270 | if (id_ptr) | |
271 | *id_ptr = free_slot; | |
272 | ||
273 | return 0; | |
274 | } | |
275 | ||
276 | int32_t | |
277 | rte_service_component_unregister(uint32_t id) | |
278 | { | |
279 | uint32_t i; | |
280 | struct rte_service_spec_impl *s; | |
281 | SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL); | |
282 | ||
283 | rte_service_count--; | |
11fdf7f2 TL |
284 | |
285 | s->internal_flags &= ~(SERVICE_F_REGISTERED); | |
286 | ||
287 | /* clear the run-bit in all cores */ | |
288 | for (i = 0; i < RTE_MAX_LCORE; i++) | |
289 | lcore_states[i].service_mask &= ~(UINT64_C(1) << id); | |
290 | ||
291 | memset(&rte_services[id], 0, sizeof(struct rte_service_spec_impl)); | |
292 | ||
293 | return 0; | |
294 | } | |
295 | ||
296 | int32_t | |
297 | rte_service_component_runstate_set(uint32_t id, uint32_t runstate) | |
298 | { | |
299 | struct rte_service_spec_impl *s; | |
300 | SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL); | |
301 | ||
f67539c2 TL |
302 | /* comp_runstate act as the guard variable. Use store-release |
303 | * memory order. This synchronizes with load-acquire in | |
304 | * service_run and service_runstate_get function. | |
305 | */ | |
11fdf7f2 | 306 | if (runstate) |
f67539c2 TL |
307 | __atomic_store_n(&s->comp_runstate, RUNSTATE_RUNNING, |
308 | __ATOMIC_RELEASE); | |
11fdf7f2 | 309 | else |
f67539c2 TL |
310 | __atomic_store_n(&s->comp_runstate, RUNSTATE_STOPPED, |
311 | __ATOMIC_RELEASE); | |
11fdf7f2 | 312 | |
11fdf7f2 TL |
313 | return 0; |
314 | } | |
315 | ||
316 | int32_t | |
317 | rte_service_runstate_set(uint32_t id, uint32_t runstate) | |
318 | { | |
319 | struct rte_service_spec_impl *s; | |
320 | SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL); | |
321 | ||
f67539c2 TL |
322 | /* app_runstate act as the guard variable. Use store-release |
323 | * memory order. This synchronizes with load-acquire in | |
324 | * service_run runstate_get function. | |
325 | */ | |
11fdf7f2 | 326 | if (runstate) |
f67539c2 TL |
327 | __atomic_store_n(&s->app_runstate, RUNSTATE_RUNNING, |
328 | __ATOMIC_RELEASE); | |
11fdf7f2 | 329 | else |
f67539c2 TL |
330 | __atomic_store_n(&s->app_runstate, RUNSTATE_STOPPED, |
331 | __ATOMIC_RELEASE); | |
11fdf7f2 | 332 | |
11fdf7f2 TL |
333 | return 0; |
334 | } | |
335 | ||
336 | int32_t | |
337 | rte_service_runstate_get(uint32_t id) | |
338 | { | |
339 | struct rte_service_spec_impl *s; | |
340 | SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL); | |
11fdf7f2 | 341 | |
f67539c2 TL |
342 | /* comp_runstate and app_runstate act as the guard variables. |
343 | * Use load-acquire memory order. This synchronizes with | |
344 | * store-release in service state set functions. | |
345 | */ | |
346 | if (__atomic_load_n(&s->comp_runstate, __ATOMIC_ACQUIRE) == | |
347 | RUNSTATE_RUNNING && | |
348 | __atomic_load_n(&s->app_runstate, __ATOMIC_ACQUIRE) == | |
349 | RUNSTATE_RUNNING) { | |
350 | int check_disabled = !(s->internal_flags & | |
351 | SERVICE_F_START_CHECK); | |
352 | int lcore_mapped = (__atomic_load_n(&s->num_mapped_cores, | |
353 | __ATOMIC_RELAXED) > 0); | |
354 | ||
355 | return (check_disabled | lcore_mapped); | |
356 | } else | |
357 | return 0; | |
11fdf7f2 | 358 | |
11fdf7f2 TL |
359 | } |
360 | ||
361 | static inline void | |
f67539c2 TL |
362 | service_runner_do_callback(struct rte_service_spec_impl *s, |
363 | struct core_state *cs, uint32_t service_idx) | |
11fdf7f2 TL |
364 | { |
365 | void *userdata = s->spec.callback_userdata; | |
366 | ||
367 | if (service_stats_enabled(s)) { | |
368 | uint64_t start = rte_rdtsc(); | |
369 | s->spec.callback(userdata); | |
370 | uint64_t end = rte_rdtsc(); | |
371 | s->cycles_spent += end - start; | |
372 | cs->calls_per_service[service_idx]++; | |
373 | s->calls++; | |
374 | } else | |
375 | s->spec.callback(userdata); | |
376 | } | |
377 | ||
378 | ||
f67539c2 TL |
379 | /* Expects the service 's' is valid. */ |
380 | static int32_t | |
381 | service_run(uint32_t i, struct core_state *cs, uint64_t service_mask, | |
382 | struct rte_service_spec_impl *s, uint32_t serialize_mt_unsafe) | |
11fdf7f2 | 383 | { |
f67539c2 | 384 | if (!s) |
11fdf7f2 | 385 | return -EINVAL; |
f67539c2 TL |
386 | |
387 | /* comp_runstate and app_runstate act as the guard variables. | |
388 | * Use load-acquire memory order. This synchronizes with | |
389 | * store-release in service state set functions. | |
390 | */ | |
391 | if (__atomic_load_n(&s->comp_runstate, __ATOMIC_ACQUIRE) != | |
392 | RUNSTATE_RUNNING || | |
393 | __atomic_load_n(&s->app_runstate, __ATOMIC_ACQUIRE) != | |
394 | RUNSTATE_RUNNING || | |
395 | !(service_mask & (UINT64_C(1) << i))) { | |
396 | cs->service_active_on_lcore[i] = 0; | |
11fdf7f2 TL |
397 | return -ENOEXEC; |
398 | } | |
399 | ||
f67539c2 | 400 | cs->service_active_on_lcore[i] = 1; |
11fdf7f2 | 401 | |
f67539c2 TL |
402 | if ((service_mt_safe(s) == 0) && (serialize_mt_unsafe == 1)) { |
403 | if (!rte_spinlock_trylock(&s->execute_lock)) | |
11fdf7f2 TL |
404 | return -EBUSY; |
405 | ||
f67539c2 TL |
406 | service_runner_do_callback(s, cs, i); |
407 | rte_spinlock_unlock(&s->execute_lock); | |
11fdf7f2 | 408 | } else |
f67539c2 | 409 | service_runner_do_callback(s, cs, i); |
11fdf7f2 TL |
410 | |
411 | return 0; | |
412 | } | |
413 | ||
f67539c2 | 414 | int32_t |
11fdf7f2 TL |
415 | rte_service_may_be_active(uint32_t id) |
416 | { | |
417 | uint32_t ids[RTE_MAX_LCORE] = {0}; | |
11fdf7f2 TL |
418 | int32_t lcore_count = rte_service_lcore_list(ids, RTE_MAX_LCORE); |
419 | int i; | |
420 | ||
f67539c2 | 421 | if (id >= RTE_SERVICE_NUM_MAX || !service_valid(id)) |
11fdf7f2 TL |
422 | return -EINVAL; |
423 | ||
424 | for (i = 0; i < lcore_count; i++) { | |
f67539c2 | 425 | if (lcore_states[i].service_active_on_lcore[id]) |
11fdf7f2 TL |
426 | return 1; |
427 | } | |
428 | ||
429 | return 0; | |
430 | } | |
431 | ||
f67539c2 TL |
432 | int32_t |
433 | rte_service_run_iter_on_app_lcore(uint32_t id, uint32_t serialize_mt_unsafe) | |
11fdf7f2 | 434 | { |
11fdf7f2 | 435 | struct core_state *cs = &lcore_states[rte_lcore_id()]; |
f67539c2 | 436 | struct rte_service_spec_impl *s; |
11fdf7f2 | 437 | |
f67539c2 | 438 | SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL); |
11fdf7f2 | 439 | |
f67539c2 TL |
440 | /* Increment num_mapped_cores to reflect that this core is |
441 | * now mapped capable of running the service. | |
442 | */ | |
443 | __atomic_add_fetch(&s->num_mapped_cores, 1, __ATOMIC_RELAXED); | |
11fdf7f2 | 444 | |
f67539c2 | 445 | int ret = service_run(id, cs, UINT64_MAX, s, serialize_mt_unsafe); |
11fdf7f2 | 446 | |
f67539c2 | 447 | __atomic_sub_fetch(&s->num_mapped_cores, 1, __ATOMIC_RELAXED); |
11fdf7f2 TL |
448 | |
449 | return ret; | |
450 | } | |
451 | ||
452 | static int32_t | |
f67539c2 | 453 | service_runner_func(void *arg) |
11fdf7f2 TL |
454 | { |
455 | RTE_SET_USED(arg); | |
456 | uint32_t i; | |
457 | const int lcore = rte_lcore_id(); | |
458 | struct core_state *cs = &lcore_states[lcore]; | |
459 | ||
f67539c2 TL |
460 | /* runstate act as the guard variable. Use load-acquire |
461 | * memory order here to synchronize with store-release | |
462 | * in runstate update functions. | |
463 | */ | |
464 | while (__atomic_load_n(&cs->runstate, __ATOMIC_ACQUIRE) == | |
465 | RUNSTATE_RUNNING) { | |
11fdf7f2 TL |
466 | const uint64_t service_mask = cs->service_mask; |
467 | ||
468 | for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) { | |
f67539c2 TL |
469 | if (!service_valid(i)) |
470 | continue; | |
11fdf7f2 | 471 | /* return value ignored as no change to code flow */ |
f67539c2 | 472 | service_run(i, cs, service_mask, service_get(i), 1); |
11fdf7f2 TL |
473 | } |
474 | ||
475 | cs->loops++; | |
11fdf7f2 TL |
476 | } |
477 | ||
478 | lcore_config[lcore].state = WAIT; | |
479 | ||
480 | return 0; | |
481 | } | |
482 | ||
483 | int32_t | |
484 | rte_service_lcore_count(void) | |
485 | { | |
486 | int32_t count = 0; | |
487 | uint32_t i; | |
488 | for (i = 0; i < RTE_MAX_LCORE; i++) | |
489 | count += lcore_states[i].is_service_core; | |
490 | return count; | |
491 | } | |
492 | ||
493 | int32_t | |
494 | rte_service_lcore_list(uint32_t array[], uint32_t n) | |
495 | { | |
496 | uint32_t count = rte_service_lcore_count(); | |
497 | if (count > n) | |
498 | return -ENOMEM; | |
499 | ||
500 | if (!array) | |
501 | return -EINVAL; | |
502 | ||
503 | uint32_t i; | |
504 | uint32_t idx = 0; | |
505 | for (i = 0; i < RTE_MAX_LCORE; i++) { | |
506 | struct core_state *cs = &lcore_states[i]; | |
507 | if (cs->is_service_core) { | |
508 | array[idx] = i; | |
509 | idx++; | |
510 | } | |
511 | } | |
512 | ||
513 | return count; | |
514 | } | |
515 | ||
516 | int32_t | |
517 | rte_service_lcore_count_services(uint32_t lcore) | |
518 | { | |
519 | if (lcore >= RTE_MAX_LCORE) | |
520 | return -EINVAL; | |
521 | ||
522 | struct core_state *cs = &lcore_states[lcore]; | |
523 | if (!cs->is_service_core) | |
524 | return -ENOTSUP; | |
525 | ||
526 | return __builtin_popcountll(cs->service_mask); | |
527 | } | |
528 | ||
529 | int32_t | |
530 | rte_service_start_with_defaults(void) | |
531 | { | |
532 | /* create a default mapping from cores to services, then start the | |
533 | * services to make them transparent to unaware applications. | |
534 | */ | |
535 | uint32_t i; | |
536 | int ret; | |
537 | uint32_t count = rte_service_get_count(); | |
538 | ||
539 | int32_t lcore_iter = 0; | |
540 | uint32_t ids[RTE_MAX_LCORE] = {0}; | |
541 | int32_t lcore_count = rte_service_lcore_list(ids, RTE_MAX_LCORE); | |
542 | ||
543 | if (lcore_count == 0) | |
544 | return -ENOTSUP; | |
545 | ||
546 | for (i = 0; (int)i < lcore_count; i++) | |
547 | rte_service_lcore_start(ids[i]); | |
548 | ||
549 | for (i = 0; i < count; i++) { | |
550 | /* do 1:1 core mapping here, with each service getting | |
551 | * assigned a single core by default. Adding multiple services | |
552 | * should multiplex to a single core, or 1:1 if there are the | |
553 | * same amount of services as service-cores | |
554 | */ | |
555 | ret = rte_service_map_lcore_set(i, ids[lcore_iter], 1); | |
556 | if (ret) | |
557 | return -ENODEV; | |
558 | ||
559 | lcore_iter++; | |
560 | if (lcore_iter >= lcore_count) | |
561 | lcore_iter = 0; | |
562 | ||
563 | ret = rte_service_runstate_set(i, 1); | |
564 | if (ret) | |
565 | return -ENOEXEC; | |
566 | } | |
567 | ||
568 | return 0; | |
569 | } | |
570 | ||
571 | static int32_t | |
f67539c2 | 572 | service_update(uint32_t sid, uint32_t lcore, uint32_t *set, uint32_t *enabled) |
11fdf7f2 | 573 | { |
f67539c2 TL |
574 | /* validate ID, or return error value */ |
575 | if (sid >= RTE_SERVICE_NUM_MAX || !service_valid(sid) || | |
576 | lcore >= RTE_MAX_LCORE || !lcore_states[lcore].is_service_core) | |
11fdf7f2 TL |
577 | return -EINVAL; |
578 | ||
579 | uint64_t sid_mask = UINT64_C(1) << sid; | |
580 | if (set) { | |
581 | uint64_t lcore_mapped = lcore_states[lcore].service_mask & | |
582 | sid_mask; | |
583 | ||
584 | if (*set && !lcore_mapped) { | |
585 | lcore_states[lcore].service_mask |= sid_mask; | |
f67539c2 TL |
586 | __atomic_add_fetch(&rte_services[sid].num_mapped_cores, |
587 | 1, __ATOMIC_RELAXED); | |
11fdf7f2 TL |
588 | } |
589 | if (!*set && lcore_mapped) { | |
590 | lcore_states[lcore].service_mask &= ~(sid_mask); | |
f67539c2 TL |
591 | __atomic_sub_fetch(&rte_services[sid].num_mapped_cores, |
592 | 1, __ATOMIC_RELAXED); | |
11fdf7f2 TL |
593 | } |
594 | } | |
595 | ||
596 | if (enabled) | |
597 | *enabled = !!(lcore_states[lcore].service_mask & (sid_mask)); | |
598 | ||
11fdf7f2 TL |
599 | return 0; |
600 | } | |
601 | ||
602 | int32_t | |
603 | rte_service_map_lcore_set(uint32_t id, uint32_t lcore, uint32_t enabled) | |
604 | { | |
11fdf7f2 | 605 | uint32_t on = enabled > 0; |
f67539c2 | 606 | return service_update(id, lcore, &on, 0); |
11fdf7f2 TL |
607 | } |
608 | ||
609 | int32_t | |
610 | rte_service_map_lcore_get(uint32_t id, uint32_t lcore) | |
611 | { | |
11fdf7f2 | 612 | uint32_t enabled; |
f67539c2 | 613 | int ret = service_update(id, lcore, 0, &enabled); |
11fdf7f2 TL |
614 | if (ret == 0) |
615 | return enabled; | |
616 | return ret; | |
617 | } | |
618 | ||
619 | static void | |
620 | set_lcore_state(uint32_t lcore, int32_t state) | |
621 | { | |
622 | /* mark core state in hugepage backed config */ | |
623 | struct rte_config *cfg = rte_eal_get_configuration(); | |
624 | cfg->lcore_role[lcore] = state; | |
625 | ||
626 | /* mark state in process local lcore_config */ | |
627 | lcore_config[lcore].core_role = state; | |
628 | ||
629 | /* update per-lcore optimized state tracking */ | |
630 | lcore_states[lcore].is_service_core = (state == ROLE_SERVICE); | |
631 | } | |
632 | ||
633 | int32_t | |
634 | rte_service_lcore_reset_all(void) | |
635 | { | |
636 | /* loop over cores, reset all to mask 0 */ | |
637 | uint32_t i; | |
638 | for (i = 0; i < RTE_MAX_LCORE; i++) { | |
639 | if (lcore_states[i].is_service_core) { | |
640 | lcore_states[i].service_mask = 0; | |
641 | set_lcore_state(i, ROLE_RTE); | |
f67539c2 TL |
642 | /* runstate act as guard variable Use |
643 | * store-release memory order here to synchronize | |
644 | * with load-acquire in runstate read functions. | |
645 | */ | |
646 | __atomic_store_n(&lcore_states[i].runstate, | |
647 | RUNSTATE_STOPPED, __ATOMIC_RELEASE); | |
11fdf7f2 TL |
648 | } |
649 | } | |
650 | for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) | |
f67539c2 TL |
651 | __atomic_store_n(&rte_services[i].num_mapped_cores, 0, |
652 | __ATOMIC_RELAXED); | |
11fdf7f2 TL |
653 | |
654 | return 0; | |
655 | } | |
656 | ||
657 | int32_t | |
658 | rte_service_lcore_add(uint32_t lcore) | |
659 | { | |
660 | if (lcore >= RTE_MAX_LCORE) | |
661 | return -EINVAL; | |
662 | if (lcore_states[lcore].is_service_core) | |
663 | return -EALREADY; | |
664 | ||
665 | set_lcore_state(lcore, ROLE_SERVICE); | |
666 | ||
667 | /* ensure that after adding a core the mask and state are defaults */ | |
668 | lcore_states[lcore].service_mask = 0; | |
f67539c2 TL |
669 | /* Use store-release memory order here to synchronize with |
670 | * load-acquire in runstate read functions. | |
671 | */ | |
672 | __atomic_store_n(&lcore_states[lcore].runstate, RUNSTATE_STOPPED, | |
673 | __ATOMIC_RELEASE); | |
11fdf7f2 TL |
674 | |
675 | return rte_eal_wait_lcore(lcore); | |
676 | } | |
677 | ||
678 | int32_t | |
679 | rte_service_lcore_del(uint32_t lcore) | |
680 | { | |
681 | if (lcore >= RTE_MAX_LCORE) | |
682 | return -EINVAL; | |
683 | ||
684 | struct core_state *cs = &lcore_states[lcore]; | |
685 | if (!cs->is_service_core) | |
686 | return -EINVAL; | |
687 | ||
f67539c2 TL |
688 | /* runstate act as the guard variable. Use load-acquire |
689 | * memory order here to synchronize with store-release | |
690 | * in runstate update functions. | |
691 | */ | |
692 | if (__atomic_load_n(&cs->runstate, __ATOMIC_ACQUIRE) != | |
693 | RUNSTATE_STOPPED) | |
11fdf7f2 TL |
694 | return -EBUSY; |
695 | ||
696 | set_lcore_state(lcore, ROLE_RTE); | |
697 | ||
698 | rte_smp_wmb(); | |
699 | return 0; | |
700 | } | |
701 | ||
702 | int32_t | |
703 | rte_service_lcore_start(uint32_t lcore) | |
704 | { | |
705 | if (lcore >= RTE_MAX_LCORE) | |
706 | return -EINVAL; | |
707 | ||
708 | struct core_state *cs = &lcore_states[lcore]; | |
709 | if (!cs->is_service_core) | |
710 | return -EINVAL; | |
711 | ||
f67539c2 TL |
712 | /* runstate act as the guard variable. Use load-acquire |
713 | * memory order here to synchronize with store-release | |
714 | * in runstate update functions. | |
715 | */ | |
716 | if (__atomic_load_n(&cs->runstate, __ATOMIC_ACQUIRE) == | |
717 | RUNSTATE_RUNNING) | |
11fdf7f2 TL |
718 | return -EALREADY; |
719 | ||
720 | /* set core to run state first, and then launch otherwise it will | |
721 | * return immediately as runstate keeps it in the service poll loop | |
722 | */ | |
f67539c2 TL |
723 | /* Use load-acquire memory order here to synchronize with |
724 | * store-release in runstate update functions. | |
725 | */ | |
726 | __atomic_store_n(&cs->runstate, RUNSTATE_RUNNING, __ATOMIC_RELEASE); | |
11fdf7f2 | 727 | |
f67539c2 | 728 | int ret = rte_eal_remote_launch(service_runner_func, 0, lcore); |
11fdf7f2 TL |
729 | /* returns -EBUSY if the core is already launched, 0 on success */ |
730 | return ret; | |
731 | } | |
732 | ||
733 | int32_t | |
734 | rte_service_lcore_stop(uint32_t lcore) | |
735 | { | |
736 | if (lcore >= RTE_MAX_LCORE) | |
737 | return -EINVAL; | |
738 | ||
f67539c2 TL |
739 | /* runstate act as the guard variable. Use load-acquire |
740 | * memory order here to synchronize with store-release | |
741 | * in runstate update functions. | |
742 | */ | |
743 | if (__atomic_load_n(&lcore_states[lcore].runstate, __ATOMIC_ACQUIRE) == | |
744 | RUNSTATE_STOPPED) | |
11fdf7f2 TL |
745 | return -EALREADY; |
746 | ||
747 | uint32_t i; | |
748 | uint64_t service_mask = lcore_states[lcore].service_mask; | |
749 | for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) { | |
750 | int32_t enabled = service_mask & (UINT64_C(1) << i); | |
751 | int32_t service_running = rte_service_runstate_get(i); | |
752 | int32_t only_core = (1 == | |
f67539c2 TL |
753 | __atomic_load_n(&rte_services[i].num_mapped_cores, |
754 | __ATOMIC_RELAXED)); | |
11fdf7f2 TL |
755 | |
756 | /* if the core is mapped, and the service is running, and this | |
757 | * is the only core that is mapped, the service would cease to | |
758 | * run if this core stopped, so fail instead. | |
759 | */ | |
760 | if (enabled && service_running && only_core) | |
761 | return -EBUSY; | |
762 | } | |
763 | ||
f67539c2 TL |
764 | /* Use store-release memory order here to synchronize with |
765 | * load-acquire in runstate read functions. | |
766 | */ | |
767 | __atomic_store_n(&lcore_states[lcore].runstate, RUNSTATE_STOPPED, | |
768 | __ATOMIC_RELEASE); | |
11fdf7f2 TL |
769 | |
770 | return 0; | |
771 | } | |
772 | ||
773 | int32_t | |
9f95a23c | 774 | rte_service_attr_get(uint32_t id, uint32_t attr_id, uint64_t *attr_value) |
11fdf7f2 TL |
775 | { |
776 | struct rte_service_spec_impl *s; | |
777 | SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL); | |
778 | ||
779 | if (!attr_value) | |
780 | return -EINVAL; | |
781 | ||
782 | switch (attr_id) { | |
783 | case RTE_SERVICE_ATTR_CYCLES: | |
784 | *attr_value = s->cycles_spent; | |
785 | return 0; | |
786 | case RTE_SERVICE_ATTR_CALL_COUNT: | |
787 | *attr_value = s->calls; | |
788 | return 0; | |
789 | default: | |
790 | return -EINVAL; | |
791 | } | |
792 | } | |
793 | ||
f67539c2 | 794 | int32_t |
11fdf7f2 TL |
795 | rte_service_lcore_attr_get(uint32_t lcore, uint32_t attr_id, |
796 | uint64_t *attr_value) | |
797 | { | |
798 | struct core_state *cs; | |
799 | ||
800 | if (lcore >= RTE_MAX_LCORE || !attr_value) | |
801 | return -EINVAL; | |
802 | ||
803 | cs = &lcore_states[lcore]; | |
804 | if (!cs->is_service_core) | |
805 | return -ENOTSUP; | |
806 | ||
807 | switch (attr_id) { | |
808 | case RTE_SERVICE_LCORE_ATTR_LOOPS: | |
809 | *attr_value = cs->loops; | |
810 | return 0; | |
811 | default: | |
812 | return -EINVAL; | |
813 | } | |
814 | } | |
815 | ||
816 | static void | |
f67539c2 | 817 | service_dump_one(FILE *f, struct rte_service_spec_impl *s, uint32_t reset) |
11fdf7f2 TL |
818 | { |
819 | /* avoid divide by zero */ | |
11fdf7f2 TL |
820 | int calls = 1; |
821 | if (s->calls != 0) | |
822 | calls = s->calls; | |
823 | ||
824 | if (reset) { | |
825 | s->cycles_spent = 0; | |
826 | s->calls = 0; | |
827 | return; | |
828 | } | |
829 | ||
9f95a23c TL |
830 | if (f == NULL) |
831 | return; | |
832 | ||
11fdf7f2 TL |
833 | fprintf(f, " %s: stats %d\tcalls %"PRIu64"\tcycles %" |
834 | PRIu64"\tavg: %"PRIu64"\n", | |
835 | s->spec.name, service_stats_enabled(s), s->calls, | |
836 | s->cycles_spent, s->cycles_spent / calls); | |
837 | } | |
838 | ||
839 | int32_t | |
840 | rte_service_attr_reset_all(uint32_t id) | |
841 | { | |
842 | struct rte_service_spec_impl *s; | |
843 | SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL); | |
844 | ||
845 | int reset = 1; | |
f67539c2 | 846 | service_dump_one(NULL, s, reset); |
11fdf7f2 TL |
847 | return 0; |
848 | } | |
849 | ||
f67539c2 | 850 | int32_t |
11fdf7f2 TL |
851 | rte_service_lcore_attr_reset_all(uint32_t lcore) |
852 | { | |
853 | struct core_state *cs; | |
854 | ||
855 | if (lcore >= RTE_MAX_LCORE) | |
856 | return -EINVAL; | |
857 | ||
858 | cs = &lcore_states[lcore]; | |
859 | if (!cs->is_service_core) | |
860 | return -ENOTSUP; | |
861 | ||
862 | cs->loops = 0; | |
863 | ||
864 | return 0; | |
865 | } | |
866 | ||
867 | static void | |
868 | service_dump_calls_per_lcore(FILE *f, uint32_t lcore, uint32_t reset) | |
869 | { | |
870 | uint32_t i; | |
871 | struct core_state *cs = &lcore_states[lcore]; | |
872 | ||
873 | fprintf(f, "%02d\t", lcore); | |
874 | for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) { | |
875 | if (!service_valid(i)) | |
876 | continue; | |
877 | fprintf(f, "%"PRIu64"\t", cs->calls_per_service[i]); | |
878 | if (reset) | |
879 | cs->calls_per_service[i] = 0; | |
880 | } | |
881 | fprintf(f, "\n"); | |
882 | } | |
883 | ||
884 | int32_t | |
885 | rte_service_dump(FILE *f, uint32_t id) | |
886 | { | |
887 | uint32_t i; | |
888 | int print_one = (id != UINT32_MAX); | |
889 | ||
11fdf7f2 TL |
890 | /* print only the specified service */ |
891 | if (print_one) { | |
892 | struct rte_service_spec_impl *s; | |
893 | SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL); | |
894 | fprintf(f, "Service %s Summary\n", s->spec.name); | |
895 | uint32_t reset = 0; | |
f67539c2 | 896 | service_dump_one(f, s, reset); |
11fdf7f2 TL |
897 | return 0; |
898 | } | |
899 | ||
900 | /* print all services, as UINT32_MAX was passed as id */ | |
901 | fprintf(f, "Services Summary\n"); | |
902 | for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) { | |
903 | if (!service_valid(i)) | |
904 | continue; | |
905 | uint32_t reset = 0; | |
f67539c2 | 906 | service_dump_one(f, &rte_services[i], reset); |
11fdf7f2 TL |
907 | } |
908 | ||
909 | fprintf(f, "Service Cores Summary\n"); | |
910 | for (i = 0; i < RTE_MAX_LCORE; i++) { | |
911 | if (lcore_config[i].core_role != ROLE_SERVICE) | |
912 | continue; | |
913 | ||
914 | uint32_t reset = 0; | |
915 | service_dump_calls_per_lcore(f, i, reset); | |
916 | } | |
917 | ||
918 | return 0; | |
919 | } |