]>
Commit | Line | Data |
---|---|---|
9f95a23c | 1 | /* SPDX-License-Identifier: BSD-3-Clause |
11fdf7f2 | 2 | * |
9f95a23c | 3 | * Copyright 2016,2018 NXP |
11fdf7f2 | 4 | * |
11fdf7f2 TL |
5 | */ |
6 | ||
7 | #include <string.h> | |
8 | #include <dirent.h> | |
9f95a23c | 9 | #include <stdbool.h> |
11fdf7f2 TL |
10 | |
11 | #include <rte_log.h> | |
12 | #include <rte_bus.h> | |
13 | #include <rte_eal_memconfig.h> | |
14 | #include <rte_malloc.h> | |
15 | #include <rte_devargs.h> | |
16 | #include <rte_memcpy.h> | |
9f95a23c TL |
17 | #include <rte_ethdev_driver.h> |
18 | ||
19 | #include <rte_fslmc.h> | |
20 | #include <fslmc_vfio.h> | |
21 | #include "fslmc_logs.h" | |
11fdf7f2 | 22 | |
9f95a23c | 23 | #include <dpaax_iova_table.h> |
11fdf7f2 | 24 | |
9f95a23c TL |
25 | int dpaa2_logtype_bus; |
26 | ||
27 | #define VFIO_IOMMU_GROUP_PATH "/sys/kernel/iommu_groups" | |
28 | #define FSLMC_BUS_NAME fslmc | |
11fdf7f2 TL |
29 | |
30 | struct rte_fslmc_bus rte_fslmc_bus; | |
9f95a23c TL |
31 | uint8_t dpaa2_virt_mode; |
32 | ||
33 | uint32_t | |
34 | rte_fslmc_get_device_count(enum rte_dpaa2_dev_type device_type) | |
35 | { | |
36 | if (device_type > DPAA2_DEVTYPE_MAX) | |
37 | return 0; | |
38 | return rte_fslmc_bus.device_count[device_type]; | |
39 | } | |
40 | ||
41 | RTE_DEFINE_PER_LCORE(struct dpaa2_portal_dqrr, dpaa2_held_bufs); | |
42 | ||
43 | static void | |
44 | cleanup_fslmc_device_list(void) | |
45 | { | |
46 | struct rte_dpaa2_device *dev; | |
47 | struct rte_dpaa2_device *t_dev; | |
48 | ||
49 | TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, t_dev) { | |
50 | TAILQ_REMOVE(&rte_fslmc_bus.device_list, dev, next); | |
51 | free(dev); | |
52 | dev = NULL; | |
53 | } | |
54 | } | |
55 | ||
56 | static int | |
57 | compare_dpaa2_devname(struct rte_dpaa2_device *dev1, | |
58 | struct rte_dpaa2_device *dev2) | |
59 | { | |
60 | int comp; | |
61 | ||
62 | if (dev1->dev_type > dev2->dev_type) { | |
63 | comp = 1; | |
64 | } else if (dev1->dev_type < dev2->dev_type) { | |
65 | comp = -1; | |
66 | } else { | |
67 | /* Check the ID as types match */ | |
68 | if (dev1->object_id > dev2->object_id) | |
69 | comp = 1; | |
70 | else if (dev1->object_id < dev2->object_id) | |
71 | comp = -1; | |
72 | else | |
73 | comp = 0; /* Duplicate device name */ | |
74 | } | |
75 | ||
76 | return comp; | |
77 | } | |
78 | ||
79 | static void | |
80 | insert_in_device_list(struct rte_dpaa2_device *newdev) | |
81 | { | |
82 | int comp, inserted = 0; | |
83 | struct rte_dpaa2_device *dev = NULL; | |
84 | struct rte_dpaa2_device *tdev = NULL; | |
85 | ||
86 | TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, tdev) { | |
87 | comp = compare_dpaa2_devname(newdev, dev); | |
88 | if (comp < 0) { | |
89 | TAILQ_INSERT_BEFORE(dev, newdev, next); | |
90 | inserted = 1; | |
91 | break; | |
92 | } | |
93 | } | |
94 | ||
95 | if (!inserted) | |
96 | TAILQ_INSERT_TAIL(&rte_fslmc_bus.device_list, newdev, next); | |
97 | } | |
98 | ||
99 | static struct rte_devargs * | |
100 | fslmc_devargs_lookup(struct rte_dpaa2_device *dev) | |
101 | { | |
102 | struct rte_devargs *devargs; | |
103 | char dev_name[32]; | |
104 | ||
105 | RTE_EAL_DEVARGS_FOREACH("fslmc", devargs) { | |
106 | devargs->bus->parse(devargs->name, &dev_name); | |
107 | if (strcmp(dev_name, dev->device.name) == 0) { | |
108 | DPAA2_BUS_INFO("**Devargs matched %s", dev_name); | |
109 | return devargs; | |
110 | } | |
111 | } | |
112 | return NULL; | |
113 | } | |
114 | ||
115 | static void | |
116 | dump_device_list(void) | |
117 | { | |
118 | struct rte_dpaa2_device *dev; | |
119 | uint32_t global_log_level; | |
120 | int local_log_level; | |
121 | ||
122 | /* Only if the log level has been set to Debugging, print list */ | |
123 | global_log_level = rte_log_get_global_level(); | |
124 | local_log_level = rte_log_get_level(dpaa2_logtype_bus); | |
125 | if (global_log_level == RTE_LOG_DEBUG || | |
126 | local_log_level == RTE_LOG_DEBUG) { | |
127 | DPAA2_BUS_LOG(DEBUG, "List of devices scanned on bus:"); | |
128 | TAILQ_FOREACH(dev, &rte_fslmc_bus.device_list, next) { | |
129 | DPAA2_BUS_LOG(DEBUG, "\t\t%s", dev->device.name); | |
130 | } | |
131 | } | |
132 | } | |
133 | ||
134 | static int | |
135 | scan_one_fslmc_device(char *dev_name) | |
136 | { | |
137 | char *dup_dev_name, *t_ptr; | |
138 | struct rte_dpaa2_device *dev; | |
139 | ||
140 | if (!dev_name) | |
141 | return -1; | |
142 | ||
143 | /* Ignore the Container name itself */ | |
144 | if (!strncmp("dprc", dev_name, 4)) | |
145 | return 0; | |
146 | ||
147 | /* Creating a temporary copy to perform cut-parse over string */ | |
148 | dup_dev_name = strdup(dev_name); | |
149 | if (!dup_dev_name) { | |
150 | DPAA2_BUS_ERR("Unable to allocate device name memory"); | |
151 | return -ENOMEM; | |
152 | } | |
153 | ||
154 | /* For all other devices, we allocate rte_dpaa2_device. | |
155 | * For those devices where there is no driver, probe would release | |
156 | * the memory associated with the rte_dpaa2_device after necessary | |
157 | * initialization. | |
158 | */ | |
159 | dev = calloc(1, sizeof(struct rte_dpaa2_device)); | |
160 | if (!dev) { | |
161 | DPAA2_BUS_ERR("Unable to allocate device object"); | |
162 | free(dup_dev_name); | |
163 | return -ENOMEM; | |
164 | } | |
165 | ||
166 | dev->device.bus = &rte_fslmc_bus.bus; | |
167 | ||
168 | /* Parse the device name and ID */ | |
169 | t_ptr = strtok(dup_dev_name, "."); | |
170 | if (!t_ptr) { | |
171 | DPAA2_BUS_ERR("Incorrect device name observed"); | |
172 | goto cleanup; | |
173 | } | |
174 | if (!strncmp("dpni", t_ptr, 4)) | |
175 | dev->dev_type = DPAA2_ETH; | |
176 | else if (!strncmp("dpseci", t_ptr, 6)) | |
177 | dev->dev_type = DPAA2_CRYPTO; | |
178 | else if (!strncmp("dpcon", t_ptr, 5)) | |
179 | dev->dev_type = DPAA2_CON; | |
180 | else if (!strncmp("dpbp", t_ptr, 4)) | |
181 | dev->dev_type = DPAA2_BPOOL; | |
182 | else if (!strncmp("dpio", t_ptr, 4)) | |
183 | dev->dev_type = DPAA2_IO; | |
184 | else if (!strncmp("dpci", t_ptr, 4)) | |
185 | dev->dev_type = DPAA2_CI; | |
186 | else if (!strncmp("dpmcp", t_ptr, 5)) | |
187 | dev->dev_type = DPAA2_MPORTAL; | |
188 | else if (!strncmp("dpdmai", t_ptr, 6)) | |
189 | dev->dev_type = DPAA2_QDMA; | |
190 | else if (!strncmp("dpdmux", t_ptr, 6)) | |
191 | dev->dev_type = DPAA2_MUX; | |
192 | else | |
193 | dev->dev_type = DPAA2_UNKNOWN; | |
194 | ||
195 | /* Update the device found into the device_count table */ | |
196 | rte_fslmc_bus.device_count[dev->dev_type]++; | |
197 | ||
198 | t_ptr = strtok(NULL, "."); | |
199 | if (!t_ptr) { | |
200 | DPAA2_BUS_ERR("Incorrect device string observed (null)"); | |
201 | goto cleanup; | |
202 | } | |
203 | ||
204 | sscanf(t_ptr, "%hu", &dev->object_id); | |
205 | dev->device.name = strdup(dev_name); | |
206 | if (!dev->device.name) { | |
207 | DPAA2_BUS_ERR("Unable to clone device name. Out of memory"); | |
208 | goto cleanup; | |
209 | } | |
210 | dev->device.devargs = fslmc_devargs_lookup(dev); | |
211 | ||
212 | /* Add device in the fslmc device list */ | |
213 | insert_in_device_list(dev); | |
214 | ||
215 | /* Don't need the duplicated device filesystem entry anymore */ | |
216 | if (dup_dev_name) | |
217 | free(dup_dev_name); | |
218 | ||
219 | return 0; | |
220 | cleanup: | |
221 | if (dup_dev_name) | |
222 | free(dup_dev_name); | |
223 | if (dev) | |
224 | free(dev); | |
225 | return -1; | |
226 | } | |
227 | ||
228 | static int | |
229 | rte_fslmc_parse(const char *name, void *addr) | |
230 | { | |
231 | uint16_t dev_id; | |
232 | char *t_ptr; | |
233 | ||
234 | /* 'name' is expected to contain name of device, for example, dpio.1, | |
235 | * dpni.2, etc. | |
236 | */ | |
237 | if (strncmp("dpni", name, 4) && | |
238 | strncmp("dpseci", name, 6) && | |
239 | strncmp("dpcon", name, 5) && | |
240 | strncmp("dpbp", name, 4) && | |
241 | strncmp("dpio", name, 4) && | |
242 | strncmp("dpci", name, 4) && | |
243 | strncmp("dpmcp", name, 5) && | |
244 | strncmp("dpdmai", name, 6) && | |
245 | strncmp("dpdmux", name, 6)) { | |
246 | DPAA2_BUS_DEBUG("Unknown or unsupported device (%s)", name); | |
247 | goto err_out; | |
248 | } | |
249 | ||
250 | t_ptr = strchr(name, '.'); | |
251 | if (!t_ptr || sscanf(t_ptr + 1, "%hu", &dev_id) != 1) { | |
252 | DPAA2_BUS_ERR("Missing device id in device name (%s)", name); | |
253 | goto err_out; | |
254 | } | |
255 | ||
256 | if (addr) | |
257 | strcpy(addr, name); | |
258 | ||
259 | return 0; | |
260 | err_out: | |
261 | return -EINVAL; | |
262 | } | |
11fdf7f2 TL |
263 | |
264 | static int | |
265 | rte_fslmc_scan(void) | |
266 | { | |
267 | int ret; | |
9f95a23c TL |
268 | int device_count = 0; |
269 | char fslmc_dirpath[PATH_MAX]; | |
270 | DIR *dir; | |
271 | struct dirent *entry; | |
272 | static int process_once; | |
273 | int groupid; | |
11fdf7f2 | 274 | |
9f95a23c TL |
275 | if (process_once) { |
276 | DPAA2_BUS_DEBUG("Fslmc bus already scanned. Not rescanning"); | |
277 | return 0; | |
11fdf7f2 | 278 | } |
9f95a23c | 279 | process_once = 1; |
11fdf7f2 | 280 | |
9f95a23c TL |
281 | ret = fslmc_get_container_group(&groupid); |
282 | if (ret != 0) | |
283 | goto scan_fail; | |
284 | ||
285 | /* Scan devices on the group */ | |
286 | snprintf(fslmc_dirpath, sizeof(fslmc_dirpath), "%s/%d/devices", | |
287 | VFIO_IOMMU_GROUP_PATH, groupid); | |
288 | dir = opendir(fslmc_dirpath); | |
289 | if (!dir) { | |
290 | DPAA2_BUS_ERR("Unable to open VFIO group directory"); | |
291 | goto scan_fail; | |
292 | } | |
293 | ||
294 | while ((entry = readdir(dir)) != NULL) { | |
295 | if (entry->d_name[0] == '.' || entry->d_type != DT_LNK) | |
296 | continue; | |
297 | ||
298 | ret = scan_one_fslmc_device(entry->d_name); | |
299 | if (ret != 0) { | |
300 | /* Error in parsing directory - exit gracefully */ | |
301 | goto scan_fail_cleanup; | |
302 | } | |
303 | device_count += 1; | |
11fdf7f2 TL |
304 | } |
305 | ||
9f95a23c TL |
306 | closedir(dir); |
307 | ||
308 | DPAA2_BUS_INFO("FSLMC Bus scan completed"); | |
309 | /* If debugging is enabled, device list is dumped to log output */ | |
310 | dump_device_list(); | |
311 | ||
312 | return 0; | |
313 | ||
314 | scan_fail_cleanup: | |
315 | closedir(dir); | |
316 | ||
317 | /* Remove all devices in the list */ | |
318 | cleanup_fslmc_device_list(); | |
319 | scan_fail: | |
320 | DPAA2_BUS_DEBUG("FSLMC Bus Not Available. Skipping"); | |
321 | /* Irrespective of failure, scan only return success */ | |
11fdf7f2 TL |
322 | return 0; |
323 | } | |
324 | ||
325 | static int | |
326 | rte_fslmc_match(struct rte_dpaa2_driver *dpaa2_drv, | |
327 | struct rte_dpaa2_device *dpaa2_dev) | |
328 | { | |
329 | if (dpaa2_drv->drv_type == dpaa2_dev->dev_type) | |
330 | return 0; | |
331 | ||
332 | return 1; | |
333 | } | |
334 | ||
335 | static int | |
336 | rte_fslmc_probe(void) | |
337 | { | |
338 | int ret = 0; | |
9f95a23c TL |
339 | int probe_all; |
340 | ||
11fdf7f2 TL |
341 | struct rte_dpaa2_device *dev; |
342 | struct rte_dpaa2_driver *drv; | |
343 | ||
9f95a23c TL |
344 | if (TAILQ_EMPTY(&rte_fslmc_bus.device_list)) |
345 | return 0; | |
346 | ||
347 | ret = fslmc_vfio_setup_group(); | |
348 | if (ret) { | |
349 | DPAA2_BUS_ERR("Unable to setup VFIO %d", ret); | |
350 | return 0; | |
351 | } | |
352 | ||
353 | /* Map existing segments as well as, in case of hotpluggable memory, | |
354 | * install callback handler. | |
355 | */ | |
356 | ret = rte_fslmc_vfio_dmamap(); | |
357 | if (ret) { | |
358 | DPAA2_BUS_ERR("Unable to DMA map existing VAs: (%d)", ret); | |
359 | /* Not continuing ahead */ | |
360 | DPAA2_BUS_ERR("FSLMC VFIO Mapping failed"); | |
361 | return 0; | |
362 | } | |
363 | ||
364 | ret = fslmc_vfio_process_group(); | |
365 | if (ret) { | |
366 | DPAA2_BUS_ERR("Unable to setup devices %d", ret); | |
367 | return 0; | |
368 | } | |
369 | ||
370 | probe_all = rte_fslmc_bus.bus.conf.scan_mode != RTE_BUS_SCAN_WHITELIST; | |
371 | ||
372 | /* In case of PA, the FD addresses returned by qbman APIs are physical | |
373 | * addresses, which need conversion into equivalent VA address for | |
374 | * rte_mbuf. For that, a table (a serial array, in memory) is used to | |
375 | * increase translation efficiency. | |
376 | * This has to be done before probe as some device initialization | |
377 | * (during) probe allocate memory (dpaa2_sec) which needs to be pinned | |
378 | * to this table. | |
379 | * | |
380 | * Error is ignored as relevant logs are handled within dpaax and | |
381 | * handling for unavailable dpaax table too is transparent to caller. | |
382 | */ | |
383 | dpaax_iova_table_populate(); | |
384 | ||
11fdf7f2 TL |
385 | TAILQ_FOREACH(dev, &rte_fslmc_bus.device_list, next) { |
386 | TAILQ_FOREACH(drv, &rte_fslmc_bus.driver_list, next) { | |
387 | ret = rte_fslmc_match(drv, dev); | |
388 | if (ret) | |
389 | continue; | |
390 | ||
391 | if (!drv->probe) | |
392 | continue; | |
393 | ||
9f95a23c TL |
394 | if (rte_dev_is_probed(&dev->device)) |
395 | continue; | |
396 | ||
397 | if (dev->device.devargs && | |
398 | dev->device.devargs->policy == RTE_DEV_BLACKLISTED) { | |
399 | DPAA2_BUS_LOG(DEBUG, "%s Blacklisted, skipping", | |
400 | dev->device.name); | |
401 | continue; | |
402 | } | |
403 | ||
404 | if (probe_all || | |
405 | (dev->device.devargs && | |
406 | dev->device.devargs->policy == | |
407 | RTE_DEV_WHITELISTED)) { | |
408 | ret = drv->probe(drv, dev); | |
409 | if (ret) { | |
410 | DPAA2_BUS_ERR("Unable to probe"); | |
411 | } else { | |
412 | dev->driver = drv; | |
413 | dev->device.driver = &drv->driver; | |
414 | } | |
415 | } | |
11fdf7f2 TL |
416 | break; |
417 | } | |
418 | } | |
9f95a23c TL |
419 | |
420 | if (rte_eal_iova_mode() == RTE_IOVA_VA) | |
421 | dpaa2_virt_mode = 1; | |
422 | ||
423 | return 0; | |
424 | } | |
425 | ||
426 | static struct rte_device * | |
427 | rte_fslmc_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, | |
428 | const void *data) | |
429 | { | |
430 | const struct rte_dpaa2_device *dstart; | |
431 | struct rte_dpaa2_device *dev; | |
432 | ||
433 | if (start != NULL) { | |
434 | dstart = RTE_DEV_TO_FSLMC_CONST(start); | |
435 | dev = TAILQ_NEXT(dstart, next); | |
436 | } else { | |
437 | dev = TAILQ_FIRST(&rte_fslmc_bus.device_list); | |
438 | } | |
439 | while (dev != NULL) { | |
440 | if (cmp(&dev->device, data) == 0) | |
441 | return &dev->device; | |
442 | dev = TAILQ_NEXT(dev, next); | |
443 | } | |
444 | ||
445 | return NULL; | |
11fdf7f2 TL |
446 | } |
447 | ||
448 | /*register a fslmc bus based dpaa2 driver */ | |
449 | void | |
450 | rte_fslmc_driver_register(struct rte_dpaa2_driver *driver) | |
451 | { | |
452 | RTE_VERIFY(driver); | |
453 | ||
454 | TAILQ_INSERT_TAIL(&rte_fslmc_bus.driver_list, driver, next); | |
455 | /* Update Bus references */ | |
456 | driver->fslmc_bus = &rte_fslmc_bus; | |
457 | } | |
458 | ||
459 | /*un-register a fslmc bus based dpaa2 driver */ | |
460 | void | |
461 | rte_fslmc_driver_unregister(struct rte_dpaa2_driver *driver) | |
462 | { | |
463 | struct rte_fslmc_bus *fslmc_bus; | |
464 | ||
465 | fslmc_bus = driver->fslmc_bus; | |
466 | ||
9f95a23c TL |
467 | /* Cleanup the PA->VA Translation table; From whereever this function |
468 | * is called from. | |
469 | */ | |
470 | dpaax_iova_table_depopulate(); | |
471 | ||
11fdf7f2 TL |
472 | TAILQ_REMOVE(&fslmc_bus->driver_list, driver, next); |
473 | /* Update Bus references */ | |
474 | driver->fslmc_bus = NULL; | |
475 | } | |
476 | ||
9f95a23c TL |
477 | /* |
478 | * All device has iova as va | |
479 | */ | |
480 | static inline int | |
481 | fslmc_all_device_support_iova(void) | |
482 | { | |
483 | int ret = 0; | |
484 | struct rte_dpaa2_device *dev; | |
485 | struct rte_dpaa2_driver *drv; | |
486 | ||
487 | TAILQ_FOREACH(dev, &rte_fslmc_bus.device_list, next) { | |
488 | TAILQ_FOREACH(drv, &rte_fslmc_bus.driver_list, next) { | |
489 | ret = rte_fslmc_match(drv, dev); | |
490 | if (ret) | |
491 | continue; | |
492 | /* if the driver is not supporting IOVA */ | |
493 | if (!(drv->drv_flags & RTE_DPAA2_DRV_IOVA_AS_VA)) | |
494 | return 0; | |
495 | } | |
496 | } | |
497 | return 1; | |
498 | } | |
499 | ||
500 | /* | |
501 | * Get iommu class of DPAA2 devices on the bus. | |
502 | */ | |
503 | static enum rte_iova_mode | |
504 | rte_dpaa2_get_iommu_class(void) | |
505 | { | |
506 | bool is_vfio_noiommu_enabled = 1; | |
507 | bool has_iova_va; | |
508 | ||
509 | if (TAILQ_EMPTY(&rte_fslmc_bus.device_list)) | |
510 | return RTE_IOVA_DC; | |
511 | ||
512 | #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA | |
513 | return RTE_IOVA_PA; | |
514 | #endif | |
515 | ||
516 | /* check if all devices on the bus support Virtual addressing or not */ | |
517 | has_iova_va = fslmc_all_device_support_iova(); | |
518 | ||
519 | #ifdef VFIO_PRESENT | |
520 | is_vfio_noiommu_enabled = rte_vfio_noiommu_is_enabled() == true ? | |
521 | true : false; | |
522 | #endif | |
523 | ||
524 | if (has_iova_va && !is_vfio_noiommu_enabled) | |
525 | return RTE_IOVA_VA; | |
526 | ||
527 | return RTE_IOVA_PA; | |
528 | } | |
529 | ||
11fdf7f2 TL |
530 | struct rte_fslmc_bus rte_fslmc_bus = { |
531 | .bus = { | |
532 | .scan = rte_fslmc_scan, | |
533 | .probe = rte_fslmc_probe, | |
9f95a23c TL |
534 | .parse = rte_fslmc_parse, |
535 | .find_device = rte_fslmc_find_device, | |
536 | .get_iommu_class = rte_dpaa2_get_iommu_class, | |
11fdf7f2 TL |
537 | }, |
538 | .device_list = TAILQ_HEAD_INITIALIZER(rte_fslmc_bus.device_list), | |
539 | .driver_list = TAILQ_HEAD_INITIALIZER(rte_fslmc_bus.driver_list), | |
9f95a23c | 540 | .device_count = {0}, |
11fdf7f2 TL |
541 | }; |
542 | ||
543 | RTE_REGISTER_BUS(FSLMC_BUS_NAME, rte_fslmc_bus.bus); | |
9f95a23c TL |
544 | |
545 | RTE_INIT(fslmc_init_log) | |
546 | { | |
547 | /* Bus level logs */ | |
548 | dpaa2_logtype_bus = rte_log_register("bus.fslmc"); | |
549 | if (dpaa2_logtype_bus >= 0) | |
550 | rte_log_set_level(dpaa2_logtype_bus, RTE_LOG_NOTICE); | |
551 | } |