]>
Commit | Line | Data |
---|---|---|
52916982 LG |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * PCI Peer 2 Peer DMA support. | |
4 | * | |
5 | * Copyright (c) 2016-2018, Logan Gunthorpe | |
6 | * Copyright (c) 2016-2017, Microsemi Corporation | |
7 | * Copyright (c) 2017, Christoph Hellwig | |
8 | * Copyright (c) 2018, Eideticom Inc. | |
9 | */ | |
10 | ||
2d7bc010 LG |
11 | #define pr_fmt(fmt) "pci-p2pdma: " fmt |
12 | #include <linux/ctype.h> | |
52916982 LG |
13 | #include <linux/pci-p2pdma.h> |
14 | #include <linux/module.h> | |
15 | #include <linux/slab.h> | |
16 | #include <linux/genalloc.h> | |
17 | #include <linux/memremap.h> | |
18 | #include <linux/percpu-refcount.h> | |
19 | #include <linux/random.h> | |
20 | #include <linux/seq_buf.h> | |
110203be | 21 | #include <linux/xarray.h> |
52916982 | 22 | |
72583084 LG |
23 | enum pci_p2pdma_map_type { |
24 | PCI_P2PDMA_MAP_UNKNOWN = 0, | |
25 | PCI_P2PDMA_MAP_NOT_SUPPORTED, | |
26 | PCI_P2PDMA_MAP_BUS_ADDR, | |
27 | PCI_P2PDMA_MAP_THRU_HOST_BRIDGE, | |
28 | }; | |
29 | ||
52916982 | 30 | struct pci_p2pdma { |
52916982 LG |
31 | struct gen_pool *pool; |
32 | bool p2pmem_published; | |
110203be | 33 | struct xarray map_types; |
52916982 LG |
34 | }; |
35 | ||
a6e6fe65 LG |
36 | struct pci_p2pdma_pagemap { |
37 | struct dev_pagemap pgmap; | |
0afea381 | 38 | struct pci_dev *provider; |
a6e6fe65 LG |
39 | u64 bus_offset; |
40 | }; | |
41 | ||
42 | static struct pci_p2pdma_pagemap *to_p2p_pgmap(struct dev_pagemap *pgmap) | |
43 | { | |
44 | return container_of(pgmap, struct pci_p2pdma_pagemap, pgmap); | |
45 | } | |
46 | ||
cbb8ca69 LG |
47 | static ssize_t size_show(struct device *dev, struct device_attribute *attr, |
48 | char *buf) | |
49 | { | |
50 | struct pci_dev *pdev = to_pci_dev(dev); | |
51 | size_t size = 0; | |
52 | ||
53 | if (pdev->p2pdma->pool) | |
54 | size = gen_pool_size(pdev->p2pdma->pool); | |
55 | ||
56 | return snprintf(buf, PAGE_SIZE, "%zd\n", size); | |
57 | } | |
58 | static DEVICE_ATTR_RO(size); | |
59 | ||
60 | static ssize_t available_show(struct device *dev, struct device_attribute *attr, | |
61 | char *buf) | |
62 | { | |
63 | struct pci_dev *pdev = to_pci_dev(dev); | |
64 | size_t avail = 0; | |
65 | ||
66 | if (pdev->p2pdma->pool) | |
67 | avail = gen_pool_avail(pdev->p2pdma->pool); | |
68 | ||
69 | return snprintf(buf, PAGE_SIZE, "%zd\n", avail); | |
70 | } | |
71 | static DEVICE_ATTR_RO(available); | |
72 | ||
73 | static ssize_t published_show(struct device *dev, struct device_attribute *attr, | |
74 | char *buf) | |
75 | { | |
76 | struct pci_dev *pdev = to_pci_dev(dev); | |
77 | ||
78 | return snprintf(buf, PAGE_SIZE, "%d\n", | |
79 | pdev->p2pdma->p2pmem_published); | |
80 | } | |
81 | static DEVICE_ATTR_RO(published); | |
82 | ||
83 | static struct attribute *p2pmem_attrs[] = { | |
84 | &dev_attr_size.attr, | |
85 | &dev_attr_available.attr, | |
86 | &dev_attr_published.attr, | |
87 | NULL, | |
88 | }; | |
89 | ||
90 | static const struct attribute_group p2pmem_group = { | |
91 | .attrs = p2pmem_attrs, | |
92 | .name = "p2pmem", | |
93 | }; | |
94 | ||
52916982 LG |
95 | static void pci_p2pdma_release(void *data) |
96 | { | |
97 | struct pci_dev *pdev = data; | |
1570175a | 98 | struct pci_p2pdma *p2pdma = pdev->p2pdma; |
52916982 | 99 | |
1570175a | 100 | if (!p2pdma) |
52916982 LG |
101 | return; |
102 | ||
1570175a DW |
103 | /* Flush and disable pci_alloc_p2p_mem() */ |
104 | pdev->p2pdma = NULL; | |
105 | synchronize_rcu(); | |
52916982 | 106 | |
1570175a | 107 | gen_pool_destroy(p2pdma->pool); |
cbb8ca69 | 108 | sysfs_remove_group(&pdev->dev.kobj, &p2pmem_group); |
110203be | 109 | xa_destroy(&p2pdma->map_types); |
52916982 LG |
110 | } |
111 | ||
112 | static int pci_p2pdma_setup(struct pci_dev *pdev) | |
113 | { | |
114 | int error = -ENOMEM; | |
115 | struct pci_p2pdma *p2p; | |
116 | ||
117 | p2p = devm_kzalloc(&pdev->dev, sizeof(*p2p), GFP_KERNEL); | |
118 | if (!p2p) | |
119 | return -ENOMEM; | |
120 | ||
110203be LG |
121 | xa_init(&p2p->map_types); |
122 | ||
52916982 LG |
123 | p2p->pool = gen_pool_create(PAGE_SHIFT, dev_to_node(&pdev->dev)); |
124 | if (!p2p->pool) | |
125 | goto out; | |
126 | ||
52916982 LG |
127 | error = devm_add_action_or_reset(&pdev->dev, pci_p2pdma_release, pdev); |
128 | if (error) | |
129 | goto out_pool_destroy; | |
130 | ||
131 | pdev->p2pdma = p2p; | |
132 | ||
cbb8ca69 LG |
133 | error = sysfs_create_group(&pdev->dev.kobj, &p2pmem_group); |
134 | if (error) | |
135 | goto out_pool_destroy; | |
136 | ||
52916982 LG |
137 | return 0; |
138 | ||
139 | out_pool_destroy: | |
cbb8ca69 | 140 | pdev->p2pdma = NULL; |
52916982 LG |
141 | gen_pool_destroy(p2p->pool); |
142 | out: | |
143 | devm_kfree(&pdev->dev, p2p); | |
144 | return error; | |
145 | } | |
146 | ||
147 | /** | |
148 | * pci_p2pdma_add_resource - add memory for use as p2p memory | |
149 | * @pdev: the device to add the memory to | |
150 | * @bar: PCI BAR to add | |
151 | * @size: size of the memory to add, may be zero to use the whole BAR | |
152 | * @offset: offset into the PCI BAR | |
153 | * | |
154 | * The memory will be given ZONE_DEVICE struct pages so that it may | |
155 | * be used with any DMA request. | |
156 | */ | |
157 | int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size, | |
158 | u64 offset) | |
159 | { | |
a6e6fe65 | 160 | struct pci_p2pdma_pagemap *p2p_pgmap; |
52916982 LG |
161 | struct dev_pagemap *pgmap; |
162 | void *addr; | |
163 | int error; | |
164 | ||
165 | if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM)) | |
166 | return -EINVAL; | |
167 | ||
168 | if (offset >= pci_resource_len(pdev, bar)) | |
169 | return -EINVAL; | |
170 | ||
171 | if (!size) | |
172 | size = pci_resource_len(pdev, bar) - offset; | |
173 | ||
174 | if (size + offset > pci_resource_len(pdev, bar)) | |
175 | return -EINVAL; | |
176 | ||
177 | if (!pdev->p2pdma) { | |
178 | error = pci_p2pdma_setup(pdev); | |
179 | if (error) | |
180 | return error; | |
181 | } | |
182 | ||
a6e6fe65 LG |
183 | p2p_pgmap = devm_kzalloc(&pdev->dev, sizeof(*p2p_pgmap), GFP_KERNEL); |
184 | if (!p2p_pgmap) | |
52916982 | 185 | return -ENOMEM; |
a6e6fe65 LG |
186 | |
187 | pgmap = &p2p_pgmap->pgmap; | |
52916982 LG |
188 | pgmap->res.start = pci_resource_start(pdev, bar) + offset; |
189 | pgmap->res.end = pgmap->res.start + size - 1; | |
190 | pgmap->res.flags = pci_resource_flags(pdev, bar); | |
52916982 | 191 | pgmap->type = MEMORY_DEVICE_PCI_P2PDMA; |
a6e6fe65 | 192 | |
0afea381 | 193 | p2p_pgmap->provider = pdev; |
a6e6fe65 | 194 | p2p_pgmap->bus_offset = pci_bus_address(pdev, bar) - |
977196b8 | 195 | pci_resource_start(pdev, bar); |
52916982 LG |
196 | |
197 | addr = devm_memremap_pages(&pdev->dev, pgmap); | |
198 | if (IS_ERR(addr)) { | |
199 | error = PTR_ERR(addr); | |
50f44ee7 | 200 | goto pgmap_free; |
52916982 LG |
201 | } |
202 | ||
1570175a | 203 | error = gen_pool_add_owner(pdev->p2pdma->pool, (unsigned long)addr, |
52916982 | 204 | pci_bus_address(pdev, bar) + offset, |
1570175a | 205 | resource_size(&pgmap->res), dev_to_node(&pdev->dev), |
d0b3517d | 206 | pgmap->ref); |
52916982 | 207 | if (error) |
e615a191 | 208 | goto pages_free; |
52916982 | 209 | |
52916982 LG |
210 | pci_info(pdev, "added peer-to-peer DMA memory %pR\n", |
211 | &pgmap->res); | |
212 | ||
213 | return 0; | |
214 | ||
e615a191 DW |
215 | pages_free: |
216 | devm_memunmap_pages(&pdev->dev, pgmap); | |
52916982 | 217 | pgmap_free: |
d0b3517d | 218 | devm_kfree(&pdev->dev, pgmap); |
52916982 LG |
219 | return error; |
220 | } | |
221 | EXPORT_SYMBOL_GPL(pci_p2pdma_add_resource); | |
222 | ||
223 | /* | |
224 | * Note this function returns the parent PCI device with a | |
f6b6aefe | 225 | * reference taken. It is the caller's responsibility to drop |
52916982 LG |
226 | * the reference. |
227 | */ | |
228 | static struct pci_dev *find_parent_pci_dev(struct device *dev) | |
229 | { | |
230 | struct device *parent; | |
231 | ||
232 | dev = get_device(dev); | |
233 | ||
234 | while (dev) { | |
235 | if (dev_is_pci(dev)) | |
236 | return to_pci_dev(dev); | |
237 | ||
238 | parent = get_device(dev->parent); | |
239 | put_device(dev); | |
240 | dev = parent; | |
241 | } | |
242 | ||
243 | return NULL; | |
244 | } | |
245 | ||
246 | /* | |
247 | * Check if a PCI bridge has its ACS redirection bits set to redirect P2P | |
248 | * TLPs upstream via ACS. Returns 1 if the packets will be redirected | |
249 | * upstream, 0 otherwise. | |
250 | */ | |
251 | static int pci_bridge_has_acs_redir(struct pci_dev *pdev) | |
252 | { | |
253 | int pos; | |
254 | u16 ctrl; | |
255 | ||
256 | pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ACS); | |
257 | if (!pos) | |
258 | return 0; | |
259 | ||
260 | pci_read_config_word(pdev, pos + PCI_ACS_CTRL, &ctrl); | |
261 | ||
262 | if (ctrl & (PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_EC)) | |
263 | return 1; | |
264 | ||
265 | return 0; | |
266 | } | |
267 | ||
268 | static void seq_buf_print_bus_devfn(struct seq_buf *buf, struct pci_dev *pdev) | |
269 | { | |
270 | if (!buf) | |
271 | return; | |
272 | ||
273 | seq_buf_printf(buf, "%s;", pci_name(pdev)); | |
274 | } | |
275 | ||
494d63b0 LG |
276 | static const struct pci_p2pdma_whitelist_entry { |
277 | unsigned short vendor; | |
278 | unsigned short device; | |
279 | enum { | |
280 | REQ_SAME_HOST_BRIDGE = 1 << 0, | |
281 | } flags; | |
282 | } pci_p2pdma_whitelist[] = { | |
283 | /* AMD ZEN */ | |
284 | {PCI_VENDOR_ID_AMD, 0x1450, 0}, | |
285 | ||
286 | /* Intel Xeon E5/Core i7 */ | |
287 | {PCI_VENDOR_ID_INTEL, 0x3c00, REQ_SAME_HOST_BRIDGE}, | |
288 | {PCI_VENDOR_ID_INTEL, 0x3c01, REQ_SAME_HOST_BRIDGE}, | |
289 | /* Intel Xeon E7 v3/Xeon E5 v3/Core i7 */ | |
290 | {PCI_VENDOR_ID_INTEL, 0x2f00, REQ_SAME_HOST_BRIDGE}, | |
291 | {PCI_VENDOR_ID_INTEL, 0x2f01, REQ_SAME_HOST_BRIDGE}, | |
bc123a51 AB |
292 | /* Intel SkyLake-E */ |
293 | {PCI_VENDOR_ID_INTEL, 0x2030, 0}, | |
294 | {PCI_VENDOR_ID_INTEL, 0x2020, 0}, | |
494d63b0 LG |
295 | {} |
296 | }; | |
297 | ||
298 | static bool __host_bridge_whitelist(struct pci_host_bridge *host, | |
299 | bool same_host_bridge) | |
0f97da83 | 300 | { |
0f97da83 | 301 | struct pci_dev *root = pci_get_slot(host->bus, PCI_DEVFN(0, 0)); |
494d63b0 | 302 | const struct pci_p2pdma_whitelist_entry *entry; |
0f97da83 CK |
303 | unsigned short vendor, device; |
304 | ||
305 | if (!root) | |
306 | return false; | |
307 | ||
308 | vendor = root->vendor; | |
309 | device = root->device; | |
310 | pci_dev_put(root); | |
311 | ||
494d63b0 LG |
312 | for (entry = pci_p2pdma_whitelist; entry->vendor; entry++) { |
313 | if (vendor != entry->vendor || device != entry->device) | |
314 | continue; | |
315 | if (entry->flags & REQ_SAME_HOST_BRIDGE && !same_host_bridge) | |
316 | return false; | |
317 | ||
0f97da83 | 318 | return true; |
494d63b0 | 319 | } |
0f97da83 CK |
320 | |
321 | return false; | |
322 | } | |
323 | ||
2c84d818 LG |
324 | /* |
325 | * If we can't find a common upstream bridge take a look at the root | |
326 | * complex and compare it to a whitelist of known good hardware. | |
327 | */ | |
328 | static bool host_bridge_whitelist(struct pci_dev *a, struct pci_dev *b) | |
329 | { | |
330 | struct pci_host_bridge *host_a = pci_find_host_bridge(a->bus); | |
331 | struct pci_host_bridge *host_b = pci_find_host_bridge(b->bus); | |
332 | ||
494d63b0 LG |
333 | if (host_a == host_b) |
334 | return __host_bridge_whitelist(host_a, true); | |
335 | ||
336 | if (__host_bridge_whitelist(host_a, false) && | |
337 | __host_bridge_whitelist(host_b, false)) | |
2c84d818 LG |
338 | return true; |
339 | ||
340 | return false; | |
341 | } | |
342 | ||
72583084 | 343 | static enum pci_p2pdma_map_type |
c6bfaeb5 | 344 | __upstream_bridge_distance(struct pci_dev *provider, struct pci_dev *client, |
72583084 | 345 | int *dist, bool *acs_redirects, struct seq_buf *acs_list) |
52916982 | 346 | { |
0f97da83 | 347 | struct pci_dev *a = provider, *b = client, *bb; |
52916982 LG |
348 | int dist_a = 0; |
349 | int dist_b = 0; | |
52916982 LG |
350 | int acs_cnt = 0; |
351 | ||
72583084 LG |
352 | if (acs_redirects) |
353 | *acs_redirects = false; | |
354 | ||
52916982 LG |
355 | /* |
356 | * Note, we don't need to take references to devices returned by | |
357 | * pci_upstream_bridge() seeing we hold a reference to a child | |
358 | * device which will already hold a reference to the upstream bridge. | |
359 | */ | |
360 | ||
361 | while (a) { | |
362 | dist_b = 0; | |
363 | ||
364 | if (pci_bridge_has_acs_redir(a)) { | |
365 | seq_buf_print_bus_devfn(acs_list, a); | |
366 | acs_cnt++; | |
367 | } | |
368 | ||
369 | bb = b; | |
370 | ||
371 | while (bb) { | |
372 | if (a == bb) | |
373 | goto check_b_path_acs; | |
374 | ||
375 | bb = pci_upstream_bridge(bb); | |
376 | dist_b++; | |
377 | } | |
378 | ||
379 | a = pci_upstream_bridge(a); | |
380 | dist_a++; | |
381 | } | |
382 | ||
72583084 LG |
383 | if (dist) |
384 | *dist = dist_a + dist_b; | |
385 | ||
e2cbfbf7 | 386 | return PCI_P2PDMA_MAP_THRU_HOST_BRIDGE; |
52916982 LG |
387 | |
388 | check_b_path_acs: | |
389 | bb = b; | |
390 | ||
391 | while (bb) { | |
392 | if (a == bb) | |
393 | break; | |
394 | ||
395 | if (pci_bridge_has_acs_redir(bb)) { | |
396 | seq_buf_print_bus_devfn(acs_list, bb); | |
397 | acs_cnt++; | |
398 | } | |
399 | ||
400 | bb = pci_upstream_bridge(bb); | |
401 | } | |
402 | ||
72583084 LG |
403 | if (dist) |
404 | *dist = dist_a + dist_b; | |
52916982 | 405 | |
72583084 LG |
406 | if (acs_cnt) { |
407 | if (acs_redirects) | |
408 | *acs_redirects = true; | |
409 | ||
e2cbfbf7 | 410 | return PCI_P2PDMA_MAP_THRU_HOST_BRIDGE; |
72583084 LG |
411 | } |
412 | ||
413 | return PCI_P2PDMA_MAP_BUS_ADDR; | |
52916982 LG |
414 | } |
415 | ||
110203be LG |
416 | static unsigned long map_types_idx(struct pci_dev *client) |
417 | { | |
418 | return (pci_domain_nr(client->bus) << 16) | | |
419 | (client->bus->number << 8) | client->devfn; | |
420 | } | |
421 | ||
c6bfaeb5 LG |
422 | /* |
423 | * Find the distance through the nearest common upstream bridge between | |
424 | * two PCI devices. | |
425 | * | |
426 | * If the two devices are the same device then 0 will be returned. | |
427 | * | |
428 | * If there are two virtual functions of the same device behind the same | |
429 | * bridge port then 2 will be returned (one step down to the PCIe switch, | |
430 | * then one step back to the same device). | |
431 | * | |
432 | * In the case where two devices are connected to the same PCIe switch, the | |
433 | * value 4 will be returned. This corresponds to the following PCI tree: | |
434 | * | |
435 | * -+ Root Port | |
436 | * \+ Switch Upstream Port | |
437 | * +-+ Switch Downstream Port | |
438 | * + \- Device A | |
439 | * \-+ Switch Downstream Port | |
440 | * \- Device B | |
441 | * | |
442 | * The distance is 4 because we traverse from Device A through the downstream | |
443 | * port of the switch, to the common upstream port, back up to the second | |
444 | * downstream port and then to Device B. | |
445 | * | |
446 | * Any two devices that cannot communicate using p2pdma will return | |
447 | * PCI_P2PDMA_MAP_NOT_SUPPORTED. | |
448 | * | |
449 | * Any two devices that have a data path that goes through the host bridge | |
450 | * will consult a whitelist. If the host bridges are on the whitelist, | |
451 | * this function will return PCI_P2PDMA_MAP_THRU_HOST_BRIDGE. | |
452 | * | |
453 | * If either bridge is not on the whitelist this function returns | |
454 | * PCI_P2PDMA_MAP_NOT_SUPPORTED. | |
455 | * | |
456 | * If a bridge which has any ACS redirection bits set is in the path, | |
457 | * acs_redirects will be set to true. In this case, a list of all infringing | |
458 | * bridge addresses will be populated in acs_list (assuming it's non-null) | |
459 | * for printk purposes. | |
460 | */ | |
461 | static enum pci_p2pdma_map_type | |
462 | upstream_bridge_distance(struct pci_dev *provider, struct pci_dev *client, | |
463 | int *dist, bool *acs_redirects, struct seq_buf *acs_list) | |
464 | { | |
e2cbfbf7 LG |
465 | enum pci_p2pdma_map_type map_type; |
466 | ||
467 | map_type = __upstream_bridge_distance(provider, client, dist, | |
468 | acs_redirects, acs_list); | |
469 | ||
470 | if (map_type == PCI_P2PDMA_MAP_THRU_HOST_BRIDGE) { | |
2c84d818 | 471 | if (!host_bridge_whitelist(provider, client)) |
110203be | 472 | map_type = PCI_P2PDMA_MAP_NOT_SUPPORTED; |
e2cbfbf7 LG |
473 | } |
474 | ||
110203be LG |
475 | if (provider->p2pdma) |
476 | xa_store(&provider->p2pdma->map_types, map_types_idx(client), | |
477 | xa_mk_value(map_type), GFP_KERNEL); | |
478 | ||
e2cbfbf7 | 479 | return map_type; |
c6bfaeb5 LG |
480 | } |
481 | ||
72583084 LG |
482 | static enum pci_p2pdma_map_type |
483 | upstream_bridge_distance_warn(struct pci_dev *provider, struct pci_dev *client, | |
484 | int *dist) | |
52916982 LG |
485 | { |
486 | struct seq_buf acs_list; | |
72583084 | 487 | bool acs_redirects; |
52916982 LG |
488 | int ret; |
489 | ||
490 | seq_buf_init(&acs_list, kmalloc(PAGE_SIZE, GFP_KERNEL), PAGE_SIZE); | |
491 | if (!acs_list.buffer) | |
492 | return -ENOMEM; | |
493 | ||
72583084 LG |
494 | ret = upstream_bridge_distance(provider, client, dist, &acs_redirects, |
495 | &acs_list); | |
496 | if (acs_redirects) { | |
497 | pci_warn(client, "ACS redirect is set between the client and provider (%s)\n", | |
52916982 LG |
498 | pci_name(provider)); |
499 | /* Drop final semicolon */ | |
500 | acs_list.buffer[acs_list.len-1] = 0; | |
501 | pci_warn(client, "to disable ACS redirect for this path, add the kernel parameter: pci=disable_acs_redir=%s\n", | |
502 | acs_list.buffer); | |
72583084 | 503 | } |
52916982 | 504 | |
72583084 LG |
505 | if (ret == PCI_P2PDMA_MAP_NOT_SUPPORTED) { |
506 | pci_warn(client, "cannot be used for peer-to-peer DMA as the client and provider (%s) do not share an upstream bridge or whitelisted host bridge\n", | |
52916982 LG |
507 | pci_name(provider)); |
508 | } | |
509 | ||
510 | kfree(acs_list.buffer); | |
511 | ||
512 | return ret; | |
513 | } | |
514 | ||
515 | /** | |
f6b6aefe | 516 | * pci_p2pdma_distance_many - Determine the cumulative distance between |
52916982 LG |
517 | * a p2pdma provider and the clients in use. |
518 | * @provider: p2pdma provider to check against the client list | |
519 | * @clients: array of devices to check (NULL-terminated) | |
520 | * @num_clients: number of clients in the array | |
521 | * @verbose: if true, print warnings for devices when we return -1 | |
522 | * | |
27235b15 LG |
523 | * Returns -1 if any of the clients are not compatible, otherwise returns a |
524 | * positive number where a lower number is the preferable choice. (If there's | |
525 | * one client that's the same as the provider it will return 0, which is best | |
526 | * choice). | |
52916982 | 527 | * |
27235b15 LG |
528 | * "compatible" means the provider and the clients are either all behind |
529 | * the same PCI root port or the host bridges connected to each of the devices | |
530 | * are listed in the 'pci_p2pdma_whitelist'. | |
52916982 LG |
531 | */ |
532 | int pci_p2pdma_distance_many(struct pci_dev *provider, struct device **clients, | |
533 | int num_clients, bool verbose) | |
534 | { | |
535 | bool not_supported = false; | |
536 | struct pci_dev *pci_client; | |
72583084 LG |
537 | int total_dist = 0; |
538 | int distance; | |
52916982 LG |
539 | int i, ret; |
540 | ||
541 | if (num_clients == 0) | |
542 | return -1; | |
543 | ||
544 | for (i = 0; i < num_clients; i++) { | |
9c002bb6 LG |
545 | if (IS_ENABLED(CONFIG_DMA_VIRT_OPS) && |
546 | clients[i]->dma_ops == &dma_virt_ops) { | |
547 | if (verbose) | |
548 | dev_warn(clients[i], | |
549 | "cannot be used for peer-to-peer DMA because the driver makes use of dma_virt_ops\n"); | |
550 | return -1; | |
551 | } | |
552 | ||
52916982 LG |
553 | pci_client = find_parent_pci_dev(clients[i]); |
554 | if (!pci_client) { | |
555 | if (verbose) | |
556 | dev_warn(clients[i], | |
557 | "cannot be used for peer-to-peer DMA as it is not a PCI device\n"); | |
558 | return -1; | |
559 | } | |
560 | ||
561 | if (verbose) | |
562 | ret = upstream_bridge_distance_warn(provider, | |
72583084 | 563 | pci_client, &distance); |
52916982 LG |
564 | else |
565 | ret = upstream_bridge_distance(provider, pci_client, | |
72583084 | 566 | &distance, NULL, NULL); |
52916982 LG |
567 | |
568 | pci_dev_put(pci_client); | |
569 | ||
72583084 | 570 | if (ret == PCI_P2PDMA_MAP_NOT_SUPPORTED) |
52916982 LG |
571 | not_supported = true; |
572 | ||
573 | if (not_supported && !verbose) | |
574 | break; | |
575 | ||
72583084 | 576 | total_dist += distance; |
52916982 LG |
577 | } |
578 | ||
579 | if (not_supported) | |
580 | return -1; | |
581 | ||
72583084 | 582 | return total_dist; |
52916982 LG |
583 | } |
584 | EXPORT_SYMBOL_GPL(pci_p2pdma_distance_many); | |
585 | ||
586 | /** | |
587 | * pci_has_p2pmem - check if a given PCI device has published any p2pmem | |
588 | * @pdev: PCI device to check | |
589 | */ | |
590 | bool pci_has_p2pmem(struct pci_dev *pdev) | |
591 | { | |
592 | return pdev->p2pdma && pdev->p2pdma->p2pmem_published; | |
593 | } | |
594 | EXPORT_SYMBOL_GPL(pci_has_p2pmem); | |
595 | ||
596 | /** | |
597 | * pci_p2pmem_find - find a peer-to-peer DMA memory device compatible with | |
598 | * the specified list of clients and shortest distance (as determined | |
599 | * by pci_p2pmem_dma()) | |
600 | * @clients: array of devices to check (NULL-terminated) | |
601 | * @num_clients: number of client devices in the list | |
602 | * | |
603 | * If multiple devices are behind the same switch, the one "closest" to the | |
fcf9ab35 | 604 | * client devices in use will be chosen first. (So if one of the providers is |
52916982 LG |
605 | * the same as one of the clients, that provider will be used ahead of any |
606 | * other providers that are unrelated). If multiple providers are an equal | |
607 | * distance away, one will be chosen at random. | |
608 | * | |
609 | * Returns a pointer to the PCI device with a reference taken (use pci_dev_put | |
610 | * to return the reference) or NULL if no compatible device is found. The | |
611 | * found provider will also be assigned to the client list. | |
612 | */ | |
613 | struct pci_dev *pci_p2pmem_find_many(struct device **clients, int num_clients) | |
614 | { | |
615 | struct pci_dev *pdev = NULL; | |
616 | int distance; | |
617 | int closest_distance = INT_MAX; | |
618 | struct pci_dev **closest_pdevs; | |
619 | int dev_cnt = 0; | |
620 | const int max_devs = PAGE_SIZE / sizeof(*closest_pdevs); | |
621 | int i; | |
622 | ||
623 | closest_pdevs = kmalloc(PAGE_SIZE, GFP_KERNEL); | |
624 | if (!closest_pdevs) | |
625 | return NULL; | |
626 | ||
627 | while ((pdev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, pdev))) { | |
628 | if (!pci_has_p2pmem(pdev)) | |
629 | continue; | |
630 | ||
631 | distance = pci_p2pdma_distance_many(pdev, clients, | |
632 | num_clients, false); | |
633 | if (distance < 0 || distance > closest_distance) | |
634 | continue; | |
635 | ||
636 | if (distance == closest_distance && dev_cnt >= max_devs) | |
637 | continue; | |
638 | ||
639 | if (distance < closest_distance) { | |
640 | for (i = 0; i < dev_cnt; i++) | |
641 | pci_dev_put(closest_pdevs[i]); | |
642 | ||
643 | dev_cnt = 0; | |
644 | closest_distance = distance; | |
645 | } | |
646 | ||
647 | closest_pdevs[dev_cnt++] = pci_dev_get(pdev); | |
648 | } | |
649 | ||
650 | if (dev_cnt) | |
651 | pdev = pci_dev_get(closest_pdevs[prandom_u32_max(dev_cnt)]); | |
652 | ||
653 | for (i = 0; i < dev_cnt; i++) | |
654 | pci_dev_put(closest_pdevs[i]); | |
655 | ||
656 | kfree(closest_pdevs); | |
657 | return pdev; | |
658 | } | |
659 | EXPORT_SYMBOL_GPL(pci_p2pmem_find_many); | |
660 | ||
661 | /** | |
662 | * pci_alloc_p2p_mem - allocate peer-to-peer DMA memory | |
663 | * @pdev: the device to allocate memory from | |
664 | * @size: number of bytes to allocate | |
665 | * | |
666 | * Returns the allocated memory or NULL on error. | |
667 | */ | |
668 | void *pci_alloc_p2pmem(struct pci_dev *pdev, size_t size) | |
669 | { | |
1570175a DW |
670 | void *ret = NULL; |
671 | struct percpu_ref *ref; | |
52916982 | 672 | |
1570175a DW |
673 | /* |
674 | * Pairs with synchronize_rcu() in pci_p2pdma_release() to | |
675 | * ensure pdev->p2pdma is non-NULL for the duration of the | |
676 | * read-lock. | |
677 | */ | |
678 | rcu_read_lock(); | |
52916982 | 679 | if (unlikely(!pdev->p2pdma)) |
1570175a | 680 | goto out; |
52916982 | 681 | |
1570175a DW |
682 | ret = (void *)gen_pool_alloc_owner(pdev->p2pdma->pool, size, |
683 | (void **) &ref); | |
684 | if (!ret) | |
685 | goto out; | |
52916982 | 686 | |
1570175a DW |
687 | if (unlikely(!percpu_ref_tryget_live(ref))) { |
688 | gen_pool_free(pdev->p2pdma->pool, (unsigned long) ret, size); | |
689 | ret = NULL; | |
690 | goto out; | |
691 | } | |
692 | out: | |
693 | rcu_read_unlock(); | |
52916982 LG |
694 | return ret; |
695 | } | |
696 | EXPORT_SYMBOL_GPL(pci_alloc_p2pmem); | |
697 | ||
698 | /** | |
699 | * pci_free_p2pmem - free peer-to-peer DMA memory | |
700 | * @pdev: the device the memory was allocated from | |
701 | * @addr: address of the memory that was allocated | |
fcf9ab35 | 702 | * @size: number of bytes that were allocated |
52916982 LG |
703 | */ |
704 | void pci_free_p2pmem(struct pci_dev *pdev, void *addr, size_t size) | |
705 | { | |
1570175a DW |
706 | struct percpu_ref *ref; |
707 | ||
708 | gen_pool_free_owner(pdev->p2pdma->pool, (uintptr_t)addr, size, | |
709 | (void **) &ref); | |
710 | percpu_ref_put(ref); | |
52916982 LG |
711 | } |
712 | EXPORT_SYMBOL_GPL(pci_free_p2pmem); | |
713 | ||
714 | /** | |
715 | * pci_virt_to_bus - return the PCI bus address for a given virtual | |
716 | * address obtained with pci_alloc_p2pmem() | |
717 | * @pdev: the device the memory was allocated from | |
718 | * @addr: address of the memory that was allocated | |
719 | */ | |
720 | pci_bus_addr_t pci_p2pmem_virt_to_bus(struct pci_dev *pdev, void *addr) | |
721 | { | |
722 | if (!addr) | |
723 | return 0; | |
724 | if (!pdev->p2pdma) | |
725 | return 0; | |
726 | ||
727 | /* | |
728 | * Note: when we added the memory to the pool we used the PCI | |
729 | * bus address as the physical address. So gen_pool_virt_to_phys() | |
730 | * actually returns the bus address despite the misleading name. | |
731 | */ | |
732 | return gen_pool_virt_to_phys(pdev->p2pdma->pool, (unsigned long)addr); | |
733 | } | |
734 | EXPORT_SYMBOL_GPL(pci_p2pmem_virt_to_bus); | |
735 | ||
736 | /** | |
737 | * pci_p2pmem_alloc_sgl - allocate peer-to-peer DMA memory in a scatterlist | |
738 | * @pdev: the device to allocate memory from | |
739 | * @nents: the number of SG entries in the list | |
740 | * @length: number of bytes to allocate | |
741 | * | |
fcf9ab35 | 742 | * Return: %NULL on error or &struct scatterlist pointer and @nents on success |
52916982 LG |
743 | */ |
744 | struct scatterlist *pci_p2pmem_alloc_sgl(struct pci_dev *pdev, | |
745 | unsigned int *nents, u32 length) | |
746 | { | |
747 | struct scatterlist *sg; | |
748 | void *addr; | |
749 | ||
750 | sg = kzalloc(sizeof(*sg), GFP_KERNEL); | |
751 | if (!sg) | |
752 | return NULL; | |
753 | ||
754 | sg_init_table(sg, 1); | |
755 | ||
756 | addr = pci_alloc_p2pmem(pdev, length); | |
757 | if (!addr) | |
758 | goto out_free_sg; | |
759 | ||
760 | sg_set_buf(sg, addr, length); | |
761 | *nents = 1; | |
762 | return sg; | |
763 | ||
764 | out_free_sg: | |
765 | kfree(sg); | |
766 | return NULL; | |
767 | } | |
768 | EXPORT_SYMBOL_GPL(pci_p2pmem_alloc_sgl); | |
769 | ||
770 | /** | |
771 | * pci_p2pmem_free_sgl - free a scatterlist allocated by pci_p2pmem_alloc_sgl() | |
772 | * @pdev: the device to allocate memory from | |
773 | * @sgl: the allocated scatterlist | |
774 | */ | |
775 | void pci_p2pmem_free_sgl(struct pci_dev *pdev, struct scatterlist *sgl) | |
776 | { | |
777 | struct scatterlist *sg; | |
778 | int count; | |
779 | ||
780 | for_each_sg(sgl, sg, INT_MAX, count) { | |
781 | if (!sg) | |
782 | break; | |
783 | ||
784 | pci_free_p2pmem(pdev, sg_virt(sg), sg->length); | |
785 | } | |
786 | kfree(sgl); | |
787 | } | |
788 | EXPORT_SYMBOL_GPL(pci_p2pmem_free_sgl); | |
789 | ||
790 | /** | |
791 | * pci_p2pmem_publish - publish the peer-to-peer DMA memory for use by | |
792 | * other devices with pci_p2pmem_find() | |
793 | * @pdev: the device with peer-to-peer DMA memory to publish | |
794 | * @publish: set to true to publish the memory, false to unpublish it | |
795 | * | |
796 | * Published memory can be used by other PCI device drivers for | |
797 | * peer-2-peer DMA operations. Non-published memory is reserved for | |
fcf9ab35 | 798 | * exclusive use of the device driver that registers the peer-to-peer |
52916982 LG |
799 | * memory. |
800 | */ | |
801 | void pci_p2pmem_publish(struct pci_dev *pdev, bool publish) | |
802 | { | |
803 | if (pdev->p2pdma) | |
804 | pdev->p2pdma->p2pmem_published = publish; | |
805 | } | |
806 | EXPORT_SYMBOL_GPL(pci_p2pmem_publish); | |
977196b8 | 807 | |
5d52e1ab LG |
808 | static enum pci_p2pdma_map_type pci_p2pdma_map_type(struct pci_dev *provider, |
809 | struct pci_dev *client) | |
810 | { | |
811 | if (!provider->p2pdma) | |
812 | return PCI_P2PDMA_MAP_NOT_SUPPORTED; | |
813 | ||
814 | return xa_to_value(xa_load(&provider->p2pdma->map_types, | |
815 | map_types_idx(client))); | |
816 | } | |
817 | ||
142c242e LG |
818 | static int __pci_p2pdma_map_sg(struct pci_p2pdma_pagemap *p2p_pgmap, |
819 | struct device *dev, struct scatterlist *sg, int nents) | |
977196b8 | 820 | { |
977196b8 LG |
821 | struct scatterlist *s; |
822 | phys_addr_t paddr; | |
823 | int i; | |
824 | ||
825 | /* | |
826 | * p2pdma mappings are not compatible with devices that use | |
827 | * dma_virt_ops. If the upper layers do the right thing | |
828 | * this should never happen because it will be prevented | |
9c002bb6 | 829 | * by the check in pci_p2pdma_distance_many() |
977196b8 LG |
830 | */ |
831 | if (WARN_ON_ONCE(IS_ENABLED(CONFIG_DMA_VIRT_OPS) && | |
832 | dev->dma_ops == &dma_virt_ops)) | |
833 | return 0; | |
834 | ||
835 | for_each_sg(sg, s, nents, i) { | |
977196b8 LG |
836 | paddr = sg_phys(s); |
837 | ||
a6e6fe65 | 838 | s->dma_address = paddr - p2p_pgmap->bus_offset; |
977196b8 LG |
839 | sg_dma_len(s) = s->length; |
840 | } | |
841 | ||
842 | return nents; | |
843 | } | |
142c242e LG |
844 | |
845 | /** | |
846 | * pci_p2pdma_map_sg - map a PCI peer-to-peer scatterlist for DMA | |
847 | * @dev: device doing the DMA request | |
848 | * @sg: scatter list to map | |
849 | * @nents: elements in the scatterlist | |
850 | * @dir: DMA direction | |
851 | * @attrs: DMA attributes passed to dma_map_sg() (if called) | |
852 | * | |
853 | * Scatterlists mapped with this function should be unmapped using | |
854 | * pci_p2pdma_unmap_sg_attrs(). | |
855 | * | |
856 | * Returns the number of SG entries mapped or 0 on error. | |
857 | */ | |
858 | int pci_p2pdma_map_sg_attrs(struct device *dev, struct scatterlist *sg, | |
859 | int nents, enum dma_data_direction dir, unsigned long attrs) | |
860 | { | |
861 | struct pci_p2pdma_pagemap *p2p_pgmap = | |
862 | to_p2p_pgmap(sg_page(sg)->pgmap); | |
5d52e1ab | 863 | struct pci_dev *client; |
142c242e | 864 | |
5d52e1ab LG |
865 | if (WARN_ON_ONCE(!dev_is_pci(dev))) |
866 | return 0; | |
867 | ||
868 | client = to_pci_dev(dev); | |
869 | ||
870 | switch (pci_p2pdma_map_type(p2p_pgmap->provider, client)) { | |
871 | case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE: | |
872 | return dma_map_sg_attrs(dev, sg, nents, dir, attrs); | |
873 | case PCI_P2PDMA_MAP_BUS_ADDR: | |
874 | return __pci_p2pdma_map_sg(p2p_pgmap, dev, sg, nents); | |
875 | default: | |
876 | WARN_ON_ONCE(1); | |
877 | return 0; | |
878 | } | |
142c242e | 879 | } |
2b9f4bb2 | 880 | EXPORT_SYMBOL_GPL(pci_p2pdma_map_sg_attrs); |
2d7bc010 | 881 | |
7f73eac3 LG |
882 | /** |
883 | * pci_p2pdma_unmap_sg - unmap a PCI peer-to-peer scatterlist that was | |
884 | * mapped with pci_p2pdma_map_sg() | |
885 | * @dev: device doing the DMA request | |
886 | * @sg: scatter list to map | |
887 | * @nents: number of elements returned by pci_p2pdma_map_sg() | |
888 | * @dir: DMA direction | |
889 | * @attrs: DMA attributes passed to dma_unmap_sg() (if called) | |
890 | */ | |
891 | void pci_p2pdma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg, | |
892 | int nents, enum dma_data_direction dir, unsigned long attrs) | |
893 | { | |
5d52e1ab LG |
894 | struct pci_p2pdma_pagemap *p2p_pgmap = |
895 | to_p2p_pgmap(sg_page(sg)->pgmap); | |
896 | enum pci_p2pdma_map_type map_type; | |
897 | struct pci_dev *client; | |
898 | ||
899 | if (WARN_ON_ONCE(!dev_is_pci(dev))) | |
900 | return; | |
901 | ||
902 | client = to_pci_dev(dev); | |
903 | ||
904 | map_type = pci_p2pdma_map_type(p2p_pgmap->provider, client); | |
905 | ||
906 | if (map_type == PCI_P2PDMA_MAP_THRU_HOST_BRIDGE) | |
907 | dma_unmap_sg_attrs(dev, sg, nents, dir, attrs); | |
7f73eac3 LG |
908 | } |
909 | EXPORT_SYMBOL_GPL(pci_p2pdma_unmap_sg_attrs); | |
910 | ||
2d7bc010 LG |
911 | /** |
912 | * pci_p2pdma_enable_store - parse a configfs/sysfs attribute store | |
913 | * to enable p2pdma | |
914 | * @page: contents of the value to be stored | |
915 | * @p2p_dev: returns the PCI device that was selected to be used | |
916 | * (if one was specified in the stored value) | |
917 | * @use_p2pdma: returns whether to enable p2pdma or not | |
918 | * | |
919 | * Parses an attribute value to decide whether to enable p2pdma. | |
fcf9ab35 | 920 | * The value can select a PCI device (using its full BDF device |
2d7bc010 LG |
921 | * name) or a boolean (in any format strtobool() accepts). A false |
922 | * value disables p2pdma, a true value expects the caller | |
923 | * to automatically find a compatible device and specifying a PCI device | |
924 | * expects the caller to use the specific provider. | |
925 | * | |
926 | * pci_p2pdma_enable_show() should be used as the show operation for | |
927 | * the attribute. | |
928 | * | |
929 | * Returns 0 on success | |
930 | */ | |
931 | int pci_p2pdma_enable_store(const char *page, struct pci_dev **p2p_dev, | |
932 | bool *use_p2pdma) | |
933 | { | |
934 | struct device *dev; | |
935 | ||
936 | dev = bus_find_device_by_name(&pci_bus_type, NULL, page); | |
937 | if (dev) { | |
938 | *use_p2pdma = true; | |
939 | *p2p_dev = to_pci_dev(dev); | |
940 | ||
941 | if (!pci_has_p2pmem(*p2p_dev)) { | |
942 | pci_err(*p2p_dev, | |
943 | "PCI device has no peer-to-peer memory: %s\n", | |
944 | page); | |
945 | pci_dev_put(*p2p_dev); | |
946 | return -ENODEV; | |
947 | } | |
948 | ||
949 | return 0; | |
950 | } else if ((page[0] == '0' || page[0] == '1') && !iscntrl(page[1])) { | |
951 | /* | |
952 | * If the user enters a PCI device that doesn't exist | |
953 | * like "0000:01:00.1", we don't want strtobool to think | |
954 | * it's a '0' when it's clearly not what the user wanted. | |
955 | * So we require 0's and 1's to be exactly one character. | |
956 | */ | |
957 | } else if (!strtobool(page, use_p2pdma)) { | |
958 | return 0; | |
959 | } | |
960 | ||
961 | pr_err("No such PCI device: %.*s\n", (int)strcspn(page, "\n"), page); | |
962 | return -ENODEV; | |
963 | } | |
964 | EXPORT_SYMBOL_GPL(pci_p2pdma_enable_store); | |
965 | ||
966 | /** | |
967 | * pci_p2pdma_enable_show - show a configfs/sysfs attribute indicating | |
968 | * whether p2pdma is enabled | |
969 | * @page: contents of the stored value | |
970 | * @p2p_dev: the selected p2p device (NULL if no device is selected) | |
fcf9ab35 | 971 | * @use_p2pdma: whether p2pdma has been enabled |
2d7bc010 LG |
972 | * |
973 | * Attributes that use pci_p2pdma_enable_store() should use this function | |
974 | * to show the value of the attribute. | |
975 | * | |
976 | * Returns 0 on success | |
977 | */ | |
978 | ssize_t pci_p2pdma_enable_show(char *page, struct pci_dev *p2p_dev, | |
979 | bool use_p2pdma) | |
980 | { | |
981 | if (!use_p2pdma) | |
982 | return sprintf(page, "0\n"); | |
983 | ||
984 | if (!p2p_dev) | |
985 | return sprintf(page, "1\n"); | |
986 | ||
987 | return sprintf(page, "%s\n", pci_name(p2p_dev)); | |
988 | } | |
989 | EXPORT_SYMBOL_GPL(pci_p2pdma_enable_show); |