]>
Commit | Line | Data |
---|---|---|
5d2aa710 AP |
1 | /* |
2 | * This file implements the DMA operations for NVLink devices. The NPU | |
3 | * devices all point to the same iommu table as the parent PCI device. | |
4 | * | |
5 | * Copyright Alistair Popple, IBM Corporation 2015. | |
6 | * | |
7 | * This program is free software; you can redistribute it and/or | |
8 | * modify it under the terms of version 2 of the GNU General Public | |
9 | * License as published by the Free Software Foundation. | |
10 | */ | |
11 | ||
1ab66d1f AP |
12 | #include <linux/slab.h> |
13 | #include <linux/mmu_notifier.h> | |
14 | #include <linux/mmu_context.h> | |
15 | #include <linux/of.h> | |
5d2aa710 AP |
16 | #include <linux/export.h> |
17 | #include <linux/pci.h> | |
18 | #include <linux/memblock.h> | |
b5cb9ab1 | 19 | #include <linux/iommu.h> |
5d2aa710 | 20 | |
1ab66d1f AP |
21 | #include <asm/tlb.h> |
22 | #include <asm/powernv.h> | |
23 | #include <asm/reg.h> | |
24 | #include <asm/opal.h> | |
25 | #include <asm/io.h> | |
5d2aa710 AP |
26 | #include <asm/iommu.h> |
27 | #include <asm/pnv-pci.h> | |
28 | #include <asm/msi_bitmap.h> | |
29 | #include <asm/opal.h> | |
30 | ||
31 | #include "powernv.h" | |
32 | #include "pci.h" | |
33 | ||
1ab66d1f AP |
34 | #define npu_to_phb(x) container_of(x, struct pnv_phb, npu) |
35 | ||
5d2aa710 AP |
36 | /* |
37 | * Other types of TCE cache invalidation are not functional in the | |
38 | * hardware. | |
39 | */ | |
5d2aa710 AP |
40 | static struct pci_dev *get_pci_dev(struct device_node *dn) |
41 | { | |
42 | return PCI_DN(dn)->pcidev; | |
43 | } | |
44 | ||
45 | /* Given a NPU device get the associated PCI device. */ | |
46 | struct pci_dev *pnv_pci_get_gpu_dev(struct pci_dev *npdev) | |
47 | { | |
48 | struct device_node *dn; | |
49 | struct pci_dev *gpdev; | |
50 | ||
4c3b89ef AP |
51 | if (WARN_ON(!npdev)) |
52 | return NULL; | |
53 | ||
54 | if (WARN_ON(!npdev->dev.of_node)) | |
55 | return NULL; | |
56 | ||
5d2aa710 AP |
57 | /* Get assoicated PCI device */ |
58 | dn = of_parse_phandle(npdev->dev.of_node, "ibm,gpu", 0); | |
59 | if (!dn) | |
60 | return NULL; | |
61 | ||
62 | gpdev = get_pci_dev(dn); | |
63 | of_node_put(dn); | |
64 | ||
65 | return gpdev; | |
66 | } | |
67 | EXPORT_SYMBOL(pnv_pci_get_gpu_dev); | |
68 | ||
69 | /* Given the real PCI device get a linked NPU device. */ | |
70 | struct pci_dev *pnv_pci_get_npu_dev(struct pci_dev *gpdev, int index) | |
71 | { | |
72 | struct device_node *dn; | |
73 | struct pci_dev *npdev; | |
74 | ||
4c3b89ef AP |
75 | if (WARN_ON(!gpdev)) |
76 | return NULL; | |
77 | ||
377aa6b0 AP |
78 | /* Not all PCI devices have device-tree nodes */ |
79 | if (!gpdev->dev.of_node) | |
4c3b89ef AP |
80 | return NULL; |
81 | ||
5d2aa710 AP |
82 | /* Get assoicated PCI device */ |
83 | dn = of_parse_phandle(gpdev->dev.of_node, "ibm,npu", index); | |
84 | if (!dn) | |
85 | return NULL; | |
86 | ||
87 | npdev = get_pci_dev(dn); | |
88 | of_node_put(dn); | |
89 | ||
90 | return npdev; | |
91 | } | |
92 | EXPORT_SYMBOL(pnv_pci_get_npu_dev); | |
93 | ||
94 | #define NPU_DMA_OP_UNSUPPORTED() \ | |
95 | dev_err_once(dev, "%s operation unsupported for NVLink devices\n", \ | |
96 | __func__) | |
97 | ||
98 | static void *dma_npu_alloc(struct device *dev, size_t size, | |
99 | dma_addr_t *dma_handle, gfp_t flag, | |
00085f1e | 100 | unsigned long attrs) |
5d2aa710 AP |
101 | { |
102 | NPU_DMA_OP_UNSUPPORTED(); | |
103 | return NULL; | |
104 | } | |
105 | ||
106 | static void dma_npu_free(struct device *dev, size_t size, | |
107 | void *vaddr, dma_addr_t dma_handle, | |
00085f1e | 108 | unsigned long attrs) |
5d2aa710 AP |
109 | { |
110 | NPU_DMA_OP_UNSUPPORTED(); | |
111 | } | |
112 | ||
113 | static dma_addr_t dma_npu_map_page(struct device *dev, struct page *page, | |
114 | unsigned long offset, size_t size, | |
115 | enum dma_data_direction direction, | |
00085f1e | 116 | unsigned long attrs) |
5d2aa710 AP |
117 | { |
118 | NPU_DMA_OP_UNSUPPORTED(); | |
119 | return 0; | |
120 | } | |
121 | ||
122 | static int dma_npu_map_sg(struct device *dev, struct scatterlist *sglist, | |
123 | int nelems, enum dma_data_direction direction, | |
00085f1e | 124 | unsigned long attrs) |
5d2aa710 AP |
125 | { |
126 | NPU_DMA_OP_UNSUPPORTED(); | |
127 | return 0; | |
128 | } | |
129 | ||
130 | static int dma_npu_dma_supported(struct device *dev, u64 mask) | |
131 | { | |
132 | NPU_DMA_OP_UNSUPPORTED(); | |
133 | return 0; | |
134 | } | |
135 | ||
136 | static u64 dma_npu_get_required_mask(struct device *dev) | |
137 | { | |
138 | NPU_DMA_OP_UNSUPPORTED(); | |
139 | return 0; | |
140 | } | |
141 | ||
5299709d | 142 | static const struct dma_map_ops dma_npu_ops = { |
5d2aa710 AP |
143 | .map_page = dma_npu_map_page, |
144 | .map_sg = dma_npu_map_sg, | |
145 | .alloc = dma_npu_alloc, | |
146 | .free = dma_npu_free, | |
147 | .dma_supported = dma_npu_dma_supported, | |
148 | .get_required_mask = dma_npu_get_required_mask, | |
149 | }; | |
150 | ||
151 | /* | |
152 | * Returns the PE assoicated with the PCI device of the given | |
153 | * NPU. Returns the linked pci device if pci_dev != NULL. | |
154 | */ | |
155 | static struct pnv_ioda_pe *get_gpu_pci_dev_and_pe(struct pnv_ioda_pe *npe, | |
156 | struct pci_dev **gpdev) | |
157 | { | |
158 | struct pnv_phb *phb; | |
159 | struct pci_controller *hose; | |
160 | struct pci_dev *pdev; | |
161 | struct pnv_ioda_pe *pe; | |
162 | struct pci_dn *pdn; | |
163 | ||
85674868 AK |
164 | pdev = pnv_pci_get_gpu_dev(npe->pdev); |
165 | if (!pdev) | |
166 | return NULL; | |
5d2aa710 | 167 | |
85674868 AK |
168 | pdn = pci_get_pdn(pdev); |
169 | if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE)) | |
170 | return NULL; | |
171 | ||
172 | hose = pci_bus_to_host(pdev->bus); | |
173 | phb = hose->private_data; | |
174 | pe = &phb->ioda.pe_array[pdn->pe_number]; | |
5d2aa710 AP |
175 | |
176 | if (gpdev) | |
177 | *gpdev = pdev; | |
178 | ||
179 | return pe; | |
180 | } | |
181 | ||
b5cb9ab1 | 182 | long pnv_npu_set_window(struct pnv_ioda_pe *npe, int num, |
b575c731 AK |
183 | struct iommu_table *tbl) |
184 | { | |
185 | struct pnv_phb *phb = npe->phb; | |
186 | int64_t rc; | |
187 | const unsigned long size = tbl->it_indirect_levels ? | |
188 | tbl->it_level_size : tbl->it_size; | |
189 | const __u64 start_addr = tbl->it_offset << tbl->it_page_shift; | |
190 | const __u64 win_size = tbl->it_size << tbl->it_page_shift; | |
191 | ||
192 | pe_info(npe, "Setting up window %llx..%llx pg=%lx\n", | |
193 | start_addr, start_addr + win_size - 1, | |
194 | IOMMU_PAGE_SIZE(tbl)); | |
195 | ||
196 | rc = opal_pci_map_pe_dma_window(phb->opal_id, | |
197 | npe->pe_number, | |
198 | npe->pe_number, | |
199 | tbl->it_indirect_levels + 1, | |
200 | __pa(tbl->it_base), | |
201 | size << 3, | |
202 | IOMMU_PAGE_SIZE(tbl)); | |
203 | if (rc) { | |
204 | pe_err(npe, "Failed to configure TCE table, err %lld\n", rc); | |
205 | return rc; | |
206 | } | |
6b3d12a9 | 207 | pnv_pci_ioda2_tce_invalidate_entire(phb, false); |
b575c731 | 208 | |
85674868 | 209 | /* Add the table to the list so its TCE cache will get invalidated */ |
b5cb9ab1 | 210 | pnv_pci_link_table_and_group(phb->hose->node, num, |
85674868 AK |
211 | tbl, &npe->table_group); |
212 | ||
b575c731 AK |
213 | return 0; |
214 | } | |
215 | ||
b5cb9ab1 | 216 | long pnv_npu_unset_window(struct pnv_ioda_pe *npe, int num) |
b575c731 AK |
217 | { |
218 | struct pnv_phb *phb = npe->phb; | |
219 | int64_t rc; | |
220 | ||
221 | pe_info(npe, "Removing DMA window\n"); | |
222 | ||
223 | rc = opal_pci_map_pe_dma_window(phb->opal_id, npe->pe_number, | |
224 | npe->pe_number, | |
225 | 0/* levels */, 0/* table address */, | |
226 | 0/* table size */, 0/* page size */); | |
227 | if (rc) { | |
228 | pe_err(npe, "Unmapping failed, ret = %lld\n", rc); | |
229 | return rc; | |
230 | } | |
6b3d12a9 | 231 | pnv_pci_ioda2_tce_invalidate_entire(phb, false); |
b575c731 | 232 | |
b5cb9ab1 | 233 | pnv_pci_unlink_table_and_group(npe->table_group.tables[num], |
85674868 | 234 | &npe->table_group); |
5d2aa710 | 235 | |
85674868 | 236 | return 0; |
5d2aa710 AP |
237 | } |
238 | ||
239 | /* | |
f9f83456 | 240 | * Enables 32 bit DMA on NPU. |
5d2aa710 | 241 | */ |
f9f83456 | 242 | static void pnv_npu_dma_set_32(struct pnv_ioda_pe *npe) |
5d2aa710 | 243 | { |
5d2aa710 AP |
244 | struct pci_dev *gpdev; |
245 | struct pnv_ioda_pe *gpe; | |
5d2aa710 AP |
246 | int64_t rc; |
247 | ||
248 | /* | |
249 | * Find the assoicated PCI devices and get the dma window | |
250 | * information from there. | |
251 | */ | |
252 | if (!npe->pdev || !(npe->flags & PNV_IODA_PE_DEV)) | |
253 | return; | |
254 | ||
255 | gpe = get_gpu_pci_dev_and_pe(npe, &gpdev); | |
256 | if (!gpe) | |
257 | return; | |
258 | ||
b5cb9ab1 | 259 | rc = pnv_npu_set_window(npe, 0, gpe->table_group.tables[0]); |
5d2aa710 AP |
260 | |
261 | /* | |
262 | * We don't initialise npu_pe->tce32_table as we always use | |
263 | * dma_npu_ops which are nops. | |
264 | */ | |
265 | set_dma_ops(&npe->pdev->dev, &dma_npu_ops); | |
266 | } | |
267 | ||
268 | /* | |
f9f83456 | 269 | * Enables bypass mode on the NPU. The NPU only supports one |
446957ba | 270 | * window per link, so bypass needs to be explicitly enabled or |
5d2aa710 AP |
271 | * disabled. Unlike for a PHB3 bypass and non-bypass modes can't be |
272 | * active at the same time. | |
273 | */ | |
f9f83456 | 274 | static int pnv_npu_dma_set_bypass(struct pnv_ioda_pe *npe) |
5d2aa710 AP |
275 | { |
276 | struct pnv_phb *phb = npe->phb; | |
277 | int64_t rc = 0; | |
f9f83456 | 278 | phys_addr_t top = memblock_end_of_DRAM(); |
5d2aa710 AP |
279 | |
280 | if (phb->type != PNV_PHB_NPU || !npe->pdev) | |
281 | return -EINVAL; | |
282 | ||
b5cb9ab1 | 283 | rc = pnv_npu_unset_window(npe, 0); |
b575c731 AK |
284 | if (rc != OPAL_SUCCESS) |
285 | return rc; | |
286 | ||
f9f83456 AK |
287 | /* Enable the bypass window */ |
288 | ||
289 | top = roundup_pow_of_two(top); | |
1f52f176 | 290 | dev_info(&npe->pdev->dev, "Enabling bypass for PE %x\n", |
f9f83456 AK |
291 | npe->pe_number); |
292 | rc = opal_pci_map_pe_dma_window_real(phb->opal_id, | |
293 | npe->pe_number, npe->pe_number, | |
294 | 0 /* bypass base */, top); | |
5d2aa710 | 295 | |
85674868 | 296 | if (rc == OPAL_SUCCESS) |
6b3d12a9 | 297 | pnv_pci_ioda2_tce_invalidate_entire(phb, false); |
85674868 | 298 | |
5d2aa710 AP |
299 | return rc; |
300 | } | |
301 | ||
f9f83456 | 302 | void pnv_npu_try_dma_set_bypass(struct pci_dev *gpdev, bool bypass) |
5d2aa710 | 303 | { |
f9f83456 AK |
304 | int i; |
305 | struct pnv_phb *phb; | |
306 | struct pci_dn *pdn; | |
307 | struct pnv_ioda_pe *npe; | |
308 | struct pci_dev *npdev; | |
5d2aa710 | 309 | |
f9f83456 AK |
310 | for (i = 0; ; ++i) { |
311 | npdev = pnv_pci_get_npu_dev(gpdev, i); | |
5d2aa710 | 312 | |
f9f83456 AK |
313 | if (!npdev) |
314 | break; | |
5d2aa710 | 315 | |
f9f83456 AK |
316 | pdn = pci_get_pdn(npdev); |
317 | if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE)) | |
318 | return; | |
5d2aa710 | 319 | |
f9f83456 | 320 | phb = pci_bus_to_host(npdev->bus)->private_data; |
5d2aa710 | 321 | |
f9f83456 AK |
322 | /* We only do bypass if it's enabled on the linked device */ |
323 | npe = &phb->ioda.pe_array[pdn->pe_number]; | |
5d2aa710 | 324 | |
f9f83456 AK |
325 | if (bypass) { |
326 | dev_info(&npdev->dev, | |
327 | "Using 64-bit DMA iommu bypass\n"); | |
328 | pnv_npu_dma_set_bypass(npe); | |
329 | } else { | |
330 | dev_info(&npdev->dev, "Using 32-bit DMA via iommu\n"); | |
331 | pnv_npu_dma_set_32(npe); | |
332 | } | |
333 | } | |
5d2aa710 | 334 | } |
b5cb9ab1 AK |
335 | |
336 | /* Switch ownership from platform code to external user (e.g. VFIO) */ | |
337 | void pnv_npu_take_ownership(struct pnv_ioda_pe *npe) | |
338 | { | |
339 | struct pnv_phb *phb = npe->phb; | |
340 | int64_t rc; | |
341 | ||
342 | /* | |
343 | * Note: NPU has just a single TVE in the hardware which means that | |
344 | * while used by the kernel, it can have either 32bit window or | |
345 | * DMA bypass but never both. So we deconfigure 32bit window only | |
346 | * if it was enabled at the moment of ownership change. | |
347 | */ | |
348 | if (npe->table_group.tables[0]) { | |
349 | pnv_npu_unset_window(npe, 0); | |
350 | return; | |
351 | } | |
352 | ||
353 | /* Disable bypass */ | |
354 | rc = opal_pci_map_pe_dma_window_real(phb->opal_id, | |
355 | npe->pe_number, npe->pe_number, | |
356 | 0 /* bypass base */, 0); | |
357 | if (rc) { | |
358 | pe_err(npe, "Failed to disable bypass, err %lld\n", rc); | |
359 | return; | |
360 | } | |
6b3d12a9 | 361 | pnv_pci_ioda2_tce_invalidate_entire(npe->phb, false); |
b5cb9ab1 AK |
362 | } |
363 | ||
364 | struct pnv_ioda_pe *pnv_pci_npu_setup_iommu(struct pnv_ioda_pe *npe) | |
365 | { | |
366 | struct pnv_phb *phb = npe->phb; | |
367 | struct pci_bus *pbus = phb->hose->bus; | |
368 | struct pci_dev *npdev, *gpdev = NULL, *gptmp; | |
369 | struct pnv_ioda_pe *gpe = get_gpu_pci_dev_and_pe(npe, &gpdev); | |
370 | ||
371 | if (!gpe || !gpdev) | |
372 | return NULL; | |
373 | ||
374 | list_for_each_entry(npdev, &pbus->devices, bus_list) { | |
375 | gptmp = pnv_pci_get_gpu_dev(npdev); | |
376 | ||
377 | if (gptmp != gpdev) | |
378 | continue; | |
379 | ||
380 | pe_info(gpe, "Attached NPU %s\n", dev_name(&npdev->dev)); | |
381 | iommu_group_add_device(gpe->table_group.group, &npdev->dev); | |
382 | } | |
383 | ||
384 | return gpe; | |
385 | } | |
1ab66d1f AP |
386 | |
387 | /* Maximum number of nvlinks per npu */ | |
388 | #define NV_MAX_LINKS 6 | |
389 | ||
390 | /* Maximum index of npu2 hosts in the system. Always < NV_MAX_NPUS */ | |
391 | static int max_npu2_index; | |
392 | ||
393 | struct npu_context { | |
394 | struct mm_struct *mm; | |
395 | struct pci_dev *npdev[NV_MAX_NPUS][NV_MAX_LINKS]; | |
396 | struct mmu_notifier mn; | |
397 | struct kref kref; | |
398 | ||
399 | /* Callback to stop translation requests on a given GPU */ | |
400 | struct npu_context *(*release_cb)(struct npu_context *, void *); | |
401 | ||
402 | /* | |
403 | * Private pointer passed to the above callback for usage by | |
404 | * device drivers. | |
405 | */ | |
406 | void *priv; | |
407 | }; | |
408 | ||
409 | /* | |
410 | * Find a free MMIO ATSD register and mark it in use. Return -ENOSPC | |
411 | * if none are available. | |
412 | */ | |
413 | static int get_mmio_atsd_reg(struct npu *npu) | |
414 | { | |
415 | int i; | |
416 | ||
417 | for (i = 0; i < npu->mmio_atsd_count; i++) { | |
418 | if (!test_and_set_bit(i, &npu->mmio_atsd_usage)) | |
419 | return i; | |
420 | } | |
421 | ||
422 | return -ENOSPC; | |
423 | } | |
424 | ||
425 | static void put_mmio_atsd_reg(struct npu *npu, int reg) | |
426 | { | |
427 | clear_bit(reg, &npu->mmio_atsd_usage); | |
428 | } | |
429 | ||
430 | /* MMIO ATSD register offsets */ | |
431 | #define XTS_ATSD_AVA 1 | |
432 | #define XTS_ATSD_STAT 2 | |
433 | ||
434 | static int mmio_launch_invalidate(struct npu *npu, unsigned long launch, | |
435 | unsigned long va) | |
436 | { | |
437 | int mmio_atsd_reg; | |
438 | ||
439 | do { | |
440 | mmio_atsd_reg = get_mmio_atsd_reg(npu); | |
441 | cpu_relax(); | |
442 | } while (mmio_atsd_reg < 0); | |
443 | ||
444 | __raw_writeq(cpu_to_be64(va), | |
445 | npu->mmio_atsd_regs[mmio_atsd_reg] + XTS_ATSD_AVA); | |
446 | eieio(); | |
447 | __raw_writeq(cpu_to_be64(launch), npu->mmio_atsd_regs[mmio_atsd_reg]); | |
448 | ||
449 | return mmio_atsd_reg; | |
450 | } | |
451 | ||
452 | static int mmio_invalidate_pid(struct npu *npu, unsigned long pid) | |
453 | { | |
454 | unsigned long launch; | |
455 | ||
456 | /* IS set to invalidate matching PID */ | |
457 | launch = PPC_BIT(12); | |
458 | ||
459 | /* PRS set to process-scoped */ | |
460 | launch |= PPC_BIT(13); | |
461 | ||
462 | /* AP */ | |
463 | launch |= (u64) mmu_get_ap(mmu_virtual_psize) << PPC_BITLSHIFT(17); | |
464 | ||
465 | /* PID */ | |
466 | launch |= pid << PPC_BITLSHIFT(38); | |
467 | ||
468 | /* Invalidating the entire process doesn't use a va */ | |
469 | return mmio_launch_invalidate(npu, launch, 0); | |
470 | } | |
471 | ||
472 | static int mmio_invalidate_va(struct npu *npu, unsigned long va, | |
473 | unsigned long pid) | |
474 | { | |
475 | unsigned long launch; | |
476 | ||
477 | /* IS set to invalidate target VA */ | |
478 | launch = 0; | |
479 | ||
480 | /* PRS set to process scoped */ | |
481 | launch |= PPC_BIT(13); | |
482 | ||
483 | /* AP */ | |
484 | launch |= (u64) mmu_get_ap(mmu_virtual_psize) << PPC_BITLSHIFT(17); | |
485 | ||
486 | /* PID */ | |
487 | launch |= pid << PPC_BITLSHIFT(38); | |
488 | ||
489 | return mmio_launch_invalidate(npu, launch, va); | |
490 | } | |
491 | ||
492 | #define mn_to_npu_context(x) container_of(x, struct npu_context, mn) | |
493 | ||
494 | /* | |
495 | * Invalidate either a single address or an entire PID depending on | |
496 | * the value of va. | |
497 | */ | |
498 | static void mmio_invalidate(struct npu_context *npu_context, int va, | |
499 | unsigned long address) | |
500 | { | |
501 | int i, j, reg; | |
502 | struct npu *npu; | |
503 | struct pnv_phb *nphb; | |
504 | struct pci_dev *npdev; | |
505 | struct { | |
506 | struct npu *npu; | |
507 | int reg; | |
508 | } mmio_atsd_reg[NV_MAX_NPUS]; | |
509 | unsigned long pid = npu_context->mm->context.id; | |
510 | ||
511 | /* | |
512 | * Loop over all the NPUs this process is active on and launch | |
513 | * an invalidate. | |
514 | */ | |
515 | for (i = 0; i <= max_npu2_index; i++) { | |
516 | mmio_atsd_reg[i].reg = -1; | |
517 | for (j = 0; j < NV_MAX_LINKS; j++) { | |
518 | npdev = npu_context->npdev[i][j]; | |
519 | if (!npdev) | |
520 | continue; | |
521 | ||
522 | nphb = pci_bus_to_host(npdev->bus)->private_data; | |
523 | npu = &nphb->npu; | |
524 | mmio_atsd_reg[i].npu = npu; | |
525 | ||
526 | if (va) | |
527 | mmio_atsd_reg[i].reg = | |
528 | mmio_invalidate_va(npu, address, pid); | |
529 | else | |
530 | mmio_atsd_reg[i].reg = | |
531 | mmio_invalidate_pid(npu, pid); | |
532 | ||
533 | /* | |
534 | * The NPU hardware forwards the shootdown to all GPUs | |
535 | * so we only have to launch one shootdown per NPU. | |
536 | */ | |
537 | break; | |
538 | } | |
539 | } | |
540 | ||
541 | /* | |
542 | * Unfortunately the nest mmu does not support flushing specific | |
543 | * addresses so we have to flush the whole mm. | |
544 | */ | |
545 | flush_tlb_mm(npu_context->mm); | |
546 | ||
547 | /* Wait for all invalidations to complete */ | |
548 | for (i = 0; i <= max_npu2_index; i++) { | |
549 | if (mmio_atsd_reg[i].reg < 0) | |
550 | continue; | |
551 | ||
552 | /* Wait for completion */ | |
553 | npu = mmio_atsd_reg[i].npu; | |
554 | reg = mmio_atsd_reg[i].reg; | |
555 | while (__raw_readq(npu->mmio_atsd_regs[reg] + XTS_ATSD_STAT)) | |
556 | cpu_relax(); | |
557 | put_mmio_atsd_reg(npu, reg); | |
558 | } | |
559 | } | |
560 | ||
561 | static void pnv_npu2_mn_release(struct mmu_notifier *mn, | |
562 | struct mm_struct *mm) | |
563 | { | |
564 | struct npu_context *npu_context = mn_to_npu_context(mn); | |
565 | ||
566 | /* Call into device driver to stop requests to the NMMU */ | |
567 | if (npu_context->release_cb) | |
568 | npu_context->release_cb(npu_context, npu_context->priv); | |
569 | ||
570 | /* | |
571 | * There should be no more translation requests for this PID, but we | |
572 | * need to ensure any entries for it are removed from the TLB. | |
573 | */ | |
574 | mmio_invalidate(npu_context, 0, 0); | |
575 | } | |
576 | ||
577 | static void pnv_npu2_mn_change_pte(struct mmu_notifier *mn, | |
578 | struct mm_struct *mm, | |
579 | unsigned long address, | |
580 | pte_t pte) | |
581 | { | |
582 | struct npu_context *npu_context = mn_to_npu_context(mn); | |
583 | ||
584 | mmio_invalidate(npu_context, 1, address); | |
585 | } | |
586 | ||
587 | static void pnv_npu2_mn_invalidate_page(struct mmu_notifier *mn, | |
588 | struct mm_struct *mm, | |
589 | unsigned long address) | |
590 | { | |
591 | struct npu_context *npu_context = mn_to_npu_context(mn); | |
592 | ||
593 | mmio_invalidate(npu_context, 1, address); | |
594 | } | |
595 | ||
596 | static void pnv_npu2_mn_invalidate_range(struct mmu_notifier *mn, | |
597 | struct mm_struct *mm, | |
598 | unsigned long start, unsigned long end) | |
599 | { | |
600 | struct npu_context *npu_context = mn_to_npu_context(mn); | |
601 | unsigned long address; | |
602 | ||
603 | for (address = start; address <= end; address += PAGE_SIZE) | |
604 | mmio_invalidate(npu_context, 1, address); | |
605 | } | |
606 | ||
607 | static const struct mmu_notifier_ops nv_nmmu_notifier_ops = { | |
608 | .release = pnv_npu2_mn_release, | |
609 | .change_pte = pnv_npu2_mn_change_pte, | |
610 | .invalidate_page = pnv_npu2_mn_invalidate_page, | |
611 | .invalidate_range = pnv_npu2_mn_invalidate_range, | |
612 | }; | |
613 | ||
614 | /* | |
615 | * Call into OPAL to setup the nmmu context for the current task in | |
616 | * the NPU. This must be called to setup the context tables before the | |
617 | * GPU issues ATRs. pdev should be a pointed to PCIe GPU device. | |
618 | * | |
619 | * A release callback should be registered to allow a device driver to | |
620 | * be notified that it should not launch any new translation requests | |
621 | * as the final TLB invalidate is about to occur. | |
622 | * | |
623 | * Returns an error if there no contexts are currently available or a | |
624 | * npu_context which should be passed to pnv_npu2_handle_fault(). | |
625 | * | |
626 | * mmap_sem must be held in write mode. | |
627 | */ | |
628 | struct npu_context *pnv_npu2_init_context(struct pci_dev *gpdev, | |
629 | unsigned long flags, | |
630 | struct npu_context *(*cb)(struct npu_context *, void *), | |
631 | void *priv) | |
632 | { | |
633 | int rc; | |
634 | u32 nvlink_index; | |
635 | struct device_node *nvlink_dn; | |
636 | struct mm_struct *mm = current->mm; | |
637 | struct pnv_phb *nphb; | |
638 | struct npu *npu; | |
639 | struct npu_context *npu_context; | |
640 | ||
641 | /* | |
642 | * At present we don't support GPUs connected to multiple NPUs and I'm | |
643 | * not sure the hardware does either. | |
644 | */ | |
645 | struct pci_dev *npdev = pnv_pci_get_npu_dev(gpdev, 0); | |
646 | ||
647 | if (!firmware_has_feature(FW_FEATURE_OPAL)) | |
648 | return ERR_PTR(-ENODEV); | |
649 | ||
650 | if (!npdev) | |
651 | /* No nvlink associated with this GPU device */ | |
652 | return ERR_PTR(-ENODEV); | |
653 | ||
654 | if (!mm) { | |
655 | /* kernel thread contexts are not supported */ | |
656 | return ERR_PTR(-EINVAL); | |
657 | } | |
658 | ||
659 | nphb = pci_bus_to_host(npdev->bus)->private_data; | |
660 | npu = &nphb->npu; | |
661 | ||
662 | /* | |
663 | * Setup the NPU context table for a particular GPU. These need to be | |
664 | * per-GPU as we need the tables to filter ATSDs when there are no | |
665 | * active contexts on a particular GPU. | |
666 | */ | |
667 | rc = opal_npu_init_context(nphb->opal_id, mm->context.id, flags, | |
668 | PCI_DEVID(gpdev->bus->number, gpdev->devfn)); | |
669 | if (rc < 0) | |
670 | return ERR_PTR(-ENOSPC); | |
671 | ||
672 | /* | |
673 | * We store the npu pci device so we can more easily get at the | |
674 | * associated npus. | |
675 | */ | |
676 | npu_context = mm->context.npu_context; | |
677 | if (!npu_context) { | |
678 | npu_context = kzalloc(sizeof(struct npu_context), GFP_KERNEL); | |
679 | if (!npu_context) | |
680 | return ERR_PTR(-ENOMEM); | |
681 | ||
682 | mm->context.npu_context = npu_context; | |
683 | npu_context->mm = mm; | |
684 | npu_context->mn.ops = &nv_nmmu_notifier_ops; | |
685 | __mmu_notifier_register(&npu_context->mn, mm); | |
686 | kref_init(&npu_context->kref); | |
687 | } else { | |
688 | kref_get(&npu_context->kref); | |
689 | } | |
690 | ||
691 | npu_context->release_cb = cb; | |
692 | npu_context->priv = priv; | |
693 | nvlink_dn = of_parse_phandle(npdev->dev.of_node, "ibm,nvlink", 0); | |
694 | if (WARN_ON(of_property_read_u32(nvlink_dn, "ibm,npu-link-index", | |
695 | &nvlink_index))) | |
696 | return ERR_PTR(-ENODEV); | |
697 | npu_context->npdev[npu->index][nvlink_index] = npdev; | |
698 | ||
699 | return npu_context; | |
700 | } | |
701 | EXPORT_SYMBOL(pnv_npu2_init_context); | |
702 | ||
703 | static void pnv_npu2_release_context(struct kref *kref) | |
704 | { | |
705 | struct npu_context *npu_context = | |
706 | container_of(kref, struct npu_context, kref); | |
707 | ||
708 | npu_context->mm->context.npu_context = NULL; | |
709 | mmu_notifier_unregister(&npu_context->mn, | |
710 | npu_context->mm); | |
711 | ||
712 | kfree(npu_context); | |
713 | } | |
714 | ||
715 | void pnv_npu2_destroy_context(struct npu_context *npu_context, | |
716 | struct pci_dev *gpdev) | |
717 | { | |
415ba3c1 | 718 | struct pnv_phb *nphb; |
1ab66d1f AP |
719 | struct npu *npu; |
720 | struct pci_dev *npdev = pnv_pci_get_npu_dev(gpdev, 0); | |
721 | struct device_node *nvlink_dn; | |
722 | u32 nvlink_index; | |
723 | ||
724 | if (WARN_ON(!npdev)) | |
725 | return; | |
726 | ||
727 | if (!firmware_has_feature(FW_FEATURE_OPAL)) | |
728 | return; | |
729 | ||
730 | nphb = pci_bus_to_host(npdev->bus)->private_data; | |
731 | npu = &nphb->npu; | |
1ab66d1f AP |
732 | nvlink_dn = of_parse_phandle(npdev->dev.of_node, "ibm,nvlink", 0); |
733 | if (WARN_ON(of_property_read_u32(nvlink_dn, "ibm,npu-link-index", | |
734 | &nvlink_index))) | |
735 | return; | |
736 | npu_context->npdev[npu->index][nvlink_index] = NULL; | |
415ba3c1 | 737 | opal_npu_destroy_context(nphb->opal_id, npu_context->mm->context.id, |
1ab66d1f AP |
738 | PCI_DEVID(gpdev->bus->number, gpdev->devfn)); |
739 | kref_put(&npu_context->kref, pnv_npu2_release_context); | |
740 | } | |
741 | EXPORT_SYMBOL(pnv_npu2_destroy_context); | |
742 | ||
743 | /* | |
744 | * Assumes mmap_sem is held for the contexts associated mm. | |
745 | */ | |
746 | int pnv_npu2_handle_fault(struct npu_context *context, uintptr_t *ea, | |
747 | unsigned long *flags, unsigned long *status, int count) | |
748 | { | |
749 | u64 rc = 0, result = 0; | |
750 | int i, is_write; | |
751 | struct page *page[1]; | |
752 | ||
753 | /* mmap_sem should be held so the struct_mm must be present */ | |
754 | struct mm_struct *mm = context->mm; | |
755 | ||
756 | if (!firmware_has_feature(FW_FEATURE_OPAL)) | |
757 | return -ENODEV; | |
758 | ||
759 | WARN_ON(!rwsem_is_locked(&mm->mmap_sem)); | |
760 | ||
761 | for (i = 0; i < count; i++) { | |
762 | is_write = flags[i] & NPU2_WRITE; | |
763 | rc = get_user_pages_remote(NULL, mm, ea[i], 1, | |
764 | is_write ? FOLL_WRITE : 0, | |
765 | page, NULL, NULL); | |
766 | ||
767 | /* | |
768 | * To support virtualised environments we will have to do an | |
769 | * access to the page to ensure it gets faulted into the | |
770 | * hypervisor. For the moment virtualisation is not supported in | |
771 | * other areas so leave the access out. | |
772 | */ | |
773 | if (rc != 1) { | |
774 | status[i] = rc; | |
775 | result = -EFAULT; | |
776 | continue; | |
777 | } | |
778 | ||
779 | status[i] = 0; | |
780 | put_page(page[0]); | |
781 | } | |
782 | ||
783 | return result; | |
784 | } | |
785 | EXPORT_SYMBOL(pnv_npu2_handle_fault); | |
786 | ||
787 | int pnv_npu2_init(struct pnv_phb *phb) | |
788 | { | |
789 | unsigned int i; | |
790 | u64 mmio_atsd; | |
791 | struct device_node *dn; | |
792 | struct pci_dev *gpdev; | |
793 | static int npu_index; | |
794 | uint64_t rc = 0; | |
795 | ||
796 | for_each_child_of_node(phb->hose->dn, dn) { | |
797 | gpdev = pnv_pci_get_gpu_dev(get_pci_dev(dn)); | |
798 | if (gpdev) { | |
799 | rc = opal_npu_map_lpar(phb->opal_id, | |
800 | PCI_DEVID(gpdev->bus->number, gpdev->devfn), | |
801 | 0, 0); | |
802 | if (rc) | |
803 | dev_err(&gpdev->dev, | |
804 | "Error %lld mapping device to LPAR\n", | |
805 | rc); | |
806 | } | |
807 | } | |
808 | ||
809 | for (i = 0; !of_property_read_u64_index(phb->hose->dn, "ibm,mmio-atsd", | |
810 | i, &mmio_atsd); i++) | |
811 | phb->npu.mmio_atsd_regs[i] = ioremap(mmio_atsd, 32); | |
812 | ||
813 | pr_info("NPU%lld: Found %d MMIO ATSD registers", phb->opal_id, i); | |
814 | phb->npu.mmio_atsd_count = i; | |
815 | phb->npu.mmio_atsd_usage = 0; | |
816 | npu_index++; | |
817 | if (WARN_ON(npu_index >= NV_MAX_NPUS)) | |
818 | return -ENOSPC; | |
819 | max_npu2_index = npu_index; | |
820 | phb->npu.index = npu_index; | |
821 | ||
822 | return 0; | |
823 | } |