2 * Copyright © 2015 Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * Authors: David Woodhouse <dwmw2@infradead.org>
16 #include <linux/intel-iommu.h>
17 #include <linux/mmu_notifier.h>
18 #include <linux/sched.h>
19 #include <linux/slab.h>
20 #include <linux/intel-svm.h>
21 #include <linux/rculist.h>
22 #include <linux/pci.h>
23 #include <linux/pci-ats.h>
24 #include <linux/dmar.h>
25 #include <linux/interrupt.h>
27 static irqreturn_t
prq_event_thread(int irq
, void *d
);
33 struct pasid_state_entry
{
37 int intel_svm_alloc_pasid_tables(struct intel_iommu
*iommu
)
42 order
= ecap_pss(iommu
->ecap
) + 7 - PAGE_SHIFT
;
46 pages
= alloc_pages(GFP_KERNEL
| __GFP_ZERO
, order
);
48 pr_warn("IOMMU: %s: Failed to allocate PASID table\n",
52 iommu
->pasid_table
= page_address(pages
);
53 pr_info("%s: Allocated order %d PASID table.\n", iommu
->name
, order
);
55 if (ecap_dis(iommu
->ecap
)) {
56 pages
= alloc_pages(GFP_KERNEL
| __GFP_ZERO
, order
);
58 iommu
->pasid_state_table
= page_address(pages
);
60 pr_warn("IOMMU: %s: Failed to allocate PASID state table\n",
64 idr_init(&iommu
->pasid_idr
);
69 int intel_svm_free_pasid_tables(struct intel_iommu
*iommu
)
73 order
= ecap_pss(iommu
->ecap
) + 7 - PAGE_SHIFT
;
77 if (iommu
->pasid_table
) {
78 free_pages((unsigned long)iommu
->pasid_table
, order
);
79 iommu
->pasid_table
= NULL
;
81 if (iommu
->pasid_state_table
) {
82 free_pages((unsigned long)iommu
->pasid_state_table
, order
);
83 iommu
->pasid_state_table
= NULL
;
85 idr_destroy(&iommu
->pasid_idr
);
91 int intel_svm_enable_prq(struct intel_iommu
*iommu
)
96 pages
= alloc_pages(GFP_KERNEL
| __GFP_ZERO
, PRQ_ORDER
);
98 pr_warn("IOMMU: %s: Failed to allocate page request queue\n",
102 iommu
->prq
= page_address(pages
);
104 irq
= dmar_alloc_hwirq(DMAR_UNITS_SUPPORTED
+ iommu
->seq_id
, iommu
->node
, iommu
);
106 pr_err("IOMMU: %s: Failed to create IRQ vector for page request queue\n",
110 free_pages((unsigned long)iommu
->prq
, PRQ_ORDER
);
116 snprintf(iommu
->prq_name
, sizeof(iommu
->prq_name
), "dmar%d-prq", iommu
->seq_id
);
118 ret
= request_threaded_irq(irq
, NULL
, prq_event_thread
, IRQF_ONESHOT
,
119 iommu
->prq_name
, iommu
);
121 pr_err("IOMMU: %s: Failed to request IRQ for page request queue\n",
123 dmar_free_hwirq(irq
);
126 dmar_writeq(iommu
->reg
+ DMAR_PQH_REG
, 0ULL);
127 dmar_writeq(iommu
->reg
+ DMAR_PQT_REG
, 0ULL);
128 dmar_writeq(iommu
->reg
+ DMAR_PQA_REG
, virt_to_phys(iommu
->prq
) | PRQ_ORDER
);
133 int intel_svm_finish_prq(struct intel_iommu
*iommu
)
135 dmar_writeq(iommu
->reg
+ DMAR_PQH_REG
, 0ULL);
136 dmar_writeq(iommu
->reg
+ DMAR_PQT_REG
, 0ULL);
137 dmar_writeq(iommu
->reg
+ DMAR_PQA_REG
, 0ULL);
139 free_irq(iommu
->pr_irq
, iommu
);
140 dmar_free_hwirq(iommu
->pr_irq
);
143 free_pages((unsigned long)iommu
->prq
, PRQ_ORDER
);
149 static void intel_flush_svm_range_dev (struct intel_svm
*svm
, struct intel_svm_dev
*sdev
,
150 unsigned long address
, unsigned long pages
, int ih
, int gl
)
155 /* For global kernel pages we have to flush them in *all* PASIDs
156 * because that's the only option the hardware gives us. Despite
157 * the fact that they are actually only accessible through one. */
159 desc
.low
= QI_EIOTLB_PASID(svm
->pasid
) | QI_EIOTLB_DID(sdev
->did
) |
160 QI_EIOTLB_GRAN(QI_GRAN_ALL_ALL
) | QI_EIOTLB_TYPE
;
162 desc
.low
= QI_EIOTLB_PASID(svm
->pasid
) | QI_EIOTLB_DID(sdev
->did
) |
163 QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID
) | QI_EIOTLB_TYPE
;
166 int mask
= ilog2(__roundup_pow_of_two(pages
));
168 desc
.low
= QI_EIOTLB_PASID(svm
->pasid
) | QI_EIOTLB_DID(sdev
->did
) |
169 QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID
) | QI_EIOTLB_TYPE
;
170 desc
.high
= QI_EIOTLB_ADDR(address
) | QI_EIOTLB_GL(gl
) |
171 QI_EIOTLB_IH(ih
) | QI_EIOTLB_AM(mask
);
173 qi_submit_sync(&desc
, svm
->iommu
);
175 if (sdev
->dev_iotlb
) {
176 desc
.low
= QI_DEV_EIOTLB_PASID(svm
->pasid
) | QI_DEV_EIOTLB_SID(sdev
->sid
) |
177 QI_DEV_EIOTLB_QDEP(sdev
->qdep
) | QI_DEIOTLB_TYPE
;
179 desc
.high
= QI_DEV_EIOTLB_ADDR(-1ULL >> 1) | QI_DEV_EIOTLB_SIZE
;
180 } else if (pages
> 1) {
181 /* The least significant zero bit indicates the size. So,
182 * for example, an "address" value of 0x12345f000 will
183 * flush from 0x123440000 to 0x12347ffff (256KiB). */
184 unsigned long last
= address
+ ((unsigned long)(pages
- 1) << VTD_PAGE_SHIFT
);
185 unsigned long mask
= __rounddown_pow_of_two(address
^ last
);;
187 desc
.high
= QI_DEV_EIOTLB_ADDR((address
& ~mask
) | (mask
- 1)) | QI_DEV_EIOTLB_SIZE
;
189 desc
.high
= QI_DEV_EIOTLB_ADDR(address
);
191 qi_submit_sync(&desc
, svm
->iommu
);
195 static void intel_flush_svm_range(struct intel_svm
*svm
, unsigned long address
,
196 unsigned long pages
, int ih
, int gl
)
198 struct intel_svm_dev
*sdev
;
200 /* Try deferred invalidate if available */
201 if (svm
->iommu
->pasid_state_table
&&
202 !cmpxchg64(&svm
->iommu
->pasid_state_table
[svm
->pasid
].val
, 0, 1ULL << 63))
206 list_for_each_entry_rcu(sdev
, &svm
->devs
, list
)
207 intel_flush_svm_range_dev(svm
, sdev
, address
, pages
, ih
, gl
);
211 static void intel_change_pte(struct mmu_notifier
*mn
, struct mm_struct
*mm
,
212 unsigned long address
, pte_t pte
)
214 struct intel_svm
*svm
= container_of(mn
, struct intel_svm
, notifier
);
216 intel_flush_svm_range(svm
, address
, 1, 1, 0);
219 static void intel_invalidate_page(struct mmu_notifier
*mn
, struct mm_struct
*mm
,
220 unsigned long address
)
222 struct intel_svm
*svm
= container_of(mn
, struct intel_svm
, notifier
);
224 intel_flush_svm_range(svm
, address
, 1, 1, 0);
227 /* Pages have been freed at this point */
228 static void intel_invalidate_range(struct mmu_notifier
*mn
,
229 struct mm_struct
*mm
,
230 unsigned long start
, unsigned long end
)
232 struct intel_svm
*svm
= container_of(mn
, struct intel_svm
, notifier
);
234 intel_flush_svm_range(svm
, start
,
235 (end
- start
+ PAGE_SIZE
- 1) >> VTD_PAGE_SHIFT
, 0, 0);
239 static void intel_flush_pasid_dev(struct intel_svm
*svm
, struct intel_svm_dev
*sdev
)
244 desc
.low
= QI_PC_TYPE
| QI_PC_DID(sdev
->did
) | QI_PC_PASID_SEL
| QI_PC_PASID(svm
->pasid
);
246 qi_submit_sync(&desc
, svm
->iommu
);
249 static void intel_mm_release(struct mmu_notifier
*mn
, struct mm_struct
*mm
)
251 struct intel_svm
*svm
= container_of(mn
, struct intel_svm
, notifier
);
253 svm
->iommu
->pasid_table
[svm
->pasid
].val
= 0;
255 /* There's no need to do any flush because we can't get here if there
256 * are any devices left anyway. */
257 WARN_ON(!list_empty(&svm
->devs
));
260 static const struct mmu_notifier_ops intel_mmuops
= {
261 .release
= intel_mm_release
,
262 .change_pte
= intel_change_pte
,
263 .invalidate_page
= intel_invalidate_page
,
264 .invalidate_range
= intel_invalidate_range
,
267 static DEFINE_MUTEX(pasid_mutex
);
269 int intel_svm_bind_mm(struct device
*dev
, int *pasid
, int flags
, struct svm_dev_ops
*ops
)
271 struct intel_iommu
*iommu
= intel_svm_device_to_iommu(dev
);
272 struct intel_svm_dev
*sdev
;
273 struct intel_svm
*svm
= NULL
;
274 struct mm_struct
*mm
= NULL
;
281 if (dev_is_pci(dev
)) {
282 pasid_max
= pci_max_pasids(to_pci_dev(dev
));
288 if ((flags
& SVM_FLAG_SUPERVISOR_MODE
)) {
289 if (!ecap_srs(iommu
->ecap
))
292 mm
= get_task_mm(current
);
296 mutex_lock(&pasid_mutex
);
297 if (pasid
&& !(flags
& SVM_FLAG_PRIVATE_PASID
)) {
300 idr_for_each_entry(&iommu
->pasid_idr
, svm
, i
) {
302 (svm
->flags
& SVM_FLAG_PRIVATE_PASID
))
305 if (svm
->pasid
>= pasid_max
) {
307 "Limited PASID width. Cannot use existing PASID %d\n",
313 list_for_each_entry(sdev
, &svm
->devs
, list
) {
314 if (dev
== sdev
->dev
) {
315 if (sdev
->ops
!= ops
) {
328 sdev
= kzalloc(sizeof(*sdev
), GFP_KERNEL
);
335 ret
= intel_iommu_enable_pasid(iommu
, sdev
);
337 /* If they don't actually want to assign a PASID, this is
338 * just an enabling check/preparation. */
342 /* Finish the setup now we know we're keeping it */
345 init_rcu_head(&sdev
->rcu
);
348 svm
= kzalloc(sizeof(*svm
), GFP_KERNEL
);
356 if (pasid_max
> 2 << ecap_pss(iommu
->ecap
))
357 pasid_max
= 2 << ecap_pss(iommu
->ecap
);
359 ret
= idr_alloc(&iommu
->pasid_idr
, svm
, 0, pasid_max
- 1,
366 svm
->notifier
.ops
= &intel_mmuops
;
369 INIT_LIST_HEAD_RCU(&svm
->devs
);
372 ret
= mmu_notifier_register(&svm
->notifier
, mm
);
374 idr_remove(&svm
->iommu
->pasid_idr
, svm
->pasid
);
379 iommu
->pasid_table
[svm
->pasid
].val
= (u64
)__pa(mm
->pgd
) | 1;
382 iommu
->pasid_table
[svm
->pasid
].val
= (u64
)__pa(init_mm
.pgd
) | 1 | (1ULL << 11);
385 list_add_rcu(&sdev
->list
, &svm
->devs
);
391 mutex_unlock(&pasid_mutex
);
396 EXPORT_SYMBOL_GPL(intel_svm_bind_mm
);
398 int intel_svm_unbind_mm(struct device
*dev
, int pasid
)
400 struct intel_svm_dev
*sdev
;
401 struct intel_iommu
*iommu
;
402 struct intel_svm
*svm
;
405 mutex_lock(&pasid_mutex
);
406 iommu
= intel_svm_device_to_iommu(dev
);
407 if (!iommu
|| !iommu
->pasid_table
)
410 svm
= idr_find(&iommu
->pasid_idr
, pasid
);
414 list_for_each_entry(sdev
, &svm
->devs
, list
) {
415 if (dev
== sdev
->dev
) {
419 list_del_rcu(&sdev
->list
);
420 /* Flush the PASID cache and IOTLB for this device.
421 * Note that we do depend on the hardware *not* using
422 * the PASID any more. Just as we depend on other
423 * devices never using PASIDs that they have no right
424 * to use. We have a *shared* PASID table, because it's
425 * large and has to be physically contiguous. So it's
426 * hard to be as defensive as we might like. */
427 intel_flush_pasid_dev(svm
, sdev
);
428 intel_flush_svm_range_dev(svm
, sdev
, 0, -1, 0, !svm
->mm
);
429 kfree_rcu(sdev
, rcu
);
431 if (list_empty(&svm
->devs
)) {
432 mmu_notifier_unregister(&svm
->notifier
, svm
->mm
);
434 idr_remove(&svm
->iommu
->pasid_idr
, svm
->pasid
);
437 /* We mandate that no page faults may be outstanding
438 * for the PASID when intel_svm_unbind_mm() is called.
439 * If that is not obeyed, subtle errors will happen.
440 * Let's make them less subtle... */
441 memset(svm
, 0x6b, sizeof(*svm
));
449 mutex_unlock(&pasid_mutex
);
453 EXPORT_SYMBOL_GPL(intel_svm_unbind_mm
);
455 /* Page request queue descriptor */
456 struct page_req_dsc
{
473 #define PRQ_RING_MASK ((0x1000 << PRQ_ORDER) - 0x10)
474 static irqreturn_t
prq_event_thread(int irq
, void *d
)
476 struct intel_iommu
*iommu
= d
;
477 struct intel_svm
*svm
= NULL
;
478 int head
, tail
, handled
= 0;
480 tail
= dmar_readq(iommu
->reg
+ DMAR_PQT_REG
) & PRQ_RING_MASK
;
481 head
= dmar_readq(iommu
->reg
+ DMAR_PQH_REG
) & PRQ_RING_MASK
;
482 while (head
!= tail
) {
483 struct intel_svm_dev
*sdev
;
484 struct vm_area_struct
*vma
;
485 struct page_req_dsc
*req
;
492 req
= &iommu
->prq
[head
/ sizeof(*req
)];
494 result
= QI_RESP_FAILURE
;
495 address
= (u64
)req
->addr
<< VTD_PAGE_SHIFT
;
496 if (!req
->pasid_present
) {
497 pr_err("%s: Page request without PASID: %08llx %08llx\n",
498 iommu
->name
, ((unsigned long long *)req
)[0],
499 ((unsigned long long *)req
)[1]);
503 if (!svm
|| svm
->pasid
!= req
->pasid
) {
505 svm
= idr_find(&iommu
->pasid_idr
, req
->pasid
);
506 /* It *can't* go away, because the driver is not permitted
507 * to unbind the mm while any page faults are outstanding.
508 * So we only need RCU to protect the internal idr code. */
512 pr_err("%s: Page request for invalid PASID %d: %08llx %08llx\n",
513 iommu
->name
, req
->pasid
, ((unsigned long long *)req
)[0],
514 ((unsigned long long *)req
)[1]);
519 result
= QI_RESP_INVALID
;
520 /* Since we're using init_mm.pgd directly, we should never take
521 * any faults on kernel addresses. */
524 down_read(&svm
->mm
->mmap_sem
);
525 vma
= find_extend_vma(svm
->mm
, address
);
526 if (!vma
|| address
< vma
->vm_start
)
529 ret
= handle_mm_fault(svm
->mm
, vma
, address
,
530 req
->wr_req
? FAULT_FLAG_WRITE
: 0);
531 if (ret
& VM_FAULT_ERROR
)
534 result
= QI_RESP_SUCCESS
;
536 up_read(&svm
->mm
->mmap_sem
);
538 /* Accounting for major/minor faults? */
540 list_for_each_entry_rcu(sdev
, &svm
->devs
, list
) {
541 if (sdev
->sid
== PCI_DEVID(req
->bus
, req
->devfn
))
544 /* Other devices can go away, but the drivers are not permitted
545 * to unbind while any page faults might be in flight. So it's
546 * OK to drop the 'lock' here now we have it. */
549 if (WARN_ON(&sdev
->list
== &svm
->devs
))
552 if (sdev
&& sdev
->ops
&& sdev
->ops
->fault_cb
) {
553 int rwxp
= (req
->rd_req
<< 3) | (req
->wr_req
<< 2) |
554 (req
->wr_req
<< 1) | (req
->exe_req
);
555 sdev
->ops
->fault_cb(sdev
->dev
, req
->pasid
, req
->addr
, req
->private, rwxp
, result
);
557 /* We get here in the error case where the PASID lookup failed,
558 and these can be NULL. Do not use them below this point! */
563 /* Page Group Response */
564 resp
.low
= QI_PGRP_PASID(req
->pasid
) |
565 QI_PGRP_DID((req
->bus
<< 8) | req
->devfn
) |
566 QI_PGRP_PASID_P(req
->pasid_present
) |
568 resp
.high
= QI_PGRP_IDX(req
->prg_index
) |
569 QI_PGRP_PRIV(req
->private) | QI_PGRP_RESP_CODE(result
);
571 qi_submit_sync(&resp
, iommu
);
572 } else if (req
->srr
) {
573 /* Page Stream Response */
574 resp
.low
= QI_PSTRM_IDX(req
->prg_index
) |
575 QI_PSTRM_PRIV(req
->private) | QI_PSTRM_BUS(req
->bus
) |
576 QI_PSTRM_PASID(req
->pasid
) | QI_PSTRM_RESP_TYPE
;
577 resp
.high
= QI_PSTRM_ADDR(address
) | QI_PSTRM_DEVFN(req
->devfn
) |
578 QI_PSTRM_RESP_CODE(result
);
580 qi_submit_sync(&resp
, iommu
);
583 head
= (head
+ sizeof(*req
)) & PRQ_RING_MASK
;
586 dmar_writeq(iommu
->reg
+ DMAR_PQH_REG
, tail
);
588 return IRQ_RETVAL(handled
);