2 * IOMMU API for s390 PCI devices
4 * Copyright IBM Corp. 2015
5 * Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com>
9 #include <linux/iommu.h>
10 #include <linux/iommu-helper.h>
11 #include <linux/sizes.h>
12 #include <asm/pci_dma.h>
15 * Physically contiguous memory regions can be mapped with 4 KiB alignment,
16 * we allow all page sizes that are an order of 4KiB (no special large page
19 #define S390_IOMMU_PGSIZES (~0xFFFUL)
22 struct iommu_domain domain
;
23 struct list_head devices
;
24 unsigned long *dma_table
;
25 spinlock_t dma_table_lock
;
29 struct s390_domain_device
{
30 struct list_head list
;
31 struct zpci_dev
*zdev
;
34 static struct s390_domain
*to_s390_domain(struct iommu_domain
*dom
)
36 return container_of(dom
, struct s390_domain
, domain
);
39 static bool s390_iommu_capable(enum iommu_cap cap
)
42 case IOMMU_CAP_CACHE_COHERENCY
:
44 case IOMMU_CAP_INTR_REMAP
:
51 static struct iommu_domain
*s390_domain_alloc(unsigned domain_type
)
53 struct s390_domain
*s390_domain
;
55 if (domain_type
!= IOMMU_DOMAIN_UNMANAGED
)
58 s390_domain
= kzalloc(sizeof(*s390_domain
), GFP_KERNEL
);
62 s390_domain
->dma_table
= dma_alloc_cpu_table();
63 if (!s390_domain
->dma_table
) {
68 spin_lock_init(&s390_domain
->dma_table_lock
);
69 spin_lock_init(&s390_domain
->list_lock
);
70 INIT_LIST_HEAD(&s390_domain
->devices
);
72 return &s390_domain
->domain
;
75 static void s390_domain_free(struct iommu_domain
*domain
)
77 struct s390_domain
*s390_domain
= to_s390_domain(domain
);
79 dma_cleanup_tables(s390_domain
->dma_table
);
83 static int s390_iommu_attach_device(struct iommu_domain
*domain
,
86 struct s390_domain
*s390_domain
= to_s390_domain(domain
);
87 struct zpci_dev
*zdev
= to_pci_dev(dev
)->sysdata
;
88 struct s390_domain_device
*domain_device
;
95 domain_device
= kzalloc(sizeof(*domain_device
), GFP_KERNEL
);
100 zpci_dma_exit_device(zdev
);
102 zdev
->dma_table
= s390_domain
->dma_table
;
103 rc
= zpci_register_ioat(zdev
, 0, zdev
->start_dma
, zdev
->end_dma
,
104 (u64
) zdev
->dma_table
);
108 spin_lock_irqsave(&s390_domain
->list_lock
, flags
);
109 /* First device defines the DMA range limits */
110 if (list_empty(&s390_domain
->devices
)) {
111 domain
->geometry
.aperture_start
= zdev
->start_dma
;
112 domain
->geometry
.aperture_end
= zdev
->end_dma
;
113 domain
->geometry
.force_aperture
= true;
114 /* Allow only devices with identical DMA range limits */
115 } else if (domain
->geometry
.aperture_start
!= zdev
->start_dma
||
116 domain
->geometry
.aperture_end
!= zdev
->end_dma
) {
118 spin_unlock_irqrestore(&s390_domain
->list_lock
, flags
);
121 domain_device
->zdev
= zdev
;
122 zdev
->s390_domain
= s390_domain
;
123 list_add(&domain_device
->list
, &s390_domain
->devices
);
124 spin_unlock_irqrestore(&s390_domain
->list_lock
, flags
);
129 zpci_dma_init_device(zdev
);
130 kfree(domain_device
);
135 static void s390_iommu_detach_device(struct iommu_domain
*domain
,
138 struct s390_domain
*s390_domain
= to_s390_domain(domain
);
139 struct zpci_dev
*zdev
= to_pci_dev(dev
)->sysdata
;
140 struct s390_domain_device
*domain_device
, *tmp
;
147 spin_lock_irqsave(&s390_domain
->list_lock
, flags
);
148 list_for_each_entry_safe(domain_device
, tmp
, &s390_domain
->devices
,
150 if (domain_device
->zdev
== zdev
) {
151 list_del(&domain_device
->list
);
152 kfree(domain_device
);
157 spin_unlock_irqrestore(&s390_domain
->list_lock
, flags
);
160 zdev
->s390_domain
= NULL
;
161 zpci_unregister_ioat(zdev
, 0);
162 zpci_dma_init_device(zdev
);
166 static int s390_iommu_add_device(struct device
*dev
)
168 struct iommu_group
*group
= iommu_group_get_for_dev(dev
);
171 return PTR_ERR(group
);
173 iommu_group_put(group
);
178 static void s390_iommu_remove_device(struct device
*dev
)
180 struct zpci_dev
*zdev
= to_pci_dev(dev
)->sysdata
;
181 struct iommu_domain
*domain
;
184 * This is a workaround for a scenario where the IOMMU API common code
185 * "forgets" to call the detach_dev callback: After binding a device
186 * to vfio-pci and completing the VFIO_SET_IOMMU ioctl (which triggers
187 * the attach_dev), removing the device via
188 * "echo 1 > /sys/bus/pci/devices/.../remove" won't trigger detach_dev,
189 * only remove_device will be called via the BUS_NOTIFY_REMOVED_DEVICE
192 * So let's call detach_dev from here if it hasn't been called before.
194 if (zdev
&& zdev
->s390_domain
) {
195 domain
= iommu_get_domain_for_dev(dev
);
197 s390_iommu_detach_device(domain
, dev
);
200 iommu_group_remove_device(dev
);
203 static int s390_iommu_update_trans(struct s390_domain
*s390_domain
,
204 unsigned long pa
, dma_addr_t dma_addr
,
205 size_t size
, int flags
)
207 struct s390_domain_device
*domain_device
;
208 u8
*page_addr
= (u8
*) (pa
& PAGE_MASK
);
209 dma_addr_t start_dma_addr
= dma_addr
;
210 unsigned long irq_flags
, nr_pages
, i
;
211 unsigned long *entry
;
214 if (dma_addr
< s390_domain
->domain
.geometry
.aperture_start
||
215 dma_addr
+ size
> s390_domain
->domain
.geometry
.aperture_end
)
218 nr_pages
= PAGE_ALIGN(size
) >> PAGE_SHIFT
;
222 spin_lock_irqsave(&s390_domain
->dma_table_lock
, irq_flags
);
223 for (i
= 0; i
< nr_pages
; i
++) {
224 entry
= dma_walk_cpu_trans(s390_domain
->dma_table
, dma_addr
);
229 dma_update_cpu_trans(entry
, page_addr
, flags
);
230 page_addr
+= PAGE_SIZE
;
231 dma_addr
+= PAGE_SIZE
;
234 spin_lock(&s390_domain
->list_lock
);
235 list_for_each_entry(domain_device
, &s390_domain
->devices
, list
) {
236 rc
= zpci_refresh_trans((u64
) domain_device
->zdev
->fh
<< 32,
237 start_dma_addr
, nr_pages
* PAGE_SIZE
);
241 spin_unlock(&s390_domain
->list_lock
);
244 if (rc
&& ((flags
& ZPCI_PTE_VALID_MASK
) == ZPCI_PTE_VALID
)) {
245 flags
= ZPCI_PTE_INVALID
;
247 page_addr
-= PAGE_SIZE
;
248 dma_addr
-= PAGE_SIZE
;
249 entry
= dma_walk_cpu_trans(s390_domain
->dma_table
,
253 dma_update_cpu_trans(entry
, page_addr
, flags
);
256 spin_unlock_irqrestore(&s390_domain
->dma_table_lock
, irq_flags
);
261 static int s390_iommu_map(struct iommu_domain
*domain
, unsigned long iova
,
262 phys_addr_t paddr
, size_t size
, int prot
)
264 struct s390_domain
*s390_domain
= to_s390_domain(domain
);
265 int flags
= ZPCI_PTE_VALID
, rc
= 0;
267 if (!(prot
& IOMMU_READ
))
270 if (!(prot
& IOMMU_WRITE
))
271 flags
|= ZPCI_TABLE_PROTECTED
;
273 rc
= s390_iommu_update_trans(s390_domain
, (unsigned long) paddr
, iova
,
279 static phys_addr_t
s390_iommu_iova_to_phys(struct iommu_domain
*domain
,
282 struct s390_domain
*s390_domain
= to_s390_domain(domain
);
283 unsigned long *sto
, *pto
, *rto
, flags
;
284 unsigned int rtx
, sx
, px
;
285 phys_addr_t phys
= 0;
287 if (iova
< domain
->geometry
.aperture_start
||
288 iova
> domain
->geometry
.aperture_end
)
291 rtx
= calc_rtx(iova
);
294 rto
= s390_domain
->dma_table
;
296 spin_lock_irqsave(&s390_domain
->dma_table_lock
, flags
);
297 if (rto
&& reg_entry_isvalid(rto
[rtx
])) {
298 sto
= get_rt_sto(rto
[rtx
]);
299 if (sto
&& reg_entry_isvalid(sto
[sx
])) {
300 pto
= get_st_pto(sto
[sx
]);
301 if (pto
&& pt_entry_isvalid(pto
[px
]))
302 phys
= pto
[px
] & ZPCI_PTE_ADDR_MASK
;
305 spin_unlock_irqrestore(&s390_domain
->dma_table_lock
, flags
);
310 static size_t s390_iommu_unmap(struct iommu_domain
*domain
,
311 unsigned long iova
, size_t size
)
313 struct s390_domain
*s390_domain
= to_s390_domain(domain
);
314 int flags
= ZPCI_PTE_INVALID
;
318 paddr
= s390_iommu_iova_to_phys(domain
, iova
);
322 rc
= s390_iommu_update_trans(s390_domain
, (unsigned long) paddr
, iova
,
330 static struct iommu_ops s390_iommu_ops
= {
331 .capable
= s390_iommu_capable
,
332 .domain_alloc
= s390_domain_alloc
,
333 .domain_free
= s390_domain_free
,
334 .attach_dev
= s390_iommu_attach_device
,
335 .detach_dev
= s390_iommu_detach_device
,
336 .map
= s390_iommu_map
,
337 .unmap
= s390_iommu_unmap
,
338 .iova_to_phys
= s390_iommu_iova_to_phys
,
339 .add_device
= s390_iommu_add_device
,
340 .remove_device
= s390_iommu_remove_device
,
341 .device_group
= generic_device_group
,
342 .pgsize_bitmap
= S390_IOMMU_PGSIZES
,
345 static int __init
s390_iommu_init(void)
347 return bus_set_iommu(&pci_bus_type
, &s390_iommu_ops
);
349 subsys_initcall(s390_iommu_init
);