2 * QEMU sPAPR IOMMU (TCE) code
4 * Copyright (c) 2010 David Gibson, IBM Corporation <dwg@au1.ibm.com>
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
21 #include "qemu/error-report.h"
23 #include "qemu/module.h"
24 #include "sysemu/kvm.h"
27 #include "migration/vmstate.h"
28 #include "sysemu/dma.h"
29 #include "exec/address-spaces.h"
32 #include "hw/ppc/spapr.h"
33 #include "hw/ppc/spapr_vio.h"
44 #define IOMMU_PAGE_SIZE(shift) (1ULL << (shift))
45 #define IOMMU_PAGE_MASK(shift) (~(IOMMU_PAGE_SIZE(shift) - 1))
47 static QLIST_HEAD(, SpaprTceTable
) spapr_tce_tables
;
49 SpaprTceTable
*spapr_tce_find_by_liobn(target_ulong liobn
)
53 if (liobn
& 0xFFFFFFFF00000000ULL
) {
54 hcall_dprintf("Request for out-of-bounds LIOBN 0x" TARGET_FMT_lx
"\n",
59 QLIST_FOREACH(tcet
, &spapr_tce_tables
, list
) {
60 if (tcet
->liobn
== (uint32_t)liobn
) {
68 static IOMMUAccessFlags
spapr_tce_iommu_access_flags(uint64_t tce
)
70 switch (tce
& SPAPR_TCE_RW
) {
77 default: /* SPAPR_TCE_RW */
82 static uint64_t *spapr_tce_alloc_table(uint32_t liobn
,
89 uint64_t *table
= NULL
;
92 table
= kvmppc_create_spapr_tce(liobn
, page_shift
, bus_offset
, nb_table
,
98 table
= g_new0(uint64_t, nb_table
);
101 trace_spapr_iommu_new_table(liobn
, table
, *fd
);
106 static void spapr_tce_free_table(uint64_t *table
, int fd
, uint32_t nb_table
)
108 if (!kvm_enabled() ||
109 (kvmppc_remove_spapr_tce(table
, fd
, nb_table
) != 0)) {
114 /* Called from RCU critical section */
115 static IOMMUTLBEntry
spapr_tce_translate_iommu(IOMMUMemoryRegion
*iommu
,
117 IOMMUAccessFlags flag
,
120 SpaprTceTable
*tcet
= container_of(iommu
, SpaprTceTable
, iommu
);
122 IOMMUTLBEntry ret
= {
123 .target_as
= &address_space_memory
,
125 .translated_addr
= 0,
126 .addr_mask
= ~(hwaddr
)0,
130 if ((addr
>> tcet
->page_shift
) < tcet
->nb_table
) {
131 /* Check if we are in bound */
132 hwaddr page_mask
= IOMMU_PAGE_MASK(tcet
->page_shift
);
134 tce
= tcet
->table
[addr
>> tcet
->page_shift
];
135 ret
.iova
= addr
& page_mask
;
136 ret
.translated_addr
= tce
& page_mask
;
137 ret
.addr_mask
= ~page_mask
;
138 ret
.perm
= spapr_tce_iommu_access_flags(tce
);
140 trace_spapr_iommu_xlate(tcet
->liobn
, addr
, ret
.iova
, ret
.perm
,
146 static void spapr_tce_replay(IOMMUMemoryRegion
*iommu_mr
, IOMMUNotifier
*n
)
148 MemoryRegion
*mr
= MEMORY_REGION(iommu_mr
);
149 IOMMUMemoryRegionClass
*imrc
= IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr
);
150 hwaddr addr
, granularity
;
152 SpaprTceTable
*tcet
= container_of(iommu_mr
, SpaprTceTable
, iommu
);
154 if (tcet
->skipping_replay
) {
158 granularity
= memory_region_iommu_get_min_page_size(iommu_mr
);
160 for (addr
= 0; addr
< memory_region_size(mr
); addr
+= granularity
) {
161 iotlb
= imrc
->translate(iommu_mr
, addr
, IOMMU_NONE
, n
->iommu_idx
);
162 if (iotlb
.perm
!= IOMMU_NONE
) {
163 n
->notify(n
, &iotlb
);
167 * if (2^64 - MR size) < granularity, it's possible to get an
168 * infinite loop here. This should catch such a wraparound.
170 if ((addr
+ granularity
) < addr
) {
176 static int spapr_tce_table_pre_save(void *opaque
)
178 SpaprTceTable
*tcet
= SPAPR_TCE_TABLE(opaque
);
180 tcet
->mig_table
= tcet
->table
;
181 tcet
->mig_nb_table
= tcet
->nb_table
;
183 trace_spapr_iommu_pre_save(tcet
->liobn
, tcet
->mig_nb_table
,
184 tcet
->bus_offset
, tcet
->page_shift
);
189 static uint64_t spapr_tce_get_min_page_size(IOMMUMemoryRegion
*iommu
)
191 SpaprTceTable
*tcet
= container_of(iommu
, SpaprTceTable
, iommu
);
193 return 1ULL << tcet
->page_shift
;
196 static int spapr_tce_get_attr(IOMMUMemoryRegion
*iommu
,
197 enum IOMMUMemoryRegionAttr attr
, void *data
)
199 SpaprTceTable
*tcet
= container_of(iommu
, SpaprTceTable
, iommu
);
201 if (attr
== IOMMU_ATTR_SPAPR_TCE_FD
&& kvmppc_has_cap_spapr_vfio()) {
202 *(int *) data
= tcet
->fd
;
209 static void spapr_tce_notify_flag_changed(IOMMUMemoryRegion
*iommu
,
210 IOMMUNotifierFlag old
,
211 IOMMUNotifierFlag
new)
213 struct SpaprTceTable
*tbl
= container_of(iommu
, SpaprTceTable
, iommu
);
215 if (old
== IOMMU_NOTIFIER_NONE
&& new != IOMMU_NOTIFIER_NONE
) {
216 spapr_tce_set_need_vfio(tbl
, true);
217 } else if (old
!= IOMMU_NOTIFIER_NONE
&& new == IOMMU_NOTIFIER_NONE
) {
218 spapr_tce_set_need_vfio(tbl
, false);
222 static int spapr_tce_table_post_load(void *opaque
, int version_id
)
224 SpaprTceTable
*tcet
= SPAPR_TCE_TABLE(opaque
);
225 uint32_t old_nb_table
= tcet
->nb_table
;
226 uint64_t old_bus_offset
= tcet
->bus_offset
;
227 uint32_t old_page_shift
= tcet
->page_shift
;
230 spapr_vio_set_bypass(tcet
->vdev
, tcet
->bypass
);
233 if (tcet
->mig_nb_table
!= tcet
->nb_table
) {
234 spapr_tce_table_disable(tcet
);
237 if (tcet
->mig_nb_table
) {
238 if (!tcet
->nb_table
) {
239 spapr_tce_table_enable(tcet
, old_page_shift
, old_bus_offset
,
243 memcpy(tcet
->table
, tcet
->mig_table
,
244 tcet
->nb_table
* sizeof(tcet
->table
[0]));
246 free(tcet
->mig_table
);
247 tcet
->mig_table
= NULL
;
250 trace_spapr_iommu_post_load(tcet
->liobn
, old_nb_table
, tcet
->nb_table
,
251 tcet
->bus_offset
, tcet
->page_shift
);
256 static bool spapr_tce_table_ex_needed(void *opaque
)
258 SpaprTceTable
*tcet
= opaque
;
260 return tcet
->bus_offset
|| tcet
->page_shift
!= 0xC;
263 static const VMStateDescription vmstate_spapr_tce_table_ex
= {
264 .name
= "spapr_iommu_ex",
266 .minimum_version_id
= 1,
267 .needed
= spapr_tce_table_ex_needed
,
268 .fields
= (VMStateField
[]) {
269 VMSTATE_UINT64(bus_offset
, SpaprTceTable
),
270 VMSTATE_UINT32(page_shift
, SpaprTceTable
),
271 VMSTATE_END_OF_LIST()
275 static const VMStateDescription vmstate_spapr_tce_table
= {
276 .name
= "spapr_iommu",
278 .minimum_version_id
= 2,
279 .pre_save
= spapr_tce_table_pre_save
,
280 .post_load
= spapr_tce_table_post_load
,
281 .fields
= (VMStateField
[]) {
283 VMSTATE_UINT32_EQUAL(liobn
, SpaprTceTable
, NULL
),
286 VMSTATE_UINT32(mig_nb_table
, SpaprTceTable
),
287 VMSTATE_BOOL(bypass
, SpaprTceTable
),
288 VMSTATE_VARRAY_UINT32_ALLOC(mig_table
, SpaprTceTable
, mig_nb_table
, 0,
289 vmstate_info_uint64
, uint64_t),
291 VMSTATE_END_OF_LIST()
293 .subsections
= (const VMStateDescription
*[]) {
294 &vmstate_spapr_tce_table_ex
,
299 static void spapr_tce_table_realize(DeviceState
*dev
, Error
**errp
)
301 SpaprTceTable
*tcet
= SPAPR_TCE_TABLE(dev
);
302 Object
*tcetobj
= OBJECT(tcet
);
306 tcet
->need_vfio
= false;
307 tmp
= g_strdup_printf("tce-root-%x", tcet
->liobn
);
308 memory_region_init(&tcet
->root
, tcetobj
, tmp
, UINT64_MAX
);
311 tmp
= g_strdup_printf("tce-iommu-%x", tcet
->liobn
);
312 memory_region_init_iommu(&tcet
->iommu
, sizeof(tcet
->iommu
),
313 TYPE_SPAPR_IOMMU_MEMORY_REGION
,
317 QLIST_INSERT_HEAD(&spapr_tce_tables
, tcet
, list
);
319 vmstate_register(DEVICE(tcet
), tcet
->liobn
, &vmstate_spapr_tce_table
,
323 void spapr_tce_set_need_vfio(SpaprTceTable
*tcet
, bool need_vfio
)
325 size_t table_size
= tcet
->nb_table
* sizeof(uint64_t);
329 g_assert(need_vfio
!= tcet
->need_vfio
);
331 tcet
->need_vfio
= need_vfio
;
333 if (!need_vfio
|| (tcet
->fd
!= -1 && kvmppc_has_cap_spapr_vfio())) {
337 oldtable
= tcet
->table
;
339 tcet
->table
= spapr_tce_alloc_table(tcet
->liobn
,
345 memcpy(tcet
->table
, oldtable
, table_size
);
347 spapr_tce_free_table(oldtable
, tcet
->fd
, tcet
->nb_table
);
352 SpaprTceTable
*spapr_tce_new_table(DeviceState
*owner
, uint32_t liobn
)
357 if (spapr_tce_find_by_liobn(liobn
)) {
358 error_report("Attempted to create TCE table with duplicate"
359 " LIOBN 0x%x", liobn
);
363 tcet
= SPAPR_TCE_TABLE(object_new(TYPE_SPAPR_TCE_TABLE
));
366 tmp
= g_strdup_printf("tce-table-%x", liobn
);
367 object_property_add_child(OBJECT(owner
), tmp
, OBJECT(tcet
), NULL
);
369 object_unref(OBJECT(tcet
));
371 object_property_set_bool(OBJECT(tcet
), true, "realized", NULL
);
376 void spapr_tce_table_enable(SpaprTceTable
*tcet
,
377 uint32_t page_shift
, uint64_t bus_offset
,
380 if (tcet
->nb_table
) {
381 warn_report("trying to enable already enabled TCE table");
385 tcet
->bus_offset
= bus_offset
;
386 tcet
->page_shift
= page_shift
;
387 tcet
->nb_table
= nb_table
;
388 tcet
->table
= spapr_tce_alloc_table(tcet
->liobn
,
395 memory_region_set_size(MEMORY_REGION(&tcet
->iommu
),
396 (uint64_t)tcet
->nb_table
<< tcet
->page_shift
);
397 memory_region_add_subregion(&tcet
->root
, tcet
->bus_offset
,
398 MEMORY_REGION(&tcet
->iommu
));
401 void spapr_tce_table_disable(SpaprTceTable
*tcet
)
403 if (!tcet
->nb_table
) {
407 memory_region_del_subregion(&tcet
->root
, MEMORY_REGION(&tcet
->iommu
));
408 memory_region_set_size(MEMORY_REGION(&tcet
->iommu
), 0);
410 spapr_tce_free_table(tcet
->table
, tcet
->fd
, tcet
->nb_table
);
413 tcet
->bus_offset
= 0;
414 tcet
->page_shift
= 0;
418 static void spapr_tce_table_unrealize(DeviceState
*dev
, Error
**errp
)
420 SpaprTceTable
*tcet
= SPAPR_TCE_TABLE(dev
);
422 vmstate_unregister(DEVICE(tcet
), &vmstate_spapr_tce_table
, tcet
);
424 QLIST_REMOVE(tcet
, list
);
426 spapr_tce_table_disable(tcet
);
429 MemoryRegion
*spapr_tce_get_iommu(SpaprTceTable
*tcet
)
434 static void spapr_tce_reset(DeviceState
*dev
)
436 SpaprTceTable
*tcet
= SPAPR_TCE_TABLE(dev
);
437 size_t table_size
= tcet
->nb_table
* sizeof(uint64_t);
439 if (tcet
->nb_table
) {
440 memset(tcet
->table
, 0, table_size
);
444 static target_ulong
put_tce_emu(SpaprTceTable
*tcet
, target_ulong ioba
,
448 hwaddr page_mask
= IOMMU_PAGE_MASK(tcet
->page_shift
);
449 unsigned long index
= (ioba
- tcet
->bus_offset
) >> tcet
->page_shift
;
451 if (index
>= tcet
->nb_table
) {
452 hcall_dprintf("spapr_vio_put_tce on out-of-bounds IOBA 0x"
453 TARGET_FMT_lx
"\n", ioba
);
457 tcet
->table
[index
] = tce
;
459 entry
.target_as
= &address_space_memory
,
460 entry
.iova
= (ioba
- tcet
->bus_offset
) & page_mask
;
461 entry
.translated_addr
= tce
& page_mask
;
462 entry
.addr_mask
= ~page_mask
;
463 entry
.perm
= spapr_tce_iommu_access_flags(tce
);
464 memory_region_notify_iommu(&tcet
->iommu
, 0, entry
);
469 static target_ulong
h_put_tce_indirect(PowerPCCPU
*cpu
,
470 SpaprMachineState
*spapr
,
471 target_ulong opcode
, target_ulong
*args
)
474 target_ulong liobn
= args
[0];
475 target_ulong ioba
= args
[1];
476 target_ulong ioba1
= ioba
;
477 target_ulong tce_list
= args
[2];
478 target_ulong npages
= args
[3];
479 target_ulong ret
= H_PARAMETER
, tce
= 0;
480 SpaprTceTable
*tcet
= spapr_tce_find_by_liobn(liobn
);
481 CPUState
*cs
= CPU(cpu
);
482 hwaddr page_mask
, page_size
;
488 if ((npages
> 512) || (tce_list
& SPAPR_TCE_PAGE_MASK
)) {
492 page_mask
= IOMMU_PAGE_MASK(tcet
->page_shift
);
493 page_size
= IOMMU_PAGE_SIZE(tcet
->page_shift
);
496 for (i
= 0; i
< npages
; ++i
, ioba
+= page_size
) {
497 tce
= ldq_be_phys(cs
->as
, tce_list
+ i
* sizeof(target_ulong
));
499 ret
= put_tce_emu(tcet
, ioba
, tce
);
505 /* Trace last successful or the first problematic entry */
507 if (SPAPR_IS_PCI_LIOBN(liobn
)) {
508 trace_spapr_iommu_pci_indirect(liobn
, ioba1
, tce_list
, i
, tce
, ret
);
510 trace_spapr_iommu_indirect(liobn
, ioba1
, tce_list
, i
, tce
, ret
);
515 static target_ulong
h_stuff_tce(PowerPCCPU
*cpu
, SpaprMachineState
*spapr
,
516 target_ulong opcode
, target_ulong
*args
)
519 target_ulong liobn
= args
[0];
520 target_ulong ioba
= args
[1];
521 target_ulong tce_value
= args
[2];
522 target_ulong npages
= args
[3];
523 target_ulong ret
= H_PARAMETER
;
524 SpaprTceTable
*tcet
= spapr_tce_find_by_liobn(liobn
);
525 hwaddr page_mask
, page_size
;
531 if (npages
> tcet
->nb_table
) {
535 page_mask
= IOMMU_PAGE_MASK(tcet
->page_shift
);
536 page_size
= IOMMU_PAGE_SIZE(tcet
->page_shift
);
539 for (i
= 0; i
< npages
; ++i
, ioba
+= page_size
) {
540 ret
= put_tce_emu(tcet
, ioba
, tce_value
);
545 if (SPAPR_IS_PCI_LIOBN(liobn
)) {
546 trace_spapr_iommu_pci_stuff(liobn
, ioba
, tce_value
, npages
, ret
);
548 trace_spapr_iommu_stuff(liobn
, ioba
, tce_value
, npages
, ret
);
554 static target_ulong
h_put_tce(PowerPCCPU
*cpu
, SpaprMachineState
*spapr
,
555 target_ulong opcode
, target_ulong
*args
)
557 target_ulong liobn
= args
[0];
558 target_ulong ioba
= args
[1];
559 target_ulong tce
= args
[2];
560 target_ulong ret
= H_PARAMETER
;
561 SpaprTceTable
*tcet
= spapr_tce_find_by_liobn(liobn
);
564 hwaddr page_mask
= IOMMU_PAGE_MASK(tcet
->page_shift
);
568 ret
= put_tce_emu(tcet
, ioba
, tce
);
570 if (SPAPR_IS_PCI_LIOBN(liobn
)) {
571 trace_spapr_iommu_pci_put(liobn
, ioba
, tce
, ret
);
573 trace_spapr_iommu_put(liobn
, ioba
, tce
, ret
);
579 static target_ulong
get_tce_emu(SpaprTceTable
*tcet
, target_ulong ioba
,
582 unsigned long index
= (ioba
- tcet
->bus_offset
) >> tcet
->page_shift
;
584 if (index
>= tcet
->nb_table
) {
585 hcall_dprintf("spapr_iommu_get_tce on out-of-bounds IOBA 0x"
586 TARGET_FMT_lx
"\n", ioba
);
590 *tce
= tcet
->table
[index
];
595 static target_ulong
h_get_tce(PowerPCCPU
*cpu
, SpaprMachineState
*spapr
,
596 target_ulong opcode
, target_ulong
*args
)
598 target_ulong liobn
= args
[0];
599 target_ulong ioba
= args
[1];
600 target_ulong tce
= 0;
601 target_ulong ret
= H_PARAMETER
;
602 SpaprTceTable
*tcet
= spapr_tce_find_by_liobn(liobn
);
605 hwaddr page_mask
= IOMMU_PAGE_MASK(tcet
->page_shift
);
609 ret
= get_tce_emu(tcet
, ioba
, &tce
);
614 if (SPAPR_IS_PCI_LIOBN(liobn
)) {
615 trace_spapr_iommu_pci_get(liobn
, ioba
, ret
, tce
);
617 trace_spapr_iommu_get(liobn
, ioba
, ret
, tce
);
623 int spapr_dma_dt(void *fdt
, int node_off
, const char *propname
,
624 uint32_t liobn
, uint64_t window
, uint32_t size
)
626 uint32_t dma_prop
[5];
629 dma_prop
[0] = cpu_to_be32(liobn
);
630 dma_prop
[1] = cpu_to_be32(window
>> 32);
631 dma_prop
[2] = cpu_to_be32(window
& 0xFFFFFFFF);
632 dma_prop
[3] = 0; /* window size is 32 bits */
633 dma_prop
[4] = cpu_to_be32(size
);
635 ret
= fdt_setprop_cell(fdt
, node_off
, "ibm,#dma-address-cells", 2);
640 ret
= fdt_setprop_cell(fdt
, node_off
, "ibm,#dma-size-cells", 2);
645 ret
= fdt_setprop(fdt
, node_off
, propname
, dma_prop
, sizeof(dma_prop
));
653 int spapr_tcet_dma_dt(void *fdt
, int node_off
, const char *propname
,
660 return spapr_dma_dt(fdt
, node_off
, propname
,
661 tcet
->liobn
, 0, tcet
->nb_table
<< tcet
->page_shift
);
664 static void spapr_tce_table_class_init(ObjectClass
*klass
, void *data
)
666 DeviceClass
*dc
= DEVICE_CLASS(klass
);
667 dc
->realize
= spapr_tce_table_realize
;
668 dc
->reset
= spapr_tce_reset
;
669 dc
->unrealize
= spapr_tce_table_unrealize
;
670 /* Reason: This is just an internal device for handling the hypercalls */
671 dc
->user_creatable
= false;
673 QLIST_INIT(&spapr_tce_tables
);
676 spapr_register_hypercall(H_PUT_TCE
, h_put_tce
);
677 spapr_register_hypercall(H_GET_TCE
, h_get_tce
);
678 spapr_register_hypercall(H_PUT_TCE_INDIRECT
, h_put_tce_indirect
);
679 spapr_register_hypercall(H_STUFF_TCE
, h_stuff_tce
);
682 static TypeInfo spapr_tce_table_info
= {
683 .name
= TYPE_SPAPR_TCE_TABLE
,
684 .parent
= TYPE_DEVICE
,
685 .instance_size
= sizeof(SpaprTceTable
),
686 .class_init
= spapr_tce_table_class_init
,
689 static void spapr_iommu_memory_region_class_init(ObjectClass
*klass
, void *data
)
691 IOMMUMemoryRegionClass
*imrc
= IOMMU_MEMORY_REGION_CLASS(klass
);
693 imrc
->translate
= spapr_tce_translate_iommu
;
694 imrc
->replay
= spapr_tce_replay
;
695 imrc
->get_min_page_size
= spapr_tce_get_min_page_size
;
696 imrc
->notify_flag_changed
= spapr_tce_notify_flag_changed
;
697 imrc
->get_attr
= spapr_tce_get_attr
;
700 static const TypeInfo spapr_iommu_memory_region_info
= {
701 .parent
= TYPE_IOMMU_MEMORY_REGION
,
702 .name
= TYPE_SPAPR_IOMMU_MEMORY_REGION
,
703 .class_init
= spapr_iommu_memory_region_class_init
,
706 static void register_types(void)
708 type_register_static(&spapr_tce_table_info
);
709 type_register_static(&spapr_iommu_memory_region_info
);
712 type_init(register_types
);