]>
Commit | Line | Data |
---|---|---|
527773ee EA |
1 | /* |
2 | * Copyright (C) 2014-2016 Broadcom Corporation | |
3 | * Copyright (c) 2017 Red Hat, Inc. | |
4 | * Written by Prem Mallappa, Eric Auger | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify | |
7 | * it under the terms of the GNU General Public License version 2 as | |
8 | * published by the Free Software Foundation. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, | |
11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
13 | * GNU General Public License for more details. | |
14 | * | |
15 | * Author: Prem Mallappa <pmallapp@broadcom.com> | |
16 | * | |
17 | */ | |
18 | ||
19 | #include "qemu/osdep.h" | |
20 | #include "sysemu/sysemu.h" | |
21 | #include "exec/address-spaces.h" | |
22 | #include "trace.h" | |
23 | #include "exec/target_page.h" | |
24 | #include "qom/cpu.h" | |
25 | #include "hw/qdev-properties.h" | |
26 | #include "qapi/error.h" | |
cc27ed81 | 27 | #include "qemu/jhash.h" |
0b8fa32f | 28 | #include "qemu/module.h" |
527773ee EA |
29 | |
30 | #include "qemu/error-report.h" | |
31 | #include "hw/arm/smmu-common.h" | |
93641948 EA |
32 | #include "smmu-internal.h" |
33 | ||
cc27ed81 EA |
34 | /* IOTLB Management */ |
35 | ||
36 | inline void smmu_iotlb_inv_all(SMMUState *s) | |
37 | { | |
38 | trace_smmu_iotlb_inv_all(); | |
39 | g_hash_table_remove_all(s->iotlb); | |
40 | } | |
41 | ||
42 | static gboolean smmu_hash_remove_by_asid(gpointer key, gpointer value, | |
43 | gpointer user_data) | |
44 | { | |
45 | uint16_t asid = *(uint16_t *)user_data; | |
46 | SMMUIOTLBKey *iotlb_key = (SMMUIOTLBKey *)key; | |
47 | ||
48 | return iotlb_key->asid == asid; | |
49 | } | |
50 | ||
51 | inline void smmu_iotlb_inv_iova(SMMUState *s, uint16_t asid, dma_addr_t iova) | |
52 | { | |
53 | SMMUIOTLBKey key = {.asid = asid, .iova = iova}; | |
54 | ||
55 | trace_smmu_iotlb_inv_iova(asid, iova); | |
56 | g_hash_table_remove(s->iotlb, &key); | |
57 | } | |
58 | ||
59 | inline void smmu_iotlb_inv_asid(SMMUState *s, uint16_t asid) | |
60 | { | |
61 | trace_smmu_iotlb_inv_asid(asid); | |
62 | g_hash_table_foreach_remove(s->iotlb, smmu_hash_remove_by_asid, &asid); | |
63 | } | |
64 | ||
93641948 EA |
65 | /* VMSAv8-64 Translation */ |
66 | ||
67 | /** | |
68 | * get_pte - Get the content of a page table entry located at | |
69 | * @base_addr[@index] | |
70 | */ | |
71 | static int get_pte(dma_addr_t baseaddr, uint32_t index, uint64_t *pte, | |
72 | SMMUPTWEventInfo *info) | |
73 | { | |
74 | int ret; | |
75 | dma_addr_t addr = baseaddr + index * sizeof(*pte); | |
76 | ||
77 | /* TODO: guarantee 64-bit single-copy atomicity */ | |
78 | ret = dma_memory_read(&address_space_memory, addr, | |
79 | (uint8_t *)pte, sizeof(*pte)); | |
80 | ||
81 | if (ret != MEMTX_OK) { | |
82 | info->type = SMMU_PTW_ERR_WALK_EABT; | |
83 | info->addr = addr; | |
84 | return -EINVAL; | |
85 | } | |
86 | trace_smmu_get_pte(baseaddr, index, addr, *pte); | |
87 | return 0; | |
88 | } | |
89 | ||
90 | /* VMSAv8-64 Translation Table Format Descriptor Decoding */ | |
91 | ||
92 | /** | |
93 | * get_page_pte_address - returns the L3 descriptor output address, | |
94 | * ie. the page frame | |
95 | * ARM ARM spec: Figure D4-17 VMSAv8-64 level 3 descriptor format | |
96 | */ | |
97 | static inline hwaddr get_page_pte_address(uint64_t pte, int granule_sz) | |
98 | { | |
99 | return PTE_ADDRESS(pte, granule_sz); | |
100 | } | |
101 | ||
102 | /** | |
103 | * get_table_pte_address - return table descriptor output address, | |
104 | * ie. address of next level table | |
105 | * ARM ARM Figure D4-16 VMSAv8-64 level0, level1, and level 2 descriptor formats | |
106 | */ | |
107 | static inline hwaddr get_table_pte_address(uint64_t pte, int granule_sz) | |
108 | { | |
109 | return PTE_ADDRESS(pte, granule_sz); | |
110 | } | |
111 | ||
112 | /** | |
113 | * get_block_pte_address - return block descriptor output address and block size | |
114 | * ARM ARM Figure D4-16 VMSAv8-64 level0, level1, and level 2 descriptor formats | |
115 | */ | |
116 | static inline hwaddr get_block_pte_address(uint64_t pte, int level, | |
117 | int granule_sz, uint64_t *bsz) | |
118 | { | |
118eee6c | 119 | int n = level_shift(level, granule_sz); |
93641948 | 120 | |
118eee6c | 121 | *bsz = 1ULL << n; |
93641948 EA |
122 | return PTE_ADDRESS(pte, n); |
123 | } | |
124 | ||
125 | SMMUTransTableInfo *select_tt(SMMUTransCfg *cfg, dma_addr_t iova) | |
126 | { | |
127 | bool tbi = extract64(iova, 55, 1) ? TBI1(cfg->tbi) : TBI0(cfg->tbi); | |
128 | uint8_t tbi_byte = tbi * 8; | |
129 | ||
130 | if (cfg->tt[0].tsz && | |
131 | !extract64(iova, 64 - cfg->tt[0].tsz, cfg->tt[0].tsz - tbi_byte)) { | |
132 | /* there is a ttbr0 region and we are in it (high bits all zero) */ | |
133 | return &cfg->tt[0]; | |
134 | } else if (cfg->tt[1].tsz && | |
135 | !extract64(iova, 64 - cfg->tt[1].tsz, cfg->tt[1].tsz - tbi_byte)) { | |
136 | /* there is a ttbr1 region and we are in it (high bits all one) */ | |
137 | return &cfg->tt[1]; | |
138 | } else if (!cfg->tt[0].tsz) { | |
139 | /* ttbr0 region is "everything not in the ttbr1 region" */ | |
140 | return &cfg->tt[0]; | |
141 | } else if (!cfg->tt[1].tsz) { | |
142 | /* ttbr1 region is "everything not in the ttbr0 region" */ | |
143 | return &cfg->tt[1]; | |
144 | } | |
145 | /* in the gap between the two regions, this is a Translation fault */ | |
146 | return NULL; | |
147 | } | |
148 | ||
149 | /** | |
150 | * smmu_ptw_64 - VMSAv8-64 Walk of the page tables for a given IOVA | |
151 | * @cfg: translation config | |
152 | * @iova: iova to translate | |
153 | * @perm: access type | |
154 | * @tlbe: IOMMUTLBEntry (out) | |
155 | * @info: handle to an error info | |
156 | * | |
157 | * Return 0 on success, < 0 on error. In case of error, @info is filled | |
158 | * and tlbe->perm is set to IOMMU_NONE. | |
159 | * Upon success, @tlbe is filled with translated_addr and entry | |
160 | * permission rights. | |
161 | */ | |
162 | static int smmu_ptw_64(SMMUTransCfg *cfg, | |
163 | dma_addr_t iova, IOMMUAccessFlags perm, | |
164 | IOMMUTLBEntry *tlbe, SMMUPTWEventInfo *info) | |
165 | { | |
166 | dma_addr_t baseaddr, indexmask; | |
167 | int stage = cfg->stage; | |
168 | SMMUTransTableInfo *tt = select_tt(cfg, iova); | |
169 | uint8_t level, granule_sz, inputsize, stride; | |
170 | ||
171 | if (!tt || tt->disabled) { | |
172 | info->type = SMMU_PTW_ERR_TRANSLATION; | |
173 | goto error; | |
174 | } | |
175 | ||
176 | granule_sz = tt->granule_sz; | |
177 | stride = granule_sz - 3; | |
178 | inputsize = 64 - tt->tsz; | |
179 | level = 4 - (inputsize - 4) / stride; | |
180 | indexmask = (1ULL << (inputsize - (stride * (4 - level)))) - 1; | |
181 | baseaddr = extract64(tt->ttb, 0, 48); | |
182 | baseaddr &= ~indexmask; | |
183 | ||
184 | tlbe->iova = iova; | |
185 | tlbe->addr_mask = (1 << granule_sz) - 1; | |
186 | ||
187 | while (level <= 3) { | |
188 | uint64_t subpage_size = 1ULL << level_shift(level, granule_sz); | |
189 | uint64_t mask = subpage_size - 1; | |
190 | uint32_t offset = iova_level_offset(iova, inputsize, level, granule_sz); | |
191 | uint64_t pte; | |
192 | dma_addr_t pte_addr = baseaddr + offset * sizeof(pte); | |
193 | uint8_t ap; | |
194 | ||
195 | if (get_pte(baseaddr, offset, &pte, info)) { | |
196 | goto error; | |
197 | } | |
198 | trace_smmu_ptw_level(level, iova, subpage_size, | |
199 | baseaddr, offset, pte); | |
200 | ||
201 | if (is_invalid_pte(pte) || is_reserved_pte(pte, level)) { | |
202 | trace_smmu_ptw_invalid_pte(stage, level, baseaddr, | |
203 | pte_addr, offset, pte); | |
204 | info->type = SMMU_PTW_ERR_TRANSLATION; | |
205 | goto error; | |
206 | } | |
207 | ||
208 | if (is_page_pte(pte, level)) { | |
209 | uint64_t gpa = get_page_pte_address(pte, granule_sz); | |
210 | ||
211 | ap = PTE_AP(pte); | |
212 | if (is_permission_fault(ap, perm)) { | |
213 | info->type = SMMU_PTW_ERR_PERMISSION; | |
214 | goto error; | |
215 | } | |
216 | ||
217 | tlbe->translated_addr = gpa + (iova & mask); | |
218 | tlbe->perm = PTE_AP_TO_PERM(ap); | |
219 | trace_smmu_ptw_page_pte(stage, level, iova, | |
220 | baseaddr, pte_addr, pte, gpa); | |
221 | return 0; | |
222 | } | |
223 | if (is_block_pte(pte, level)) { | |
224 | uint64_t block_size; | |
225 | hwaddr gpa = get_block_pte_address(pte, level, granule_sz, | |
226 | &block_size); | |
227 | ||
228 | ap = PTE_AP(pte); | |
229 | if (is_permission_fault(ap, perm)) { | |
230 | info->type = SMMU_PTW_ERR_PERMISSION; | |
231 | goto error; | |
232 | } | |
233 | ||
234 | trace_smmu_ptw_block_pte(stage, level, baseaddr, | |
235 | pte_addr, pte, iova, gpa, | |
236 | block_size >> 20); | |
237 | ||
238 | tlbe->translated_addr = gpa + (iova & mask); | |
239 | tlbe->perm = PTE_AP_TO_PERM(ap); | |
240 | return 0; | |
241 | } | |
242 | ||
243 | /* table pte */ | |
244 | ap = PTE_APTABLE(pte); | |
245 | ||
246 | if (is_permission_fault(ap, perm)) { | |
247 | info->type = SMMU_PTW_ERR_PERMISSION; | |
248 | goto error; | |
249 | } | |
250 | baseaddr = get_table_pte_address(pte, granule_sz); | |
251 | level++; | |
252 | } | |
253 | ||
254 | info->type = SMMU_PTW_ERR_TRANSLATION; | |
255 | ||
256 | error: | |
257 | tlbe->perm = IOMMU_NONE; | |
258 | return -EINVAL; | |
259 | } | |
260 | ||
261 | /** | |
262 | * smmu_ptw - Walk the page tables for an IOVA, according to @cfg | |
263 | * | |
264 | * @cfg: translation configuration | |
265 | * @iova: iova to translate | |
266 | * @perm: tentative access type | |
267 | * @tlbe: returned entry | |
268 | * @info: ptw event handle | |
269 | * | |
270 | * return 0 on success | |
271 | */ | |
272 | inline int smmu_ptw(SMMUTransCfg *cfg, dma_addr_t iova, IOMMUAccessFlags perm, | |
273 | IOMMUTLBEntry *tlbe, SMMUPTWEventInfo *info) | |
274 | { | |
275 | if (!cfg->aa64) { | |
276 | /* | |
277 | * This code path is not entered as we check this while decoding | |
278 | * the configuration data in the derived SMMU model. | |
279 | */ | |
280 | g_assert_not_reached(); | |
281 | } | |
282 | ||
283 | return smmu_ptw_64(cfg, iova, perm, tlbe, info); | |
284 | } | |
527773ee | 285 | |
cac994ef EA |
286 | /** |
287 | * The bus number is used for lookup when SID based invalidation occurs. | |
288 | * In that case we lazily populate the SMMUPciBus array from the bus hash | |
289 | * table. At the time the SMMUPciBus is created (smmu_find_add_as), the bus | |
290 | * numbers may not be always initialized yet. | |
291 | */ | |
292 | SMMUPciBus *smmu_find_smmu_pcibus(SMMUState *s, uint8_t bus_num) | |
293 | { | |
294 | SMMUPciBus *smmu_pci_bus = s->smmu_pcibus_by_bus_num[bus_num]; | |
295 | ||
296 | if (!smmu_pci_bus) { | |
297 | GHashTableIter iter; | |
298 | ||
299 | g_hash_table_iter_init(&iter, s->smmu_pcibus_by_busptr); | |
300 | while (g_hash_table_iter_next(&iter, NULL, (void **)&smmu_pci_bus)) { | |
301 | if (pci_bus_num(smmu_pci_bus->bus) == bus_num) { | |
302 | s->smmu_pcibus_by_bus_num[bus_num] = smmu_pci_bus; | |
303 | return smmu_pci_bus; | |
304 | } | |
305 | } | |
306 | } | |
307 | return smmu_pci_bus; | |
308 | } | |
309 | ||
310 | static AddressSpace *smmu_find_add_as(PCIBus *bus, void *opaque, int devfn) | |
311 | { | |
312 | SMMUState *s = opaque; | |
313 | SMMUPciBus *sbus = g_hash_table_lookup(s->smmu_pcibus_by_busptr, bus); | |
314 | SMMUDevice *sdev; | |
6ce9297b | 315 | static unsigned int index; |
cac994ef EA |
316 | |
317 | if (!sbus) { | |
318 | sbus = g_malloc0(sizeof(SMMUPciBus) + | |
319 | sizeof(SMMUDevice *) * SMMU_PCI_DEVFN_MAX); | |
320 | sbus->bus = bus; | |
321 | g_hash_table_insert(s->smmu_pcibus_by_busptr, bus, sbus); | |
322 | } | |
323 | ||
324 | sdev = sbus->pbdev[devfn]; | |
325 | if (!sdev) { | |
6ce9297b EA |
326 | char *name = g_strdup_printf("%s-%d-%d", s->mrtypename, devfn, index++); |
327 | ||
cac994ef EA |
328 | sdev = sbus->pbdev[devfn] = g_new0(SMMUDevice, 1); |
329 | ||
330 | sdev->smmu = s; | |
331 | sdev->bus = bus; | |
332 | sdev->devfn = devfn; | |
333 | ||
334 | memory_region_init_iommu(&sdev->iommu, sizeof(sdev->iommu), | |
335 | s->mrtypename, | |
336 | OBJECT(s), name, 1ULL << SMMU_MAX_VA_BITS); | |
337 | address_space_init(&sdev->as, | |
338 | MEMORY_REGION(&sdev->iommu), name); | |
339 | trace_smmu_add_mr(name); | |
340 | g_free(name); | |
341 | } | |
342 | ||
343 | return &sdev->as; | |
344 | } | |
345 | ||
32cfd7f3 EA |
346 | IOMMUMemoryRegion *smmu_iommu_mr(SMMUState *s, uint32_t sid) |
347 | { | |
348 | uint8_t bus_n, devfn; | |
349 | SMMUPciBus *smmu_bus; | |
350 | SMMUDevice *smmu; | |
351 | ||
352 | bus_n = PCI_BUS_NUM(sid); | |
353 | smmu_bus = smmu_find_smmu_pcibus(s, bus_n); | |
354 | if (smmu_bus) { | |
b78aae9b | 355 | devfn = SMMU_PCI_DEVFN(sid); |
32cfd7f3 EA |
356 | smmu = smmu_bus->pbdev[devfn]; |
357 | if (smmu) { | |
358 | return &smmu->iommu; | |
359 | } | |
360 | } | |
361 | return NULL; | |
362 | } | |
363 | ||
cc27ed81 EA |
364 | static guint smmu_iotlb_key_hash(gconstpointer v) |
365 | { | |
366 | SMMUIOTLBKey *key = (SMMUIOTLBKey *)v; | |
367 | uint32_t a, b, c; | |
368 | ||
369 | /* Jenkins hash */ | |
370 | a = b = c = JHASH_INITVAL + sizeof(*key); | |
371 | a += key->asid; | |
372 | b += extract64(key->iova, 0, 32); | |
373 | c += extract64(key->iova, 32, 32); | |
374 | ||
375 | __jhash_mix(a, b, c); | |
376 | __jhash_final(a, b, c); | |
377 | ||
378 | return c; | |
379 | } | |
380 | ||
381 | static gboolean smmu_iotlb_key_equal(gconstpointer v1, gconstpointer v2) | |
382 | { | |
383 | const SMMUIOTLBKey *k1 = v1; | |
384 | const SMMUIOTLBKey *k2 = v2; | |
385 | ||
386 | return (k1->asid == k2->asid) && (k1->iova == k2->iova); | |
387 | } | |
388 | ||
832e4222 EA |
389 | /* Unmap the whole notifier's range */ |
390 | static void smmu_unmap_notifier_range(IOMMUNotifier *n) | |
391 | { | |
392 | IOMMUTLBEntry entry; | |
393 | ||
394 | entry.target_as = &address_space_memory; | |
395 | entry.iova = n->start; | |
396 | entry.perm = IOMMU_NONE; | |
397 | entry.addr_mask = n->end - n->start; | |
398 | ||
399 | memory_region_notify_one(n, &entry); | |
400 | } | |
401 | ||
402 | /* Unmap all notifiers attached to @mr */ | |
403 | inline void smmu_inv_notifiers_mr(IOMMUMemoryRegion *mr) | |
404 | { | |
405 | IOMMUNotifier *n; | |
406 | ||
407 | trace_smmu_inv_notifiers_mr(mr->parent_obj.name); | |
408 | IOMMU_NOTIFIER_FOREACH(n, mr) { | |
409 | smmu_unmap_notifier_range(n); | |
410 | } | |
411 | } | |
412 | ||
413 | /* Unmap all notifiers of all mr's */ | |
414 | void smmu_inv_notifiers_all(SMMUState *s) | |
415 | { | |
c6370441 | 416 | SMMUDevice *sdev; |
832e4222 | 417 | |
c6370441 EA |
418 | QLIST_FOREACH(sdev, &s->devices_with_notifiers, next) { |
419 | smmu_inv_notifiers_mr(&sdev->iommu); | |
832e4222 EA |
420 | } |
421 | } | |
422 | ||
527773ee EA |
423 | static void smmu_base_realize(DeviceState *dev, Error **errp) |
424 | { | |
cac994ef | 425 | SMMUState *s = ARM_SMMU(dev); |
527773ee EA |
426 | SMMUBaseClass *sbc = ARM_SMMU_GET_CLASS(dev); |
427 | Error *local_err = NULL; | |
428 | ||
429 | sbc->parent_realize(dev, &local_err); | |
430 | if (local_err) { | |
431 | error_propagate(errp, local_err); | |
432 | return; | |
433 | } | |
32cfd7f3 | 434 | s->configs = g_hash_table_new_full(NULL, NULL, NULL, g_free); |
cc27ed81 EA |
435 | s->iotlb = g_hash_table_new_full(smmu_iotlb_key_hash, smmu_iotlb_key_equal, |
436 | g_free, g_free); | |
cac994ef EA |
437 | s->smmu_pcibus_by_busptr = g_hash_table_new(NULL, NULL); |
438 | ||
439 | if (s->primary_bus) { | |
440 | pci_setup_iommu(s->primary_bus, smmu_find_add_as, s); | |
441 | } else { | |
442 | error_setg(errp, "SMMU is not attached to any PCI bus!"); | |
443 | } | |
527773ee EA |
444 | } |
445 | ||
446 | static void smmu_base_reset(DeviceState *dev) | |
447 | { | |
32cfd7f3 EA |
448 | SMMUState *s = ARM_SMMU(dev); |
449 | ||
450 | g_hash_table_remove_all(s->configs); | |
cc27ed81 | 451 | g_hash_table_remove_all(s->iotlb); |
527773ee EA |
452 | } |
453 | ||
454 | static Property smmu_dev_properties[] = { | |
455 | DEFINE_PROP_UINT8("bus_num", SMMUState, bus_num, 0), | |
456 | DEFINE_PROP_LINK("primary-bus", SMMUState, primary_bus, "PCI", PCIBus *), | |
457 | DEFINE_PROP_END_OF_LIST(), | |
458 | }; | |
459 | ||
460 | static void smmu_base_class_init(ObjectClass *klass, void *data) | |
461 | { | |
462 | DeviceClass *dc = DEVICE_CLASS(klass); | |
463 | SMMUBaseClass *sbc = ARM_SMMU_CLASS(klass); | |
464 | ||
465 | dc->props = smmu_dev_properties; | |
466 | device_class_set_parent_realize(dc, smmu_base_realize, | |
467 | &sbc->parent_realize); | |
468 | dc->reset = smmu_base_reset; | |
469 | } | |
470 | ||
471 | static const TypeInfo smmu_base_info = { | |
472 | .name = TYPE_ARM_SMMU, | |
473 | .parent = TYPE_SYS_BUS_DEVICE, | |
474 | .instance_size = sizeof(SMMUState), | |
475 | .class_data = NULL, | |
476 | .class_size = sizeof(SMMUBaseClass), | |
477 | .class_init = smmu_base_class_init, | |
478 | .abstract = true, | |
479 | }; | |
480 | ||
481 | static void smmu_base_register_types(void) | |
482 | { | |
483 | type_register_static(&smmu_base_info); | |
484 | } | |
485 | ||
486 | type_init(smmu_base_register_types) | |
487 |