]>
Commit | Line | Data |
---|---|---|
527773ee EA |
1 | /* |
2 | * Copyright (C) 2014-2016 Broadcom Corporation | |
3 | * Copyright (c) 2017 Red Hat, Inc. | |
4 | * Written by Prem Mallappa, Eric Auger | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify | |
7 | * it under the terms of the GNU General Public License version 2 as | |
8 | * published by the Free Software Foundation. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, | |
11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
13 | * GNU General Public License for more details. | |
14 | * | |
15 | * Author: Prem Mallappa <pmallapp@broadcom.com> | |
16 | * | |
17 | */ | |
18 | ||
19 | #include "qemu/osdep.h" | |
527773ee EA |
20 | #include "exec/address-spaces.h" |
21 | #include "trace.h" | |
22 | #include "exec/target_page.h" | |
2e5b09fd | 23 | #include "hw/core/cpu.h" |
527773ee EA |
24 | #include "hw/qdev-properties.h" |
25 | #include "qapi/error.h" | |
cc27ed81 | 26 | #include "qemu/jhash.h" |
0b8fa32f | 27 | #include "qemu/module.h" |
527773ee EA |
28 | |
29 | #include "qemu/error-report.h" | |
30 | #include "hw/arm/smmu-common.h" | |
93641948 EA |
31 | #include "smmu-internal.h" |
32 | ||
cc27ed81 EA |
33 | /* IOTLB Management */ |
34 | ||
35 | inline void smmu_iotlb_inv_all(SMMUState *s) | |
36 | { | |
37 | trace_smmu_iotlb_inv_all(); | |
38 | g_hash_table_remove_all(s->iotlb); | |
39 | } | |
40 | ||
41 | static gboolean smmu_hash_remove_by_asid(gpointer key, gpointer value, | |
42 | gpointer user_data) | |
43 | { | |
44 | uint16_t asid = *(uint16_t *)user_data; | |
45 | SMMUIOTLBKey *iotlb_key = (SMMUIOTLBKey *)key; | |
46 | ||
47 | return iotlb_key->asid == asid; | |
48 | } | |
49 | ||
50 | inline void smmu_iotlb_inv_iova(SMMUState *s, uint16_t asid, dma_addr_t iova) | |
51 | { | |
52 | SMMUIOTLBKey key = {.asid = asid, .iova = iova}; | |
53 | ||
54 | trace_smmu_iotlb_inv_iova(asid, iova); | |
55 | g_hash_table_remove(s->iotlb, &key); | |
56 | } | |
57 | ||
58 | inline void smmu_iotlb_inv_asid(SMMUState *s, uint16_t asid) | |
59 | { | |
60 | trace_smmu_iotlb_inv_asid(asid); | |
61 | g_hash_table_foreach_remove(s->iotlb, smmu_hash_remove_by_asid, &asid); | |
62 | } | |
63 | ||
93641948 EA |
64 | /* VMSAv8-64 Translation */ |
65 | ||
66 | /** | |
67 | * get_pte - Get the content of a page table entry located at | |
68 | * @base_addr[@index] | |
69 | */ | |
70 | static int get_pte(dma_addr_t baseaddr, uint32_t index, uint64_t *pte, | |
71 | SMMUPTWEventInfo *info) | |
72 | { | |
73 | int ret; | |
74 | dma_addr_t addr = baseaddr + index * sizeof(*pte); | |
75 | ||
76 | /* TODO: guarantee 64-bit single-copy atomicity */ | |
18610bfd | 77 | ret = dma_memory_read(&address_space_memory, addr, pte, sizeof(*pte)); |
93641948 EA |
78 | |
79 | if (ret != MEMTX_OK) { | |
80 | info->type = SMMU_PTW_ERR_WALK_EABT; | |
81 | info->addr = addr; | |
82 | return -EINVAL; | |
83 | } | |
84 | trace_smmu_get_pte(baseaddr, index, addr, *pte); | |
85 | return 0; | |
86 | } | |
87 | ||
88 | /* VMSAv8-64 Translation Table Format Descriptor Decoding */ | |
89 | ||
90 | /** | |
91 | * get_page_pte_address - returns the L3 descriptor output address, | |
92 | * ie. the page frame | |
93 | * ARM ARM spec: Figure D4-17 VMSAv8-64 level 3 descriptor format | |
94 | */ | |
95 | static inline hwaddr get_page_pte_address(uint64_t pte, int granule_sz) | |
96 | { | |
97 | return PTE_ADDRESS(pte, granule_sz); | |
98 | } | |
99 | ||
100 | /** | |
101 | * get_table_pte_address - return table descriptor output address, | |
102 | * ie. address of next level table | |
103 | * ARM ARM Figure D4-16 VMSAv8-64 level0, level1, and level 2 descriptor formats | |
104 | */ | |
105 | static inline hwaddr get_table_pte_address(uint64_t pte, int granule_sz) | |
106 | { | |
107 | return PTE_ADDRESS(pte, granule_sz); | |
108 | } | |
109 | ||
110 | /** | |
111 | * get_block_pte_address - return block descriptor output address and block size | |
112 | * ARM ARM Figure D4-16 VMSAv8-64 level0, level1, and level 2 descriptor formats | |
113 | */ | |
114 | static inline hwaddr get_block_pte_address(uint64_t pte, int level, | |
115 | int granule_sz, uint64_t *bsz) | |
116 | { | |
118eee6c | 117 | int n = level_shift(level, granule_sz); |
93641948 | 118 | |
118eee6c | 119 | *bsz = 1ULL << n; |
93641948 EA |
120 | return PTE_ADDRESS(pte, n); |
121 | } | |
122 | ||
123 | SMMUTransTableInfo *select_tt(SMMUTransCfg *cfg, dma_addr_t iova) | |
124 | { | |
125 | bool tbi = extract64(iova, 55, 1) ? TBI1(cfg->tbi) : TBI0(cfg->tbi); | |
126 | uint8_t tbi_byte = tbi * 8; | |
127 | ||
128 | if (cfg->tt[0].tsz && | |
129 | !extract64(iova, 64 - cfg->tt[0].tsz, cfg->tt[0].tsz - tbi_byte)) { | |
130 | /* there is a ttbr0 region and we are in it (high bits all zero) */ | |
131 | return &cfg->tt[0]; | |
132 | } else if (cfg->tt[1].tsz && | |
133 | !extract64(iova, 64 - cfg->tt[1].tsz, cfg->tt[1].tsz - tbi_byte)) { | |
134 | /* there is a ttbr1 region and we are in it (high bits all one) */ | |
135 | return &cfg->tt[1]; | |
136 | } else if (!cfg->tt[0].tsz) { | |
137 | /* ttbr0 region is "everything not in the ttbr1 region" */ | |
138 | return &cfg->tt[0]; | |
139 | } else if (!cfg->tt[1].tsz) { | |
140 | /* ttbr1 region is "everything not in the ttbr0 region" */ | |
141 | return &cfg->tt[1]; | |
142 | } | |
143 | /* in the gap between the two regions, this is a Translation fault */ | |
144 | return NULL; | |
145 | } | |
146 | ||
147 | /** | |
148 | * smmu_ptw_64 - VMSAv8-64 Walk of the page tables for a given IOVA | |
149 | * @cfg: translation config | |
150 | * @iova: iova to translate | |
151 | * @perm: access type | |
152 | * @tlbe: IOMMUTLBEntry (out) | |
153 | * @info: handle to an error info | |
154 | * | |
155 | * Return 0 on success, < 0 on error. In case of error, @info is filled | |
156 | * and tlbe->perm is set to IOMMU_NONE. | |
157 | * Upon success, @tlbe is filled with translated_addr and entry | |
158 | * permission rights. | |
159 | */ | |
160 | static int smmu_ptw_64(SMMUTransCfg *cfg, | |
161 | dma_addr_t iova, IOMMUAccessFlags perm, | |
162 | IOMMUTLBEntry *tlbe, SMMUPTWEventInfo *info) | |
163 | { | |
164 | dma_addr_t baseaddr, indexmask; | |
165 | int stage = cfg->stage; | |
166 | SMMUTransTableInfo *tt = select_tt(cfg, iova); | |
167 | uint8_t level, granule_sz, inputsize, stride; | |
168 | ||
169 | if (!tt || tt->disabled) { | |
170 | info->type = SMMU_PTW_ERR_TRANSLATION; | |
171 | goto error; | |
172 | } | |
173 | ||
174 | granule_sz = tt->granule_sz; | |
175 | stride = granule_sz - 3; | |
176 | inputsize = 64 - tt->tsz; | |
177 | level = 4 - (inputsize - 4) / stride; | |
178 | indexmask = (1ULL << (inputsize - (stride * (4 - level)))) - 1; | |
179 | baseaddr = extract64(tt->ttb, 0, 48); | |
180 | baseaddr &= ~indexmask; | |
181 | ||
182 | tlbe->iova = iova; | |
183 | tlbe->addr_mask = (1 << granule_sz) - 1; | |
184 | ||
185 | while (level <= 3) { | |
186 | uint64_t subpage_size = 1ULL << level_shift(level, granule_sz); | |
187 | uint64_t mask = subpage_size - 1; | |
188 | uint32_t offset = iova_level_offset(iova, inputsize, level, granule_sz); | |
189 | uint64_t pte; | |
190 | dma_addr_t pte_addr = baseaddr + offset * sizeof(pte); | |
191 | uint8_t ap; | |
192 | ||
193 | if (get_pte(baseaddr, offset, &pte, info)) { | |
194 | goto error; | |
195 | } | |
196 | trace_smmu_ptw_level(level, iova, subpage_size, | |
197 | baseaddr, offset, pte); | |
198 | ||
199 | if (is_invalid_pte(pte) || is_reserved_pte(pte, level)) { | |
200 | trace_smmu_ptw_invalid_pte(stage, level, baseaddr, | |
201 | pte_addr, offset, pte); | |
202 | info->type = SMMU_PTW_ERR_TRANSLATION; | |
203 | goto error; | |
204 | } | |
205 | ||
206 | if (is_page_pte(pte, level)) { | |
207 | uint64_t gpa = get_page_pte_address(pte, granule_sz); | |
208 | ||
209 | ap = PTE_AP(pte); | |
210 | if (is_permission_fault(ap, perm)) { | |
211 | info->type = SMMU_PTW_ERR_PERMISSION; | |
212 | goto error; | |
213 | } | |
214 | ||
215 | tlbe->translated_addr = gpa + (iova & mask); | |
216 | tlbe->perm = PTE_AP_TO_PERM(ap); | |
217 | trace_smmu_ptw_page_pte(stage, level, iova, | |
218 | baseaddr, pte_addr, pte, gpa); | |
219 | return 0; | |
220 | } | |
221 | if (is_block_pte(pte, level)) { | |
222 | uint64_t block_size; | |
223 | hwaddr gpa = get_block_pte_address(pte, level, granule_sz, | |
224 | &block_size); | |
225 | ||
226 | ap = PTE_AP(pte); | |
227 | if (is_permission_fault(ap, perm)) { | |
228 | info->type = SMMU_PTW_ERR_PERMISSION; | |
229 | goto error; | |
230 | } | |
231 | ||
232 | trace_smmu_ptw_block_pte(stage, level, baseaddr, | |
233 | pte_addr, pte, iova, gpa, | |
234 | block_size >> 20); | |
235 | ||
236 | tlbe->translated_addr = gpa + (iova & mask); | |
237 | tlbe->perm = PTE_AP_TO_PERM(ap); | |
238 | return 0; | |
239 | } | |
240 | ||
241 | /* table pte */ | |
242 | ap = PTE_APTABLE(pte); | |
243 | ||
244 | if (is_permission_fault(ap, perm)) { | |
245 | info->type = SMMU_PTW_ERR_PERMISSION; | |
246 | goto error; | |
247 | } | |
248 | baseaddr = get_table_pte_address(pte, granule_sz); | |
249 | level++; | |
250 | } | |
251 | ||
252 | info->type = SMMU_PTW_ERR_TRANSLATION; | |
253 | ||
254 | error: | |
255 | tlbe->perm = IOMMU_NONE; | |
256 | return -EINVAL; | |
257 | } | |
258 | ||
259 | /** | |
260 | * smmu_ptw - Walk the page tables for an IOVA, according to @cfg | |
261 | * | |
262 | * @cfg: translation configuration | |
263 | * @iova: iova to translate | |
264 | * @perm: tentative access type | |
265 | * @tlbe: returned entry | |
266 | * @info: ptw event handle | |
267 | * | |
268 | * return 0 on success | |
269 | */ | |
270 | inline int smmu_ptw(SMMUTransCfg *cfg, dma_addr_t iova, IOMMUAccessFlags perm, | |
271 | IOMMUTLBEntry *tlbe, SMMUPTWEventInfo *info) | |
272 | { | |
273 | if (!cfg->aa64) { | |
274 | /* | |
275 | * This code path is not entered as we check this while decoding | |
276 | * the configuration data in the derived SMMU model. | |
277 | */ | |
278 | g_assert_not_reached(); | |
279 | } | |
280 | ||
281 | return smmu_ptw_64(cfg, iova, perm, tlbe, info); | |
282 | } | |
527773ee | 283 | |
cac994ef EA |
284 | /** |
285 | * The bus number is used for lookup when SID based invalidation occurs. | |
286 | * In that case we lazily populate the SMMUPciBus array from the bus hash | |
287 | * table. At the time the SMMUPciBus is created (smmu_find_add_as), the bus | |
288 | * numbers may not be always initialized yet. | |
289 | */ | |
290 | SMMUPciBus *smmu_find_smmu_pcibus(SMMUState *s, uint8_t bus_num) | |
291 | { | |
292 | SMMUPciBus *smmu_pci_bus = s->smmu_pcibus_by_bus_num[bus_num]; | |
5ca0e6fe | 293 | GHashTableIter iter; |
cac994ef | 294 | |
5ca0e6fe PMD |
295 | if (smmu_pci_bus) { |
296 | return smmu_pci_bus; | |
297 | } | |
cac994ef | 298 | |
5ca0e6fe PMD |
299 | g_hash_table_iter_init(&iter, s->smmu_pcibus_by_busptr); |
300 | while (g_hash_table_iter_next(&iter, NULL, (void **)&smmu_pci_bus)) { | |
301 | if (pci_bus_num(smmu_pci_bus->bus) == bus_num) { | |
302 | s->smmu_pcibus_by_bus_num[bus_num] = smmu_pci_bus; | |
303 | return smmu_pci_bus; | |
cac994ef EA |
304 | } |
305 | } | |
5ca0e6fe PMD |
306 | |
307 | return NULL; | |
cac994ef EA |
308 | } |
309 | ||
310 | static AddressSpace *smmu_find_add_as(PCIBus *bus, void *opaque, int devfn) | |
311 | { | |
312 | SMMUState *s = opaque; | |
313 | SMMUPciBus *sbus = g_hash_table_lookup(s->smmu_pcibus_by_busptr, bus); | |
314 | SMMUDevice *sdev; | |
6ce9297b | 315 | static unsigned int index; |
cac994ef EA |
316 | |
317 | if (!sbus) { | |
318 | sbus = g_malloc0(sizeof(SMMUPciBus) + | |
319 | sizeof(SMMUDevice *) * SMMU_PCI_DEVFN_MAX); | |
320 | sbus->bus = bus; | |
321 | g_hash_table_insert(s->smmu_pcibus_by_busptr, bus, sbus); | |
322 | } | |
323 | ||
324 | sdev = sbus->pbdev[devfn]; | |
325 | if (!sdev) { | |
6ce9297b EA |
326 | char *name = g_strdup_printf("%s-%d-%d", s->mrtypename, devfn, index++); |
327 | ||
cac994ef EA |
328 | sdev = sbus->pbdev[devfn] = g_new0(SMMUDevice, 1); |
329 | ||
330 | sdev->smmu = s; | |
331 | sdev->bus = bus; | |
332 | sdev->devfn = devfn; | |
333 | ||
334 | memory_region_init_iommu(&sdev->iommu, sizeof(sdev->iommu), | |
335 | s->mrtypename, | |
336 | OBJECT(s), name, 1ULL << SMMU_MAX_VA_BITS); | |
337 | address_space_init(&sdev->as, | |
338 | MEMORY_REGION(&sdev->iommu), name); | |
339 | trace_smmu_add_mr(name); | |
340 | g_free(name); | |
341 | } | |
342 | ||
343 | return &sdev->as; | |
344 | } | |
345 | ||
32cfd7f3 EA |
346 | IOMMUMemoryRegion *smmu_iommu_mr(SMMUState *s, uint32_t sid) |
347 | { | |
348 | uint8_t bus_n, devfn; | |
349 | SMMUPciBus *smmu_bus; | |
350 | SMMUDevice *smmu; | |
351 | ||
352 | bus_n = PCI_BUS_NUM(sid); | |
353 | smmu_bus = smmu_find_smmu_pcibus(s, bus_n); | |
354 | if (smmu_bus) { | |
b78aae9b | 355 | devfn = SMMU_PCI_DEVFN(sid); |
32cfd7f3 EA |
356 | smmu = smmu_bus->pbdev[devfn]; |
357 | if (smmu) { | |
358 | return &smmu->iommu; | |
359 | } | |
360 | } | |
361 | return NULL; | |
362 | } | |
363 | ||
cc27ed81 EA |
364 | static guint smmu_iotlb_key_hash(gconstpointer v) |
365 | { | |
366 | SMMUIOTLBKey *key = (SMMUIOTLBKey *)v; | |
367 | uint32_t a, b, c; | |
368 | ||
369 | /* Jenkins hash */ | |
370 | a = b = c = JHASH_INITVAL + sizeof(*key); | |
371 | a += key->asid; | |
372 | b += extract64(key->iova, 0, 32); | |
373 | c += extract64(key->iova, 32, 32); | |
374 | ||
375 | __jhash_mix(a, b, c); | |
376 | __jhash_final(a, b, c); | |
377 | ||
378 | return c; | |
379 | } | |
380 | ||
381 | static gboolean smmu_iotlb_key_equal(gconstpointer v1, gconstpointer v2) | |
382 | { | |
383 | const SMMUIOTLBKey *k1 = v1; | |
384 | const SMMUIOTLBKey *k2 = v2; | |
385 | ||
386 | return (k1->asid == k2->asid) && (k1->iova == k2->iova); | |
387 | } | |
388 | ||
832e4222 EA |
389 | /* Unmap the whole notifier's range */ |
390 | static void smmu_unmap_notifier_range(IOMMUNotifier *n) | |
391 | { | |
392 | IOMMUTLBEntry entry; | |
393 | ||
394 | entry.target_as = &address_space_memory; | |
395 | entry.iova = n->start; | |
396 | entry.perm = IOMMU_NONE; | |
397 | entry.addr_mask = n->end - n->start; | |
398 | ||
399 | memory_region_notify_one(n, &entry); | |
400 | } | |
401 | ||
402 | /* Unmap all notifiers attached to @mr */ | |
403 | inline void smmu_inv_notifiers_mr(IOMMUMemoryRegion *mr) | |
404 | { | |
405 | IOMMUNotifier *n; | |
406 | ||
407 | trace_smmu_inv_notifiers_mr(mr->parent_obj.name); | |
408 | IOMMU_NOTIFIER_FOREACH(n, mr) { | |
409 | smmu_unmap_notifier_range(n); | |
410 | } | |
411 | } | |
412 | ||
413 | /* Unmap all notifiers of all mr's */ | |
414 | void smmu_inv_notifiers_all(SMMUState *s) | |
415 | { | |
c6370441 | 416 | SMMUDevice *sdev; |
832e4222 | 417 | |
c6370441 EA |
418 | QLIST_FOREACH(sdev, &s->devices_with_notifiers, next) { |
419 | smmu_inv_notifiers_mr(&sdev->iommu); | |
832e4222 EA |
420 | } |
421 | } | |
422 | ||
527773ee EA |
423 | static void smmu_base_realize(DeviceState *dev, Error **errp) |
424 | { | |
cac994ef | 425 | SMMUState *s = ARM_SMMU(dev); |
527773ee EA |
426 | SMMUBaseClass *sbc = ARM_SMMU_GET_CLASS(dev); |
427 | Error *local_err = NULL; | |
428 | ||
429 | sbc->parent_realize(dev, &local_err); | |
430 | if (local_err) { | |
431 | error_propagate(errp, local_err); | |
432 | return; | |
433 | } | |
32cfd7f3 | 434 | s->configs = g_hash_table_new_full(NULL, NULL, NULL, g_free); |
cc27ed81 EA |
435 | s->iotlb = g_hash_table_new_full(smmu_iotlb_key_hash, smmu_iotlb_key_equal, |
436 | g_free, g_free); | |
cac994ef EA |
437 | s->smmu_pcibus_by_busptr = g_hash_table_new(NULL, NULL); |
438 | ||
439 | if (s->primary_bus) { | |
440 | pci_setup_iommu(s->primary_bus, smmu_find_add_as, s); | |
441 | } else { | |
442 | error_setg(errp, "SMMU is not attached to any PCI bus!"); | |
443 | } | |
527773ee EA |
444 | } |
445 | ||
446 | static void smmu_base_reset(DeviceState *dev) | |
447 | { | |
32cfd7f3 EA |
448 | SMMUState *s = ARM_SMMU(dev); |
449 | ||
450 | g_hash_table_remove_all(s->configs); | |
cc27ed81 | 451 | g_hash_table_remove_all(s->iotlb); |
527773ee EA |
452 | } |
453 | ||
454 | static Property smmu_dev_properties[] = { | |
455 | DEFINE_PROP_UINT8("bus_num", SMMUState, bus_num, 0), | |
456 | DEFINE_PROP_LINK("primary-bus", SMMUState, primary_bus, "PCI", PCIBus *), | |
457 | DEFINE_PROP_END_OF_LIST(), | |
458 | }; | |
459 | ||
460 | static void smmu_base_class_init(ObjectClass *klass, void *data) | |
461 | { | |
462 | DeviceClass *dc = DEVICE_CLASS(klass); | |
463 | SMMUBaseClass *sbc = ARM_SMMU_CLASS(klass); | |
464 | ||
4f67d30b | 465 | device_class_set_props(dc, smmu_dev_properties); |
527773ee EA |
466 | device_class_set_parent_realize(dc, smmu_base_realize, |
467 | &sbc->parent_realize); | |
468 | dc->reset = smmu_base_reset; | |
469 | } | |
470 | ||
471 | static const TypeInfo smmu_base_info = { | |
472 | .name = TYPE_ARM_SMMU, | |
473 | .parent = TYPE_SYS_BUS_DEVICE, | |
474 | .instance_size = sizeof(SMMUState), | |
475 | .class_data = NULL, | |
476 | .class_size = sizeof(SMMUBaseClass), | |
477 | .class_init = smmu_base_class_init, | |
478 | .abstract = true, | |
479 | }; | |
480 | ||
481 | static void smmu_base_register_types(void) | |
482 | { | |
483 | type_register_static(&smmu_base_info); | |
484 | } | |
485 | ||
486 | type_init(smmu_base_register_types) | |
487 |