]>
Commit | Line | Data |
---|---|---|
527773ee EA |
1 | /* |
2 | * Copyright (C) 2014-2016 Broadcom Corporation | |
3 | * Copyright (c) 2017 Red Hat, Inc. | |
4 | * Written by Prem Mallappa, Eric Auger | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify | |
7 | * it under the terms of the GNU General Public License version 2 as | |
8 | * published by the Free Software Foundation. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, | |
11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
13 | * GNU General Public License for more details. | |
14 | * | |
15 | * Author: Prem Mallappa <pmallapp@broadcom.com> | |
16 | * | |
17 | */ | |
18 | ||
19 | #include "qemu/osdep.h" | |
527773ee EA |
20 | #include "trace.h" |
21 | #include "exec/target_page.h" | |
2e5b09fd | 22 | #include "hw/core/cpu.h" |
527773ee EA |
23 | #include "hw/qdev-properties.h" |
24 | #include "qapi/error.h" | |
cc27ed81 | 25 | #include "qemu/jhash.h" |
0b8fa32f | 26 | #include "qemu/module.h" |
527773ee EA |
27 | |
28 | #include "qemu/error-report.h" | |
29 | #include "hw/arm/smmu-common.h" | |
93641948 EA |
30 | #include "smmu-internal.h" |
31 | ||
cc27ed81 EA |
32 | /* IOTLB Management */ |
33 | ||
60a61f1b EA |
34 | static guint smmu_iotlb_key_hash(gconstpointer v) |
35 | { | |
36 | SMMUIOTLBKey *key = (SMMUIOTLBKey *)v; | |
37 | uint32_t a, b, c; | |
38 | ||
39 | /* Jenkins hash */ | |
40 | a = b = c = JHASH_INITVAL + sizeof(*key); | |
2eaeb7d5 | 41 | a += key->asid + key->vmid + key->level + key->tg; |
60a61f1b EA |
42 | b += extract64(key->iova, 0, 32); |
43 | c += extract64(key->iova, 32, 32); | |
44 | ||
45 | __jhash_mix(a, b, c); | |
46 | __jhash_final(a, b, c); | |
47 | ||
48 | return c; | |
49 | } | |
50 | ||
51 | static gboolean smmu_iotlb_key_equal(gconstpointer v1, gconstpointer v2) | |
52 | { | |
9e54dee7 | 53 | SMMUIOTLBKey *k1 = (SMMUIOTLBKey *)v1, *k2 = (SMMUIOTLBKey *)v2; |
60a61f1b | 54 | |
9e54dee7 | 55 | return (k1->asid == k2->asid) && (k1->iova == k2->iova) && |
2eaeb7d5 MS |
56 | (k1->level == k2->level) && (k1->tg == k2->tg) && |
57 | (k1->vmid == k2->vmid); | |
60a61f1b EA |
58 | } |
59 | ||
2eaeb7d5 | 60 | SMMUIOTLBKey smmu_get_iotlb_key(uint16_t asid, uint16_t vmid, uint64_t iova, |
9e54dee7 | 61 | uint8_t tg, uint8_t level) |
60a61f1b | 62 | { |
2eaeb7d5 MS |
63 | SMMUIOTLBKey key = {.asid = asid, .vmid = vmid, .iova = iova, |
64 | .tg = tg, .level = level}; | |
60a61f1b EA |
65 | |
66 | return key; | |
67 | } | |
68 | ||
a7550158 | 69 | SMMUTLBEntry *smmu_iotlb_lookup(SMMUState *bs, SMMUTransCfg *cfg, |
9e54dee7 | 70 | SMMUTransTableInfo *tt, hwaddr iova) |
6808bca9 | 71 | { |
9e54dee7 EA |
72 | uint8_t tg = (tt->granule_sz - 10) / 2; |
73 | uint8_t inputsize = 64 - tt->tsz; | |
74 | uint8_t stride = tt->granule_sz - 3; | |
75 | uint8_t level = 4 - (inputsize - 4) / stride; | |
76 | SMMUTLBEntry *entry = NULL; | |
77 | ||
78 | while (level <= 3) { | |
79 | uint64_t subpage_size = 1ULL << level_shift(level, tt->granule_sz); | |
80 | uint64_t mask = subpage_size - 1; | |
81 | SMMUIOTLBKey key; | |
82 | ||
2eaeb7d5 MS |
83 | key = smmu_get_iotlb_key(cfg->asid, cfg->s2cfg.vmid, |
84 | iova & ~mask, tg, level); | |
9e54dee7 EA |
85 | entry = g_hash_table_lookup(bs->iotlb, &key); |
86 | if (entry) { | |
87 | break; | |
88 | } | |
89 | level++; | |
90 | } | |
6808bca9 EA |
91 | |
92 | if (entry) { | |
93 | cfg->iotlb_hits++; | |
2eaeb7d5 | 94 | trace_smmu_iotlb_lookup_hit(cfg->asid, cfg->s2cfg.vmid, iova, |
6808bca9 EA |
95 | cfg->iotlb_hits, cfg->iotlb_misses, |
96 | 100 * cfg->iotlb_hits / | |
97 | (cfg->iotlb_hits + cfg->iotlb_misses)); | |
98 | } else { | |
99 | cfg->iotlb_misses++; | |
2eaeb7d5 | 100 | trace_smmu_iotlb_lookup_miss(cfg->asid, cfg->s2cfg.vmid, iova, |
6808bca9 EA |
101 | cfg->iotlb_hits, cfg->iotlb_misses, |
102 | 100 * cfg->iotlb_hits / | |
103 | (cfg->iotlb_hits + cfg->iotlb_misses)); | |
104 | } | |
105 | return entry; | |
106 | } | |
107 | ||
a7550158 | 108 | void smmu_iotlb_insert(SMMUState *bs, SMMUTransCfg *cfg, SMMUTLBEntry *new) |
6808bca9 EA |
109 | { |
110 | SMMUIOTLBKey *key = g_new0(SMMUIOTLBKey, 1); | |
9e54dee7 | 111 | uint8_t tg = (new->granule - 10) / 2; |
6808bca9 EA |
112 | |
113 | if (g_hash_table_size(bs->iotlb) >= SMMU_IOTLB_MAX_SIZE) { | |
114 | smmu_iotlb_inv_all(bs); | |
115 | } | |
116 | ||
2eaeb7d5 MS |
117 | *key = smmu_get_iotlb_key(cfg->asid, cfg->s2cfg.vmid, new->entry.iova, |
118 | tg, new->level); | |
119 | trace_smmu_iotlb_insert(cfg->asid, cfg->s2cfg.vmid, new->entry.iova, | |
120 | tg, new->level); | |
a7550158 | 121 | g_hash_table_insert(bs->iotlb, key, new); |
6808bca9 EA |
122 | } |
123 | ||
9de9fa5c | 124 | void smmu_iotlb_inv_all(SMMUState *s) |
cc27ed81 EA |
125 | { |
126 | trace_smmu_iotlb_inv_all(); | |
127 | g_hash_table_remove_all(s->iotlb); | |
128 | } | |
129 | ||
130 | static gboolean smmu_hash_remove_by_asid(gpointer key, gpointer value, | |
131 | gpointer user_data) | |
132 | { | |
133 | uint16_t asid = *(uint16_t *)user_data; | |
134 | SMMUIOTLBKey *iotlb_key = (SMMUIOTLBKey *)key; | |
135 | ||
60a61f1b | 136 | return SMMU_IOTLB_ASID(*iotlb_key) == asid; |
cc27ed81 | 137 | } |
ccc3ee38 MS |
138 | |
139 | static gboolean smmu_hash_remove_by_vmid(gpointer key, gpointer value, | |
140 | gpointer user_data) | |
141 | { | |
142 | uint16_t vmid = *(uint16_t *)user_data; | |
143 | SMMUIOTLBKey *iotlb_key = (SMMUIOTLBKey *)key; | |
144 | ||
145 | return SMMU_IOTLB_VMID(*iotlb_key) == vmid; | |
146 | } | |
147 | ||
2eaeb7d5 | 148 | static gboolean smmu_hash_remove_by_asid_vmid_iova(gpointer key, gpointer value, |
9e54dee7 | 149 | gpointer user_data) |
cc27ed81 | 150 | { |
9e54dee7 EA |
151 | SMMUTLBEntry *iter = (SMMUTLBEntry *)value; |
152 | IOMMUTLBEntry *entry = &iter->entry; | |
153 | SMMUIOTLBPageInvInfo *info = (SMMUIOTLBPageInvInfo *)user_data; | |
154 | SMMUIOTLBKey iotlb_key = *(SMMUIOTLBKey *)key; | |
155 | ||
156 | if (info->asid >= 0 && info->asid != SMMU_IOTLB_ASID(iotlb_key)) { | |
157 | return false; | |
158 | } | |
2eaeb7d5 MS |
159 | if (info->vmid >= 0 && info->vmid != SMMU_IOTLB_VMID(iotlb_key)) { |
160 | return false; | |
161 | } | |
d5291561 EA |
162 | return ((info->iova & ~entry->addr_mask) == entry->iova) || |
163 | ((entry->iova & ~info->mask) == info->iova); | |
9e54dee7 EA |
164 | } |
165 | ||
2eaeb7d5 | 166 | void smmu_iotlb_inv_iova(SMMUState *s, int asid, int vmid, dma_addr_t iova, |
9de9fa5c | 167 | uint8_t tg, uint64_t num_pages, uint8_t ttl) |
9e54dee7 | 168 | { |
6d9cd115 EA |
169 | /* if tg is not set we use 4KB range invalidation */ |
170 | uint8_t granule = tg ? tg * 2 + 10 : 12; | |
171 | ||
a4b6e1be | 172 | if (ttl && (num_pages == 1) && (asid >= 0)) { |
2eaeb7d5 | 173 | SMMUIOTLBKey key = smmu_get_iotlb_key(asid, vmid, iova, tg, ttl); |
cc27ed81 | 174 | |
6d9cd115 EA |
175 | if (g_hash_table_remove(s->iotlb, &key)) { |
176 | return; | |
177 | } | |
178 | /* | |
179 | * if the entry is not found, let's see if it does not | |
180 | * belong to a larger IOTLB entry | |
181 | */ | |
182 | } | |
d5291561 | 183 | |
6d9cd115 EA |
184 | SMMUIOTLBPageInvInfo info = { |
185 | .asid = asid, .iova = iova, | |
2eaeb7d5 | 186 | .vmid = vmid, |
6d9cd115 | 187 | .mask = (num_pages * 1 << granule) - 1}; |
d5291561 | 188 | |
6d9cd115 | 189 | g_hash_table_foreach_remove(s->iotlb, |
2eaeb7d5 | 190 | smmu_hash_remove_by_asid_vmid_iova, |
6d9cd115 | 191 | &info); |
cc27ed81 EA |
192 | } |
193 | ||
9de9fa5c | 194 | void smmu_iotlb_inv_asid(SMMUState *s, uint16_t asid) |
cc27ed81 EA |
195 | { |
196 | trace_smmu_iotlb_inv_asid(asid); | |
197 | g_hash_table_foreach_remove(s->iotlb, smmu_hash_remove_by_asid, &asid); | |
198 | } | |
199 | ||
ccc3ee38 MS |
200 | inline void smmu_iotlb_inv_vmid(SMMUState *s, uint16_t vmid) |
201 | { | |
202 | trace_smmu_iotlb_inv_vmid(vmid); | |
203 | g_hash_table_foreach_remove(s->iotlb, smmu_hash_remove_by_vmid, &vmid); | |
204 | } | |
205 | ||
93641948 EA |
206 | /* VMSAv8-64 Translation */ |
207 | ||
208 | /** | |
209 | * get_pte - Get the content of a page table entry located at | |
210 | * @base_addr[@index] | |
211 | */ | |
212 | static int get_pte(dma_addr_t baseaddr, uint32_t index, uint64_t *pte, | |
213 | SMMUPTWEventInfo *info) | |
214 | { | |
215 | int ret; | |
216 | dma_addr_t addr = baseaddr + index * sizeof(*pte); | |
217 | ||
218 | /* TODO: guarantee 64-bit single-copy atomicity */ | |
c6445544 | 219 | ret = ldq_le_dma(&address_space_memory, addr, pte, MEMTXATTRS_UNSPECIFIED); |
93641948 EA |
220 | |
221 | if (ret != MEMTX_OK) { | |
222 | info->type = SMMU_PTW_ERR_WALK_EABT; | |
223 | info->addr = addr; | |
224 | return -EINVAL; | |
225 | } | |
226 | trace_smmu_get_pte(baseaddr, index, addr, *pte); | |
227 | return 0; | |
228 | } | |
229 | ||
230 | /* VMSAv8-64 Translation Table Format Descriptor Decoding */ | |
231 | ||
232 | /** | |
233 | * get_page_pte_address - returns the L3 descriptor output address, | |
234 | * ie. the page frame | |
235 | * ARM ARM spec: Figure D4-17 VMSAv8-64 level 3 descriptor format | |
236 | */ | |
237 | static inline hwaddr get_page_pte_address(uint64_t pte, int granule_sz) | |
238 | { | |
239 | return PTE_ADDRESS(pte, granule_sz); | |
240 | } | |
241 | ||
242 | /** | |
243 | * get_table_pte_address - return table descriptor output address, | |
244 | * ie. address of next level table | |
245 | * ARM ARM Figure D4-16 VMSAv8-64 level0, level1, and level 2 descriptor formats | |
246 | */ | |
247 | static inline hwaddr get_table_pte_address(uint64_t pte, int granule_sz) | |
248 | { | |
249 | return PTE_ADDRESS(pte, granule_sz); | |
250 | } | |
251 | ||
252 | /** | |
253 | * get_block_pte_address - return block descriptor output address and block size | |
254 | * ARM ARM Figure D4-16 VMSAv8-64 level0, level1, and level 2 descriptor formats | |
255 | */ | |
256 | static inline hwaddr get_block_pte_address(uint64_t pte, int level, | |
257 | int granule_sz, uint64_t *bsz) | |
258 | { | |
118eee6c | 259 | int n = level_shift(level, granule_sz); |
93641948 | 260 | |
118eee6c | 261 | *bsz = 1ULL << n; |
93641948 EA |
262 | return PTE_ADDRESS(pte, n); |
263 | } | |
264 | ||
265 | SMMUTransTableInfo *select_tt(SMMUTransCfg *cfg, dma_addr_t iova) | |
266 | { | |
267 | bool tbi = extract64(iova, 55, 1) ? TBI1(cfg->tbi) : TBI0(cfg->tbi); | |
268 | uint8_t tbi_byte = tbi * 8; | |
269 | ||
270 | if (cfg->tt[0].tsz && | |
271 | !extract64(iova, 64 - cfg->tt[0].tsz, cfg->tt[0].tsz - tbi_byte)) { | |
272 | /* there is a ttbr0 region and we are in it (high bits all zero) */ | |
273 | return &cfg->tt[0]; | |
274 | } else if (cfg->tt[1].tsz && | |
e431b8f6 | 275 | sextract64(iova, 64 - cfg->tt[1].tsz, cfg->tt[1].tsz - tbi_byte) == -1) { |
93641948 EA |
276 | /* there is a ttbr1 region and we are in it (high bits all one) */ |
277 | return &cfg->tt[1]; | |
278 | } else if (!cfg->tt[0].tsz) { | |
279 | /* ttbr0 region is "everything not in the ttbr1 region" */ | |
280 | return &cfg->tt[0]; | |
281 | } else if (!cfg->tt[1].tsz) { | |
282 | /* ttbr1 region is "everything not in the ttbr0 region" */ | |
283 | return &cfg->tt[1]; | |
284 | } | |
285 | /* in the gap between the two regions, this is a Translation fault */ | |
286 | return NULL; | |
287 | } | |
288 | ||
289 | /** | |
bcc919e7 | 290 | * smmu_ptw_64_s1 - VMSAv8-64 Walk of the page tables for a given IOVA |
93641948 EA |
291 | * @cfg: translation config |
292 | * @iova: iova to translate | |
293 | * @perm: access type | |
a7550158 | 294 | * @tlbe: SMMUTLBEntry (out) |
93641948 EA |
295 | * @info: handle to an error info |
296 | * | |
297 | * Return 0 on success, < 0 on error. In case of error, @info is filled | |
298 | * and tlbe->perm is set to IOMMU_NONE. | |
299 | * Upon success, @tlbe is filled with translated_addr and entry | |
300 | * permission rights. | |
301 | */ | |
bcc919e7 MS |
302 | static int smmu_ptw_64_s1(SMMUTransCfg *cfg, |
303 | dma_addr_t iova, IOMMUAccessFlags perm, | |
304 | SMMUTLBEntry *tlbe, SMMUPTWEventInfo *info) | |
93641948 EA |
305 | { |
306 | dma_addr_t baseaddr, indexmask; | |
307 | int stage = cfg->stage; | |
308 | SMMUTransTableInfo *tt = select_tt(cfg, iova); | |
309 | uint8_t level, granule_sz, inputsize, stride; | |
310 | ||
311 | if (!tt || tt->disabled) { | |
312 | info->type = SMMU_PTW_ERR_TRANSLATION; | |
313 | goto error; | |
314 | } | |
315 | ||
316 | granule_sz = tt->granule_sz; | |
bcc919e7 | 317 | stride = VMSA_STRIDE(granule_sz); |
93641948 EA |
318 | inputsize = 64 - tt->tsz; |
319 | level = 4 - (inputsize - 4) / stride; | |
bcc919e7 | 320 | indexmask = VMSA_IDXMSK(inputsize, stride, level); |
93641948 EA |
321 | baseaddr = extract64(tt->ttb, 0, 48); |
322 | baseaddr &= ~indexmask; | |
323 | ||
bcc919e7 | 324 | while (level < VMSA_LEVELS) { |
93641948 EA |
325 | uint64_t subpage_size = 1ULL << level_shift(level, granule_sz); |
326 | uint64_t mask = subpage_size - 1; | |
327 | uint32_t offset = iova_level_offset(iova, inputsize, level, granule_sz); | |
1733837d | 328 | uint64_t pte, gpa; |
93641948 EA |
329 | dma_addr_t pte_addr = baseaddr + offset * sizeof(pte); |
330 | uint8_t ap; | |
331 | ||
332 | if (get_pte(baseaddr, offset, &pte, info)) { | |
333 | goto error; | |
334 | } | |
bcc919e7 | 335 | trace_smmu_ptw_level(stage, level, iova, subpage_size, |
93641948 EA |
336 | baseaddr, offset, pte); |
337 | ||
338 | if (is_invalid_pte(pte) || is_reserved_pte(pte, level)) { | |
339 | trace_smmu_ptw_invalid_pte(stage, level, baseaddr, | |
340 | pte_addr, offset, pte); | |
1733837d | 341 | break; |
93641948 EA |
342 | } |
343 | ||
1733837d EA |
344 | if (is_table_pte(pte, level)) { |
345 | ap = PTE_APTABLE(pte); | |
93641948 | 346 | |
e7c3b9d9 | 347 | if (is_permission_fault(ap, perm) && !tt->had) { |
93641948 EA |
348 | info->type = SMMU_PTW_ERR_PERMISSION; |
349 | goto error; | |
350 | } | |
1733837d EA |
351 | baseaddr = get_table_pte_address(pte, granule_sz); |
352 | level++; | |
353 | continue; | |
354 | } else if (is_page_pte(pte, level)) { | |
355 | gpa = get_page_pte_address(pte, granule_sz); | |
93641948 EA |
356 | trace_smmu_ptw_page_pte(stage, level, iova, |
357 | baseaddr, pte_addr, pte, gpa); | |
1733837d | 358 | } else { |
93641948 | 359 | uint64_t block_size; |
93641948 | 360 | |
1733837d EA |
361 | gpa = get_block_pte_address(pte, level, granule_sz, |
362 | &block_size); | |
93641948 EA |
363 | trace_smmu_ptw_block_pte(stage, level, baseaddr, |
364 | pte_addr, pte, iova, gpa, | |
365 | block_size >> 20); | |
93641948 | 366 | } |
1733837d | 367 | ap = PTE_AP(pte); |
93641948 EA |
368 | if (is_permission_fault(ap, perm)) { |
369 | info->type = SMMU_PTW_ERR_PERMISSION; | |
370 | goto error; | |
371 | } | |
93641948 | 372 | |
9e54dee7 EA |
373 | tlbe->entry.translated_addr = gpa; |
374 | tlbe->entry.iova = iova & ~mask; | |
375 | tlbe->entry.addr_mask = mask; | |
a7550158 EA |
376 | tlbe->entry.perm = PTE_AP_TO_PERM(ap); |
377 | tlbe->level = level; | |
378 | tlbe->granule = granule_sz; | |
1733837d EA |
379 | return 0; |
380 | } | |
93641948 EA |
381 | info->type = SMMU_PTW_ERR_TRANSLATION; |
382 | ||
383 | error: | |
bcc919e7 | 384 | info->stage = 1; |
a7550158 | 385 | tlbe->entry.perm = IOMMU_NONE; |
93641948 EA |
386 | return -EINVAL; |
387 | } | |
388 | ||
e703f707 MS |
389 | /** |
390 | * smmu_ptw_64_s2 - VMSAv8-64 Walk of the page tables for a given ipa | |
391 | * for stage-2. | |
392 | * @cfg: translation config | |
393 | * @ipa: ipa to translate | |
394 | * @perm: access type | |
395 | * @tlbe: SMMUTLBEntry (out) | |
396 | * @info: handle to an error info | |
397 | * | |
398 | * Return 0 on success, < 0 on error. In case of error, @info is filled | |
399 | * and tlbe->perm is set to IOMMU_NONE. | |
400 | * Upon success, @tlbe is filled with translated_addr and entry | |
401 | * permission rights. | |
402 | */ | |
403 | static int smmu_ptw_64_s2(SMMUTransCfg *cfg, | |
404 | dma_addr_t ipa, IOMMUAccessFlags perm, | |
405 | SMMUTLBEntry *tlbe, SMMUPTWEventInfo *info) | |
406 | { | |
407 | const int stage = 2; | |
408 | int granule_sz = cfg->s2cfg.granule_sz; | |
409 | /* ARM DDI0487I.a: Table D8-7. */ | |
410 | int inputsize = 64 - cfg->s2cfg.tsz; | |
411 | int level = get_start_level(cfg->s2cfg.sl0, granule_sz); | |
412 | int stride = VMSA_STRIDE(granule_sz); | |
413 | int idx = pgd_concat_idx(level, granule_sz, ipa); | |
414 | /* | |
415 | * Get the ttb from concatenated structure. | |
416 | * The offset is the idx * size of each ttb(number of ptes * (sizeof(pte)) | |
417 | */ | |
418 | uint64_t baseaddr = extract64(cfg->s2cfg.vttb, 0, 48) + (1 << stride) * | |
419 | idx * sizeof(uint64_t); | |
420 | dma_addr_t indexmask = VMSA_IDXMSK(inputsize, stride, level); | |
421 | ||
422 | baseaddr &= ~indexmask; | |
423 | ||
424 | /* | |
425 | * On input, a stage 2 Translation fault occurs if the IPA is outside the | |
426 | * range configured by the relevant S2T0SZ field of the STE. | |
427 | */ | |
428 | if (ipa >= (1ULL << inputsize)) { | |
429 | info->type = SMMU_PTW_ERR_TRANSLATION; | |
430 | goto error; | |
431 | } | |
432 | ||
433 | while (level < VMSA_LEVELS) { | |
434 | uint64_t subpage_size = 1ULL << level_shift(level, granule_sz); | |
435 | uint64_t mask = subpage_size - 1; | |
436 | uint32_t offset = iova_level_offset(ipa, inputsize, level, granule_sz); | |
437 | uint64_t pte, gpa; | |
438 | dma_addr_t pte_addr = baseaddr + offset * sizeof(pte); | |
439 | uint8_t s2ap; | |
440 | ||
441 | if (get_pte(baseaddr, offset, &pte, info)) { | |
442 | goto error; | |
443 | } | |
444 | trace_smmu_ptw_level(stage, level, ipa, subpage_size, | |
445 | baseaddr, offset, pte); | |
446 | if (is_invalid_pte(pte) || is_reserved_pte(pte, level)) { | |
447 | trace_smmu_ptw_invalid_pte(stage, level, baseaddr, | |
448 | pte_addr, offset, pte); | |
449 | break; | |
450 | } | |
451 | ||
452 | if (is_table_pte(pte, level)) { | |
453 | baseaddr = get_table_pte_address(pte, granule_sz); | |
454 | level++; | |
455 | continue; | |
456 | } else if (is_page_pte(pte, level)) { | |
457 | gpa = get_page_pte_address(pte, granule_sz); | |
458 | trace_smmu_ptw_page_pte(stage, level, ipa, | |
459 | baseaddr, pte_addr, pte, gpa); | |
460 | } else { | |
461 | uint64_t block_size; | |
462 | ||
463 | gpa = get_block_pte_address(pte, level, granule_sz, | |
464 | &block_size); | |
465 | trace_smmu_ptw_block_pte(stage, level, baseaddr, | |
466 | pte_addr, pte, ipa, gpa, | |
467 | block_size >> 20); | |
468 | } | |
469 | ||
470 | /* | |
471 | * If S2AFFD and PTE.AF are 0 => fault. (5.2. Stream Table Entry) | |
472 | * An Access fault takes priority over a Permission fault. | |
473 | */ | |
474 | if (!PTE_AF(pte) && !cfg->s2cfg.affd) { | |
475 | info->type = SMMU_PTW_ERR_ACCESS; | |
476 | goto error; | |
477 | } | |
478 | ||
479 | s2ap = PTE_AP(pte); | |
480 | if (is_permission_fault_s2(s2ap, perm)) { | |
481 | info->type = SMMU_PTW_ERR_PERMISSION; | |
482 | goto error; | |
483 | } | |
484 | ||
485 | /* | |
486 | * The address output from the translation causes a stage 2 Address | |
487 | * Size fault if it exceeds the effective PA output range. | |
488 | */ | |
489 | if (gpa >= (1ULL << cfg->s2cfg.eff_ps)) { | |
490 | info->type = SMMU_PTW_ERR_ADDR_SIZE; | |
491 | goto error; | |
492 | } | |
493 | ||
494 | tlbe->entry.translated_addr = gpa; | |
495 | tlbe->entry.iova = ipa & ~mask; | |
496 | tlbe->entry.addr_mask = mask; | |
497 | tlbe->entry.perm = s2ap; | |
498 | tlbe->level = level; | |
499 | tlbe->granule = granule_sz; | |
500 | return 0; | |
501 | } | |
502 | info->type = SMMU_PTW_ERR_TRANSLATION; | |
503 | ||
504 | error: | |
505 | info->stage = 2; | |
506 | tlbe->entry.perm = IOMMU_NONE; | |
507 | return -EINVAL; | |
508 | } | |
509 | ||
93641948 EA |
510 | /** |
511 | * smmu_ptw - Walk the page tables for an IOVA, according to @cfg | |
512 | * | |
513 | * @cfg: translation configuration | |
514 | * @iova: iova to translate | |
515 | * @perm: tentative access type | |
516 | * @tlbe: returned entry | |
517 | * @info: ptw event handle | |
518 | * | |
519 | * return 0 on success | |
520 | */ | |
9de9fa5c PMD |
521 | int smmu_ptw(SMMUTransCfg *cfg, dma_addr_t iova, IOMMUAccessFlags perm, |
522 | SMMUTLBEntry *tlbe, SMMUPTWEventInfo *info) | |
93641948 | 523 | { |
e703f707 MS |
524 | if (cfg->stage == 1) { |
525 | return smmu_ptw_64_s1(cfg, iova, perm, tlbe, info); | |
526 | } else if (cfg->stage == 2) { | |
527 | /* | |
528 | * If bypassing stage 1(or unimplemented), the input address is passed | |
529 | * directly to stage 2 as IPA. If the input address of a transaction | |
530 | * exceeds the size of the IAS, a stage 1 Address Size fault occurs. | |
531 | * For AA64, IAS = OAS according to (IHI 0070.E.a) "3.4 Address sizes" | |
532 | */ | |
533 | if (iova >= (1ULL << cfg->oas)) { | |
534 | info->type = SMMU_PTW_ERR_ADDR_SIZE; | |
535 | info->stage = 1; | |
536 | tlbe->entry.perm = IOMMU_NONE; | |
537 | return -EINVAL; | |
538 | } | |
539 | ||
540 | return smmu_ptw_64_s2(cfg, iova, perm, tlbe, info); | |
541 | } | |
542 | ||
543 | g_assert_not_reached(); | |
93641948 | 544 | } |
527773ee | 545 | |
cac994ef EA |
546 | /** |
547 | * The bus number is used for lookup when SID based invalidation occurs. | |
548 | * In that case we lazily populate the SMMUPciBus array from the bus hash | |
549 | * table. At the time the SMMUPciBus is created (smmu_find_add_as), the bus | |
550 | * numbers may not be always initialized yet. | |
551 | */ | |
552 | SMMUPciBus *smmu_find_smmu_pcibus(SMMUState *s, uint8_t bus_num) | |
553 | { | |
554 | SMMUPciBus *smmu_pci_bus = s->smmu_pcibus_by_bus_num[bus_num]; | |
5ca0e6fe | 555 | GHashTableIter iter; |
cac994ef | 556 | |
5ca0e6fe PMD |
557 | if (smmu_pci_bus) { |
558 | return smmu_pci_bus; | |
559 | } | |
cac994ef | 560 | |
5ca0e6fe PMD |
561 | g_hash_table_iter_init(&iter, s->smmu_pcibus_by_busptr); |
562 | while (g_hash_table_iter_next(&iter, NULL, (void **)&smmu_pci_bus)) { | |
563 | if (pci_bus_num(smmu_pci_bus->bus) == bus_num) { | |
564 | s->smmu_pcibus_by_bus_num[bus_num] = smmu_pci_bus; | |
565 | return smmu_pci_bus; | |
cac994ef EA |
566 | } |
567 | } | |
5ca0e6fe PMD |
568 | |
569 | return NULL; | |
cac994ef EA |
570 | } |
571 | ||
572 | static AddressSpace *smmu_find_add_as(PCIBus *bus, void *opaque, int devfn) | |
573 | { | |
574 | SMMUState *s = opaque; | |
575 | SMMUPciBus *sbus = g_hash_table_lookup(s->smmu_pcibus_by_busptr, bus); | |
576 | SMMUDevice *sdev; | |
6ce9297b | 577 | static unsigned int index; |
cac994ef EA |
578 | |
579 | if (!sbus) { | |
580 | sbus = g_malloc0(sizeof(SMMUPciBus) + | |
581 | sizeof(SMMUDevice *) * SMMU_PCI_DEVFN_MAX); | |
582 | sbus->bus = bus; | |
583 | g_hash_table_insert(s->smmu_pcibus_by_busptr, bus, sbus); | |
584 | } | |
585 | ||
586 | sdev = sbus->pbdev[devfn]; | |
587 | if (!sdev) { | |
6ce9297b EA |
588 | char *name = g_strdup_printf("%s-%d-%d", s->mrtypename, devfn, index++); |
589 | ||
cac994ef EA |
590 | sdev = sbus->pbdev[devfn] = g_new0(SMMUDevice, 1); |
591 | ||
592 | sdev->smmu = s; | |
593 | sdev->bus = bus; | |
594 | sdev->devfn = devfn; | |
595 | ||
596 | memory_region_init_iommu(&sdev->iommu, sizeof(sdev->iommu), | |
597 | s->mrtypename, | |
ca3fbed8 | 598 | OBJECT(s), name, UINT64_MAX); |
cac994ef EA |
599 | address_space_init(&sdev->as, |
600 | MEMORY_REGION(&sdev->iommu), name); | |
601 | trace_smmu_add_mr(name); | |
602 | g_free(name); | |
603 | } | |
604 | ||
605 | return &sdev->as; | |
606 | } | |
607 | ||
32cfd7f3 EA |
608 | IOMMUMemoryRegion *smmu_iommu_mr(SMMUState *s, uint32_t sid) |
609 | { | |
610 | uint8_t bus_n, devfn; | |
611 | SMMUPciBus *smmu_bus; | |
612 | SMMUDevice *smmu; | |
613 | ||
614 | bus_n = PCI_BUS_NUM(sid); | |
615 | smmu_bus = smmu_find_smmu_pcibus(s, bus_n); | |
616 | if (smmu_bus) { | |
b78aae9b | 617 | devfn = SMMU_PCI_DEVFN(sid); |
32cfd7f3 EA |
618 | smmu = smmu_bus->pbdev[devfn]; |
619 | if (smmu) { | |
620 | return &smmu->iommu; | |
621 | } | |
622 | } | |
623 | return NULL; | |
624 | } | |
625 | ||
832e4222 | 626 | /* Unmap all notifiers attached to @mr */ |
1e793dd6 | 627 | static void smmu_inv_notifiers_mr(IOMMUMemoryRegion *mr) |
832e4222 EA |
628 | { |
629 | IOMMUNotifier *n; | |
630 | ||
631 | trace_smmu_inv_notifiers_mr(mr->parent_obj.name); | |
632 | IOMMU_NOTIFIER_FOREACH(n, mr) { | |
98332f64 | 633 | memory_region_unmap_iommu_notifier_range(n); |
832e4222 EA |
634 | } |
635 | } | |
636 | ||
637 | /* Unmap all notifiers of all mr's */ | |
638 | void smmu_inv_notifiers_all(SMMUState *s) | |
639 | { | |
c6370441 | 640 | SMMUDevice *sdev; |
832e4222 | 641 | |
c6370441 EA |
642 | QLIST_FOREACH(sdev, &s->devices_with_notifiers, next) { |
643 | smmu_inv_notifiers_mr(&sdev->iommu); | |
832e4222 EA |
644 | } |
645 | } | |
646 | ||
527773ee EA |
647 | static void smmu_base_realize(DeviceState *dev, Error **errp) |
648 | { | |
cac994ef | 649 | SMMUState *s = ARM_SMMU(dev); |
527773ee EA |
650 | SMMUBaseClass *sbc = ARM_SMMU_GET_CLASS(dev); |
651 | Error *local_err = NULL; | |
652 | ||
653 | sbc->parent_realize(dev, &local_err); | |
654 | if (local_err) { | |
655 | error_propagate(errp, local_err); | |
656 | return; | |
657 | } | |
32cfd7f3 | 658 | s->configs = g_hash_table_new_full(NULL, NULL, NULL, g_free); |
cc27ed81 EA |
659 | s->iotlb = g_hash_table_new_full(smmu_iotlb_key_hash, smmu_iotlb_key_equal, |
660 | g_free, g_free); | |
cac994ef EA |
661 | s->smmu_pcibus_by_busptr = g_hash_table_new(NULL, NULL); |
662 | ||
663 | if (s->primary_bus) { | |
664 | pci_setup_iommu(s->primary_bus, smmu_find_add_as, s); | |
665 | } else { | |
666 | error_setg(errp, "SMMU is not attached to any PCI bus!"); | |
667 | } | |
527773ee EA |
668 | } |
669 | ||
3c1a7c41 | 670 | static void smmu_base_reset_hold(Object *obj) |
527773ee | 671 | { |
3c1a7c41 | 672 | SMMUState *s = ARM_SMMU(obj); |
32cfd7f3 EA |
673 | |
674 | g_hash_table_remove_all(s->configs); | |
cc27ed81 | 675 | g_hash_table_remove_all(s->iotlb); |
527773ee EA |
676 | } |
677 | ||
678 | static Property smmu_dev_properties[] = { | |
679 | DEFINE_PROP_UINT8("bus_num", SMMUState, bus_num, 0), | |
c45e7619 PMD |
680 | DEFINE_PROP_LINK("primary-bus", SMMUState, primary_bus, |
681 | TYPE_PCI_BUS, PCIBus *), | |
527773ee EA |
682 | DEFINE_PROP_END_OF_LIST(), |
683 | }; | |
684 | ||
685 | static void smmu_base_class_init(ObjectClass *klass, void *data) | |
686 | { | |
687 | DeviceClass *dc = DEVICE_CLASS(klass); | |
3c1a7c41 | 688 | ResettableClass *rc = RESETTABLE_CLASS(klass); |
527773ee EA |
689 | SMMUBaseClass *sbc = ARM_SMMU_CLASS(klass); |
690 | ||
4f67d30b | 691 | device_class_set_props(dc, smmu_dev_properties); |
527773ee EA |
692 | device_class_set_parent_realize(dc, smmu_base_realize, |
693 | &sbc->parent_realize); | |
3c1a7c41 | 694 | rc->phases.hold = smmu_base_reset_hold; |
527773ee EA |
695 | } |
696 | ||
697 | static const TypeInfo smmu_base_info = { | |
698 | .name = TYPE_ARM_SMMU, | |
699 | .parent = TYPE_SYS_BUS_DEVICE, | |
700 | .instance_size = sizeof(SMMUState), | |
701 | .class_data = NULL, | |
702 | .class_size = sizeof(SMMUBaseClass), | |
703 | .class_init = smmu_base_class_init, | |
704 | .abstract = true, | |
705 | }; | |
706 | ||
707 | static void smmu_base_register_types(void) | |
708 | { | |
709 | type_register_static(&smmu_base_info); | |
710 | } | |
711 | ||
712 | type_init(smmu_base_register_types) | |
713 |