]> git.proxmox.com Git - mirror_qemu.git/blame - hw/arm/smmuv3.c
projects / mirror_qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
hw/arm: Convert TYPE_ARM_SMMU to 3-phase reset
[mirror_qemu.git] / hw / arm / smmuv3.c
CommitLineData
10a83cb9
PM		 1/*
2 * Copyright (C) 2014-2016 Broadcom Corporation
3 * Copyright (c) 2017 Red Hat, Inc.
4 * Written by Prem Mallappa, Eric Auger
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, see <http://www.gnu.org/licenses/>.
17 */
18
19#include "qemu/osdep.h"
744a790e 20#include "qemu/bitops.h"
64552b6b 21#include "hw/irq.h"
10a83cb9 22#include "hw/sysbus.h"
d6454270 23#include "migration/vmstate.h"
10a83cb9
PM		 24#include "hw/qdev-core.h"
25#include "hw/pci/pci.h"
9122bea9 26#include "cpu.h"
10a83cb9
PM		 27#include "trace.h"
28#include "qemu/log.h"
29#include "qemu/error-report.h"
30#include "qapi/error.h"
31
32#include "hw/arm/smmuv3.h"
33#include "smmuv3-internal.h"
1194140b 34#include "smmu-internal.h"
10a83cb9 35
6a736033
EA		 36/**
37 * smmuv3_trigger_irq - pulse @irq if enabled and update
38 * GERROR register in case of GERROR interrupt
39 *
40 * @irq: irq type
41 * @gerror_mask: mask of gerrors to toggle (relevant if @irq is GERROR)
42 */
fae4be38
EA		 43static void smmuv3_trigger_irq(SMMUv3State *s, SMMUIrq irq,
44 uint32_t gerror_mask)
6a736033
EA		 45{
46
47 bool pulse = false;
48
49 switch (irq) {
50 case SMMU_IRQ_EVTQ:
51 pulse = smmuv3_eventq_irq_enabled(s);
52 break;
53 case SMMU_IRQ_PRIQ:
54 qemu_log_mask(LOG_UNIMP, "PRI not yet supported\n");
55 break;
56 case SMMU_IRQ_CMD_SYNC:
57 pulse = true;
58 break;
59 case SMMU_IRQ_GERROR:
60 {
61 uint32_t pending = s->gerror ^ s->gerrorn;
62 uint32_t new_gerrors = ~pending & gerror_mask;
63
64 if (!new_gerrors) {
65 /* only toggle non pending errors */
66 return;
67 }
68 s->gerror ^= new_gerrors;
69 trace_smmuv3_write_gerror(new_gerrors, s->gerror);
70
71 pulse = smmuv3_gerror_irq_enabled(s);
72 break;
73 }
74 }
75 if (pulse) {
76 trace_smmuv3_trigger_irq(irq);
77 qemu_irq_pulse(s->irq[irq]);
78 }
79}
80
fae4be38 81static void smmuv3_write_gerrorn(SMMUv3State *s, uint32_t new_gerrorn)
6a736033
EA		 82{
83 uint32_t pending = s->gerror ^ s->gerrorn;
84 uint32_t toggled = s->gerrorn ^ new_gerrorn;
85
86 if (toggled & ~pending) {
87 qemu_log_mask(LOG_GUEST_ERROR,
88 "guest toggles non pending errors = 0x%x\n",
89 toggled & ~pending);
90 }
91
92 /*
93 * We do not raise any error in case guest toggles bits corresponding
94 * to not active IRQs (CONSTRAINED UNPREDICTABLE)
95 */
96 s->gerrorn = new_gerrorn;
97
98 trace_smmuv3_write_gerrorn(toggled & pending, s->gerrorn);
99}
100
dadd1a08
EA		 101static inline MemTxResult queue_read(SMMUQueue *q, void *data)
102{
103 dma_addr_t addr = Q_CONS_ENTRY(q);
104
ba06fe8a
PMD		 105 return dma_memory_read(&address_space_memory, addr, data, q->entry_size,
106 MEMTXATTRS_UNSPECIFIED);
dadd1a08
EA		 107}
108
109static MemTxResult queue_write(SMMUQueue *q, void *data)
110{
111 dma_addr_t addr = Q_PROD_ENTRY(q);
112 MemTxResult ret;
113
ba06fe8a
PMD		 114 ret = dma_memory_write(&address_space_memory, addr, data, q->entry_size,
115 MEMTXATTRS_UNSPECIFIED);
dadd1a08
EA		 116 if (ret != MEMTX_OK) {
117 return ret;
118 }
119
120 queue_prod_incr(q);
121 return MEMTX_OK;
122}
123
bb981004 124static MemTxResult smmuv3_write_eventq(SMMUv3State *s, Evt *evt)
dadd1a08
EA		 125{
126 SMMUQueue *q = &s->eventq;
bb981004 127 MemTxResult r;
dadd1a08
EA		 128
129 if (!smmuv3_eventq_enabled(s)) {
bb981004 130 return MEMTX_ERROR;
dadd1a08
EA		 131 }
132
133 if (smmuv3_q_full(q)) {
bb981004 134 return MEMTX_ERROR;
dadd1a08
EA		 135 }
136
bb981004
EA		 137 r = queue_write(q, evt);
138 if (r != MEMTX_OK) {
139 return r;
140 }
dadd1a08 141
9f4d2a13 142 if (!smmuv3_q_empty(q)) {
dadd1a08
EA		 143 smmuv3_trigger_irq(s, SMMU_IRQ_EVTQ, 0);
144 }
bb981004
EA		 145 return MEMTX_OK;
146}
147
148void smmuv3_record_event(SMMUv3State *s, SMMUEventInfo *info)
149{
24af32e0 150 Evt evt = {};
bb981004
EA		 151 MemTxResult r;
152
153 if (!smmuv3_eventq_enabled(s)) {
154 return;
155 }
156
157 EVT_SET_TYPE(&evt, info->type);
158 EVT_SET_SID(&evt, info->sid);
159
160 switch (info->type) {
9122bea9 161 case SMMU_EVT_NONE:
bb981004
EA		 162 return;
163 case SMMU_EVT_F_UUT:
164 EVT_SET_SSID(&evt, info->u.f_uut.ssid);
165 EVT_SET_SSV(&evt, info->u.f_uut.ssv);
166 EVT_SET_ADDR(&evt, info->u.f_uut.addr);
167 EVT_SET_RNW(&evt, info->u.f_uut.rnw);
168 EVT_SET_PNU(&evt, info->u.f_uut.pnu);
169 EVT_SET_IND(&evt, info->u.f_uut.ind);
170 break;
171 case SMMU_EVT_C_BAD_STREAMID:
172 EVT_SET_SSID(&evt, info->u.c_bad_streamid.ssid);
173 EVT_SET_SSV(&evt, info->u.c_bad_streamid.ssv);
174 break;
175 case SMMU_EVT_F_STE_FETCH:
176 EVT_SET_SSID(&evt, info->u.f_ste_fetch.ssid);
177 EVT_SET_SSV(&evt, info->u.f_ste_fetch.ssv);
b255cafb 178 EVT_SET_ADDR2(&evt, info->u.f_ste_fetch.addr);
bb981004
EA		 179 break;
180 case SMMU_EVT_C_BAD_STE:
181 EVT_SET_SSID(&evt, info->u.c_bad_ste.ssid);
182 EVT_SET_SSV(&evt, info->u.c_bad_ste.ssv);
183 break;
184 case SMMU_EVT_F_STREAM_DISABLED:
185 break;
186 case SMMU_EVT_F_TRANS_FORBIDDEN:
187 EVT_SET_ADDR(&evt, info->u.f_transl_forbidden.addr);
188 EVT_SET_RNW(&evt, info->u.f_transl_forbidden.rnw);
189 break;
190 case SMMU_EVT_C_BAD_SUBSTREAMID:
191 EVT_SET_SSID(&evt, info->u.c_bad_substream.ssid);
192 break;
193 case SMMU_EVT_F_CD_FETCH:
194 EVT_SET_SSID(&evt, info->u.f_cd_fetch.ssid);
195 EVT_SET_SSV(&evt, info->u.f_cd_fetch.ssv);
196 EVT_SET_ADDR(&evt, info->u.f_cd_fetch.addr);
197 break;
198 case SMMU_EVT_C_BAD_CD:
199 EVT_SET_SSID(&evt, info->u.c_bad_cd.ssid);
200 EVT_SET_SSV(&evt, info->u.c_bad_cd.ssv);
201 break;
202 case SMMU_EVT_F_WALK_EABT:
203 case SMMU_EVT_F_TRANSLATION:
204 case SMMU_EVT_F_ADDR_SIZE:
205 case SMMU_EVT_F_ACCESS:
206 case SMMU_EVT_F_PERMISSION:
207 EVT_SET_STALL(&evt, info->u.f_walk_eabt.stall);
208 EVT_SET_STAG(&evt, info->u.f_walk_eabt.stag);
209 EVT_SET_SSID(&evt, info->u.f_walk_eabt.ssid);
210 EVT_SET_SSV(&evt, info->u.f_walk_eabt.ssv);
211 EVT_SET_S2(&evt, info->u.f_walk_eabt.s2);
212 EVT_SET_ADDR(&evt, info->u.f_walk_eabt.addr);
213 EVT_SET_RNW(&evt, info->u.f_walk_eabt.rnw);
214 EVT_SET_PNU(&evt, info->u.f_walk_eabt.pnu);
215 EVT_SET_IND(&evt, info->u.f_walk_eabt.ind);
216 EVT_SET_CLASS(&evt, info->u.f_walk_eabt.class);
217 EVT_SET_ADDR2(&evt, info->u.f_walk_eabt.addr2);
218 break;
219 case SMMU_EVT_F_CFG_CONFLICT:
220 EVT_SET_SSID(&evt, info->u.f_cfg_conflict.ssid);
221 EVT_SET_SSV(&evt, info->u.f_cfg_conflict.ssv);
222 break;
223 /* rest is not implemented */
224 case SMMU_EVT_F_BAD_ATS_TREQ:
225 case SMMU_EVT_F_TLB_CONFLICT:
226 case SMMU_EVT_E_PAGE_REQ:
227 default:
228 g_assert_not_reached();
229 }
230
231 trace_smmuv3_record_event(smmu_event_string(info->type), info->sid);
232 r = smmuv3_write_eventq(s, &evt);
233 if (r != MEMTX_OK) {
234 smmuv3_trigger_irq(s, SMMU_IRQ_GERROR, R_GERROR_EVENTQ_ABT_ERR_MASK);
235 }
236 info->recorded = true;
dadd1a08
EA		 237}
238
10a83cb9
PM		 239static void smmuv3_init_regs(SMMUv3State *s)
240{
241 /**
242 * IDR0: stage1 only, AArch64 only, coherent access, 16b ASID,
243 * multi-level stream table
244 */
245 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, S1P, 1); /* stage 1 supported */
246 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, TTF, 2); /* AArch64 PTW only */
247 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, COHACC, 1); /* IO coherent */
248 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, ASID16, 1); /* 16-bit ASID */
249 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, TTENDIAN, 2); /* little endian */
250 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, STALL_MODEL, 1); /* No stall */
251 /* terminated transaction will always be aborted/error returned */
252 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, TERM_MODEL, 1);
253 /* 2-level stream table supported */
254 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, STLEVEL, 1);
255
256 s->idr[1] = FIELD_DP32(s->idr[1], IDR1, SIDSIZE, SMMU_IDR1_SIDSIZE);
257 s->idr[1] = FIELD_DP32(s->idr[1], IDR1, EVENTQS, SMMU_EVENTQS);
258 s->idr[1] = FIELD_DP32(s->idr[1], IDR1, CMDQS, SMMU_CMDQS);
259
de206dfd 260 s->idr[3] = FIELD_DP32(s->idr[3], IDR3, RIL, 1);
e7c3b9d9 261 s->idr[3] = FIELD_DP32(s->idr[3], IDR3, HAD, 1);
f8e7163d 262 s->idr[3] = FIELD_DP32(s->idr[3], IDR3, BBML, 2);
e7c3b9d9 263
bf559ee4 264 /* 4K, 16K and 64K granule support */
10a83cb9 265 s->idr[5] = FIELD_DP32(s->idr[5], IDR5, GRAN4K, 1);
bf559ee4 266 s->idr[5] = FIELD_DP32(s->idr[5], IDR5, GRAN16K, 1);
10a83cb9
PM		 267 s->idr[5] = FIELD_DP32(s->idr[5], IDR5, GRAN64K, 1);
268 s->idr[5] = FIELD_DP32(s->idr[5], IDR5, OAS, SMMU_IDR5_OAS); /* 44 bits */
269
270 s->cmdq.base = deposit64(s->cmdq.base, 0, 5, SMMU_CMDQS);
271 s->cmdq.prod = 0;
272 s->cmdq.cons = 0;
273 s->cmdq.entry_size = sizeof(struct Cmd);
274 s->eventq.base = deposit64(s->eventq.base, 0, 5, SMMU_EVENTQS);
275 s->eventq.prod = 0;
276 s->eventq.cons = 0;
277 s->eventq.entry_size = sizeof(struct Evt);
278
279 s->features = 0;
280 s->sid_split = 0;
e7c3b9d9 281 s->aidr = 0x1;
43530095
EA		 282 s->cr[0] = 0;
283 s->cr0ack = 0;
284 s->irq_ctrl = 0;
285 s->gerror = 0;
286 s->gerrorn = 0;
287 s->statusr = 0;
10a83cb9
PM		 288}
289
9bde7f06
EA		 290static int smmu_get_ste(SMMUv3State *s, dma_addr_t addr, STE *buf,
291 SMMUEventInfo *event)
292{
293 int ret;
294
295 trace_smmuv3_get_ste(addr);
296 /* TODO: guarantee 64-bit single-copy atomicity */
ba06fe8a
PMD		 297 ret = dma_memory_read(&address_space_memory, addr, buf, sizeof(*buf),
298 MEMTXATTRS_UNSPECIFIED);
9bde7f06
EA		 299 if (ret != MEMTX_OK) {
300 qemu_log_mask(LOG_GUEST_ERROR,
301 "Cannot fetch pte at address=0x%"PRIx64"\n", addr);
302 event->type = SMMU_EVT_F_STE_FETCH;
303 event->u.f_ste_fetch.addr = addr;
304 return -EINVAL;
305 }
306 return 0;
307
308}
309
310/* @ssid > 0 not supported yet */
311static int smmu_get_cd(SMMUv3State *s, STE *ste, uint32_t ssid,
312 CD *buf, SMMUEventInfo *event)
313{
314 dma_addr_t addr = STE_CTXPTR(ste);
315 int ret;
316
317 trace_smmuv3_get_cd(addr);
318 /* TODO: guarantee 64-bit single-copy atomicity */
ba06fe8a
PMD		 319 ret = dma_memory_read(&address_space_memory, addr, buf, sizeof(*buf),
320 MEMTXATTRS_UNSPECIFIED);
9bde7f06
EA		 321 if (ret != MEMTX_OK) {
322 qemu_log_mask(LOG_GUEST_ERROR,
323 "Cannot fetch pte at address=0x%"PRIx64"\n", addr);
324 event->type = SMMU_EVT_F_CD_FETCH;
325 event->u.f_ste_fetch.addr = addr;
326 return -EINVAL;
327 }
328 return 0;
329}
330
9122bea9 331/* Returns < 0 in case of invalid STE, 0 otherwise */
9bde7f06
EA		 332static int decode_ste(SMMUv3State *s, SMMUTransCfg *cfg,
333 STE *ste, SMMUEventInfo *event)
334{
335 uint32_t config;
9bde7f06
EA		 336
337 if (!STE_VALID(ste)) {
3499ec08
EA		 338 if (!event->inval_ste_allowed) {
339 qemu_log_mask(LOG_GUEST_ERROR, "invalid STE\n");
340 }
9bde7f06
EA		 341 goto bad_ste;
342 }
343
344 config = STE_CONFIG(ste);
345
346 if (STE_CFG_ABORT(config)) {
9122bea9
JH		 347 cfg->aborted = true;
348 return 0;
9bde7f06
EA		 349 }
350
351 if (STE_CFG_BYPASS(config)) {
352 cfg->bypassed = true;
9122bea9 353 return 0;
9bde7f06
EA		 354 }
355
356 if (STE_CFG_S2_ENABLED(config)) {
357 qemu_log_mask(LOG_UNIMP, "SMMUv3 does not support stage 2 yet\n");
358 goto bad_ste;
359 }
360
361 if (STE_S1CDMAX(ste) != 0) {
362 qemu_log_mask(LOG_UNIMP,
363 "SMMUv3 does not support multiple context descriptors yet\n");
364 goto bad_ste;
365 }
366
367 if (STE_S1STALLD(ste)) {
368 qemu_log_mask(LOG_UNIMP,
369 "SMMUv3 S1 stalling fault model not allowed yet\n");
370 goto bad_ste;
371 }
372 return 0;
373
374bad_ste:
375 event->type = SMMU_EVT_C_BAD_STE;
376 return -EINVAL;
377}
378
379/**
380 * smmu_find_ste - Return the stream table entry associated
381 * to the sid
382 *
383 * @s: smmuv3 handle
384 * @sid: stream ID
385 * @ste: returned stream table entry
386 * @event: handle to an event info
387 *
388 * Supports linear and 2-level stream table
389 * Return 0 on success, -EINVAL otherwise
390 */
391static int smmu_find_ste(SMMUv3State *s, uint32_t sid, STE *ste,
392 SMMUEventInfo *event)
393{
41678c33 394 dma_addr_t addr, strtab_base;
05ff2fb8 395 uint32_t log2size;
41678c33 396 int strtab_size_shift;
9bde7f06
EA		 397 int ret;
398
399 trace_smmuv3_find_ste(sid, s->features, s->sid_split);
05ff2fb8
SV		 400 log2size = FIELD_EX32(s->strtab_base_cfg, STRTAB_BASE_CFG, LOG2SIZE);
401 /*
402 * Check SID range against both guest-configured and implementation limits
403 */
404 if (sid >= (1 << MIN(log2size, SMMU_IDR1_SIDSIZE))) {
9bde7f06
EA		 405 event->type = SMMU_EVT_C_BAD_STREAMID;
406 return -EINVAL;
407 }
408 if (s->features & SMMU_FEATURE_2LVL_STE) {
409 int l1_ste_offset, l2_ste_offset, max_l2_ste, span;
41678c33 410 dma_addr_t l1ptr, l2ptr;
9bde7f06
EA		 411 STEDesc l1std;
412
41678c33
SV		 413 /*
414 * Align strtab base address to table size. For this purpose, assume it
415 * is not bounded by SMMU_IDR1_SIDSIZE.
416 */
417 strtab_size_shift = MAX(5, (int)log2size - s->sid_split - 1 + 3);
418 strtab_base = s->strtab_base & SMMU_BASE_ADDR_MASK &
419 ~MAKE_64BIT_MASK(0, strtab_size_shift);
9bde7f06
EA		 420 l1_ste_offset = sid >> s->sid_split;
421 l2_ste_offset = sid & ((1 << s->sid_split) - 1);
422 l1ptr = (dma_addr_t)(strtab_base + l1_ste_offset * sizeof(l1std));
423 /* TODO: guarantee 64-bit single-copy atomicity */
18610bfd 424 ret = dma_memory_read(&address_space_memory, l1ptr, &l1std,
ba06fe8a 425 sizeof(l1std), MEMTXATTRS_UNSPECIFIED);
9bde7f06
EA		 426 if (ret != MEMTX_OK) {
427 qemu_log_mask(LOG_GUEST_ERROR,
428 "Could not read L1PTR at 0X%"PRIx64"\n", l1ptr);
429 event->type = SMMU_EVT_F_STE_FETCH;
430 event->u.f_ste_fetch.addr = l1ptr;
431 return -EINVAL;
432 }
433
434 span = L1STD_SPAN(&l1std);
435
436 if (!span) {
437 /* l2ptr is not valid */
3499ec08
EA		 438 if (!event->inval_ste_allowed) {
439 qemu_log_mask(LOG_GUEST_ERROR,
440 "invalid sid=%d (L1STD span=0)\n", sid);
441 }
9bde7f06
EA		 442 event->type = SMMU_EVT_C_BAD_STREAMID;
443 return -EINVAL;
444 }
445 max_l2_ste = (1 << span) - 1;
446 l2ptr = l1std_l2ptr(&l1std);
447 trace_smmuv3_find_ste_2lvl(s->strtab_base, l1ptr, l1_ste_offset,
448 l2ptr, l2_ste_offset, max_l2_ste);
449 if (l2_ste_offset > max_l2_ste) {
450 qemu_log_mask(LOG_GUEST_ERROR,
451 "l2_ste_offset=%d > max_l2_ste=%d\n",
452 l2_ste_offset, max_l2_ste);
453 event->type = SMMU_EVT_C_BAD_STE;
454 return -EINVAL;
455 }
456 addr = l2ptr + l2_ste_offset * sizeof(*ste);
457 } else {
41678c33
SV		 458 strtab_size_shift = log2size + 5;
459 strtab_base = s->strtab_base & SMMU_BASE_ADDR_MASK &
460 ~MAKE_64BIT_MASK(0, strtab_size_shift);
461 addr = strtab_base + sid * sizeof(*ste);
9bde7f06
EA		 462 }
463
464 if (smmu_get_ste(s, addr, ste, event)) {
465 return -EINVAL;
466 }
467
468 return 0;
469}
470
471static int decode_cd(SMMUTransCfg *cfg, CD *cd, SMMUEventInfo *event)
472{
473 int ret = -EINVAL;
474 int i;
475
476 if (!CD_VALID(cd) || !CD_AARCH64(cd)) {
477 goto bad_cd;
478 }
479 if (!CD_A(cd)) {
480 goto bad_cd; /* SMMU_IDR0.TERM_MODEL == 1 */
481 }
482 if (CD_S(cd)) {
483 goto bad_cd; /* !STE_SECURE && SMMU_IDR0.STALL_MODEL == 1 */
484 }
485 if (CD_HA(cd) || CD_HD(cd)) {
486 goto bad_cd; /* HTTU = 0 */
487 }
488
489 /* we support only those at the moment */
490 cfg->aa64 = true;
491 cfg->stage = 1;
492
493 cfg->oas = oas2bits(CD_IPS(cd));
494 cfg->oas = MIN(oas2bits(SMMU_IDR5_OAS), cfg->oas);
495 cfg->tbi = CD_TBI(cd);
496 cfg->asid = CD_ASID(cd);
497
498 trace_smmuv3_decode_cd(cfg->oas);
499
500 /* decode data dependent on TT */
501 for (i = 0; i <= 1; i++) {
502 int tg, tsz;
503 SMMUTransTableInfo *tt = &cfg->tt[i];
504
505 cfg->tt[i].disabled = CD_EPD(cd, i);
506 if (cfg->tt[i].disabled) {
507 continue;
508 }
509
510 tsz = CD_TSZ(cd, i);
511 if (tsz < 16 || tsz > 39) {
512 goto bad_cd;
513 }
514
515 tg = CD_TG(cd, i);
516 tt->granule_sz = tg2granule(tg, i);
bf559ee4
KJ		 517 if ((tt->granule_sz != 12 && tt->granule_sz != 14 &&
518 tt->granule_sz != 16) || CD_ENDI(cd)) {
9bde7f06
EA		 519 goto bad_cd;
520 }
521
522 tt->tsz = tsz;
523 tt->ttb = CD_TTB(cd, i);
524 if (tt->ttb & ~(MAKE_64BIT_MASK(0, cfg->oas))) {
525 goto bad_cd;
526 }
e7c3b9d9
EA		 527 tt->had = CD_HAD(cd, i);
528 trace_smmuv3_decode_cd_tt(i, tt->tsz, tt->ttb, tt->granule_sz, tt->had);
9bde7f06
EA		 529 }
530
ced71694 531 cfg->record_faults = CD_R(cd);
9bde7f06
EA		 532
533 return 0;
534
535bad_cd:
536 event->type = SMMU_EVT_C_BAD_CD;
537 return ret;
538}
539
540/**
541 * smmuv3_decode_config - Prepare the translation configuration
542 * for the @mr iommu region
543 * @mr: iommu memory region the translation config must be prepared for
544 * @cfg: output translation configuration which is populated through
545 * the different configuration decoding steps
546 * @event: must be zero'ed by the caller
547 *
9122bea9 548 * return < 0 in case of config decoding error (@event is filled
9bde7f06
EA		 549 * accordingly). Return 0 otherwise.
550 */
551static int smmuv3_decode_config(IOMMUMemoryRegion *mr, SMMUTransCfg *cfg,
552 SMMUEventInfo *event)
553{
554 SMMUDevice *sdev = container_of(mr, SMMUDevice, iommu);
555 uint32_t sid = smmu_get_sid(sdev);
556 SMMUv3State *s = sdev->smmu;
9122bea9 557 int ret;
9bde7f06
EA		 558 STE ste;
559 CD cd;
560
9122bea9
JH		 561 ret = smmu_find_ste(s, sid, &ste, event);
562 if (ret) {
9bde7f06
EA		 563 return ret;
564 }
565
9122bea9
JH		 566 ret = decode_ste(s, cfg, &ste, event);
567 if (ret) {
9bde7f06
EA		 568 return ret;
569 }
570
9122bea9
JH		 571 if (cfg->aborted || cfg->bypassed) {
572 return 0;
573 }
574
575 ret = smmu_get_cd(s, &ste, 0 /* ssid */, &cd, event);
576 if (ret) {
9bde7f06
EA		 577 return ret;
578 }
579
580 return decode_cd(cfg, &cd, event);
581}
582
32cfd7f3
EA		 583/**
584 * smmuv3_get_config - Look up for a cached copy of configuration data for
585 * @sdev and on cache miss performs a configuration structure decoding from
586 * guest RAM.
587 *
588 * @sdev: SMMUDevice handle
589 * @event: output event info
590 *
591 * The configuration cache contains data resulting from both STE and CD
592 * decoding under the form of an SMMUTransCfg struct. The hash table is indexed
593 * by the SMMUDevice handle.
594 */
595static SMMUTransCfg *smmuv3_get_config(SMMUDevice *sdev, SMMUEventInfo *event)
596{
597 SMMUv3State *s = sdev->smmu;
598 SMMUState *bc = &s->smmu_state;
599 SMMUTransCfg *cfg;
600
601 cfg = g_hash_table_lookup(bc->configs, sdev);
602 if (cfg) {
603 sdev->cfg_cache_hits++;
604 trace_smmuv3_config_cache_hit(smmu_get_sid(sdev),
605 sdev->cfg_cache_hits, sdev->cfg_cache_misses,
606 100 * sdev->cfg_cache_hits /
607 (sdev->cfg_cache_hits + sdev->cfg_cache_misses));
608 } else {
609 sdev->cfg_cache_misses++;
610 trace_smmuv3_config_cache_miss(smmu_get_sid(sdev),
611 sdev->cfg_cache_hits, sdev->cfg_cache_misses,
612 100 * sdev->cfg_cache_hits /
613 (sdev->cfg_cache_hits + sdev->cfg_cache_misses));
614 cfg = g_new0(SMMUTransCfg, 1);
615
616 if (!smmuv3_decode_config(&sdev->iommu, cfg, event)) {
617 g_hash_table_insert(bc->configs, sdev, cfg);
618 } else {
619 g_free(cfg);
620 cfg = NULL;
621 }
622 }
623 return cfg;
624}
625
626static void smmuv3_flush_config(SMMUDevice *sdev)
627{
628 SMMUv3State *s = sdev->smmu;
629 SMMUState *bc = &s->smmu_state;
630
631 trace_smmuv3_config_cache_inv(smmu_get_sid(sdev));
632 g_hash_table_remove(bc->configs, sdev);
633}
634
9bde7f06 635static IOMMUTLBEntry smmuv3_translate(IOMMUMemoryRegion *mr, hwaddr addr,
2c91bcf2 636 IOMMUAccessFlags flag, int iommu_idx)
9bde7f06
EA		 637{
638 SMMUDevice *sdev = container_of(mr, SMMUDevice, iommu);
639 SMMUv3State *s = sdev->smmu;
640 uint32_t sid = smmu_get_sid(sdev);
3499ec08
EA		 641 SMMUEventInfo event = {.type = SMMU_EVT_NONE,
642 .sid = sid,
643 .inval_ste_allowed = false};
9bde7f06 644 SMMUPTWEventInfo ptw_info = {};
9122bea9 645 SMMUTranslationStatus status;
cc27ed81
EA		 646 SMMUState *bs = ARM_SMMU(s);
647 uint64_t page_mask, aligned_addr;
a7550158 648 SMMUTLBEntry *cached_entry = NULL;
cc27ed81 649 SMMUTransTableInfo *tt;
32cfd7f3 650 SMMUTransCfg *cfg = NULL;
9bde7f06
EA		 651 IOMMUTLBEntry entry = {
652 .target_as = &address_space_memory,
653 .iova = addr,
654 .translated_addr = addr,
655 .addr_mask = ~(hwaddr)0,
656 .perm = IOMMU_NONE,
657 };
9bde7f06 658
32cfd7f3
EA		 659 qemu_mutex_lock(&s->mutex);
660
9bde7f06 661 if (!smmu_enabled(s)) {
9122bea9
JH		 662 status = SMMU_TRANS_DISABLE;
663 goto epilogue;
9bde7f06
EA		 664 }
665
32cfd7f3
EA		 666 cfg = smmuv3_get_config(sdev, &event);
667 if (!cfg) {
9122bea9
JH		 668 status = SMMU_TRANS_ERROR;
669 goto epilogue;
9bde7f06
EA		 670 }
671
32cfd7f3 672 if (cfg->aborted) {
9122bea9
JH		 673 status = SMMU_TRANS_ABORT;
674 goto epilogue;
9bde7f06
EA		 675 }
676
32cfd7f3 677 if (cfg->bypassed) {
9122bea9
JH		 678 status = SMMU_TRANS_BYPASS;
679 goto epilogue;
680 }
681
cc27ed81
EA		 682 tt = select_tt(cfg, addr);
683 if (!tt) {
ced71694 684 if (cfg->record_faults) {
cc27ed81
EA		 685 event.type = SMMU_EVT_F_TRANSLATION;
686 event.u.f_translation.addr = addr;
687 event.u.f_translation.rnw = flag & 0x1;
688 }
689 status = SMMU_TRANS_ERROR;
690 goto epilogue;
691 }
692
693 page_mask = (1ULL << (tt->granule_sz)) - 1;
694 aligned_addr = addr & ~page_mask;
695
9e54dee7 696 cached_entry = smmu_iotlb_lookup(bs, cfg, tt, aligned_addr);
cc27ed81 697 if (cached_entry) {
a7550158 698 if ((flag & IOMMU_WO) && !(cached_entry->entry.perm & IOMMU_WO)) {
cc27ed81 699 status = SMMU_TRANS_ERROR;
ced71694 700 if (cfg->record_faults) {
cc27ed81
EA		 701 event.type = SMMU_EVT_F_PERMISSION;
702 event.u.f_permission.addr = addr;
703 event.u.f_permission.rnw = flag & 0x1;
704 }
705 } else {
706 status = SMMU_TRANS_SUCCESS;
707 }
708 goto epilogue;
709 }
710
a7550158 711 cached_entry = g_new0(SMMUTLBEntry, 1);
cc27ed81
EA		 712
713 if (smmu_ptw(cfg, aligned_addr, flag, cached_entry, &ptw_info)) {
714 g_free(cached_entry);
9bde7f06
EA		 715 switch (ptw_info.type) {
716 case SMMU_PTW_ERR_WALK_EABT:
717 event.type = SMMU_EVT_F_WALK_EABT;
718 event.u.f_walk_eabt.addr = addr;
719 event.u.f_walk_eabt.rnw = flag & 0x1;
720 event.u.f_walk_eabt.class = 0x1;
721 event.u.f_walk_eabt.addr2 = ptw_info.addr;
722 break;
723 case SMMU_PTW_ERR_TRANSLATION:
ced71694 724 if (cfg->record_faults) {
9bde7f06
EA		 725 event.type = SMMU_EVT_F_TRANSLATION;
726 event.u.f_translation.addr = addr;
727 event.u.f_translation.rnw = flag & 0x1;
728 }
729 break;
730 case SMMU_PTW_ERR_ADDR_SIZE:
ced71694 731 if (cfg->record_faults) {
9bde7f06
EA		 732 event.type = SMMU_EVT_F_ADDR_SIZE;
733 event.u.f_addr_size.addr = addr;
734 event.u.f_addr_size.rnw = flag & 0x1;
735 }
736 break;
737 case SMMU_PTW_ERR_ACCESS:
ced71694 738 if (cfg->record_faults) {
9bde7f06
EA		 739 event.type = SMMU_EVT_F_ACCESS;
740 event.u.f_access.addr = addr;
741 event.u.f_access.rnw = flag & 0x1;
742 }
743 break;
744 case SMMU_PTW_ERR_PERMISSION:
ced71694 745 if (cfg->record_faults) {
9bde7f06
EA		 746 event.type = SMMU_EVT_F_PERMISSION;
747 event.u.f_permission.addr = addr;
748 event.u.f_permission.rnw = flag & 0x1;
749 }
750 break;
751 default:
752 g_assert_not_reached();
753 }
9122bea9
JH		 754 status = SMMU_TRANS_ERROR;
755 } else {
6808bca9 756 smmu_iotlb_insert(bs, cfg, cached_entry);
9122bea9 757 status = SMMU_TRANS_SUCCESS;
9bde7f06 758 }
9122bea9
JH		 759
760epilogue:
32cfd7f3 761 qemu_mutex_unlock(&s->mutex);
9122bea9
JH		 762 switch (status) {
763 case SMMU_TRANS_SUCCESS:
c3ca7d56 764 entry.perm = cached_entry->entry.perm;
a7550158 765 entry.translated_addr = cached_entry->entry.translated_addr +
9e54dee7 766 (addr & cached_entry->entry.addr_mask);
a7550158 767 entry.addr_mask = cached_entry->entry.addr_mask;
9122bea9
JH		 768 trace_smmuv3_translate_success(mr->parent_obj.name, sid, addr,
769 entry.translated_addr, entry.perm);
770 break;
771 case SMMU_TRANS_DISABLE:
772 entry.perm = flag;
773 entry.addr_mask = ~TARGET_PAGE_MASK;
774 trace_smmuv3_translate_disable(mr->parent_obj.name, sid, addr,
775 entry.perm);
776 break;
777 case SMMU_TRANS_BYPASS:
778 entry.perm = flag;
779 entry.addr_mask = ~TARGET_PAGE_MASK;
780 trace_smmuv3_translate_bypass(mr->parent_obj.name, sid, addr,
781 entry.perm);
782 break;
783 case SMMU_TRANS_ABORT:
784 /* no event is recorded on abort */
785 trace_smmuv3_translate_abort(mr->parent_obj.name, sid, addr,
786 entry.perm);
787 break;
788 case SMMU_TRANS_ERROR:
9bde7f06 789 qemu_log_mask(LOG_GUEST_ERROR,
264a3b2e 790 "%s translation failed for iova=0x%"PRIx64" (%s)\n",
9122bea9 791 mr->parent_obj.name, addr, smmu_event_string(event.type));
9bde7f06 792 smmuv3_record_event(s, &event);
9122bea9 793 break;
9bde7f06
EA		 794 }
795
796 return entry;
797}
798
832e4222
EA		 799/**
800 * smmuv3_notify_iova - call the notifier @n for a given
801 * @asid and @iova tuple.
802 *
803 * @mr: IOMMU mr region handle
804 * @n: notifier to be called
805 * @asid: address space ID or negative value if we don't care
806 * @iova: iova
d5291561
EA		 807 * @tg: translation granule (if communicated through range invalidation)
808 * @num_pages: number of @granule sized pages (if tg != 0), otherwise 1
832e4222
EA		 809 */
810static void smmuv3_notify_iova(IOMMUMemoryRegion *mr,
811 IOMMUNotifier *n,
d5291561
EA		 812 int asid, dma_addr_t iova,
813 uint8_t tg, uint64_t num_pages)
832e4222
EA		 814{
815 SMMUDevice *sdev = container_of(mr, SMMUDevice, iommu);
5039caf3 816 IOMMUTLBEvent event;
dcda883c 817 uint8_t granule;
832e4222 818
d5291561
EA		 819 if (!tg) {
820 SMMUEventInfo event = {.inval_ste_allowed = true};
821 SMMUTransCfg *cfg = smmuv3_get_config(sdev, &event);
822 SMMUTransTableInfo *tt;
832e4222 823
d5291561
EA		 824 if (!cfg) {
825 return;
826 }
832e4222 827
d5291561
EA		 828 if (asid >= 0 && cfg->asid != asid) {
829 return;
830 }
831
832 tt = select_tt(cfg, iova);
833 if (!tt) {
834 return;
835 }
836 granule = tt->granule_sz;
dcda883c
ZY		 837 } else {
838 granule = tg * 2 + 10;
832e4222
EA		 839 }
840
5039caf3
EP		 841 event.type = IOMMU_NOTIFIER_UNMAP;
842 event.entry.target_as = &address_space_memory;
843 event.entry.iova = iova;
844 event.entry.addr_mask = num_pages * (1 << granule) - 1;
845 event.entry.perm = IOMMU_NONE;
832e4222 846
5039caf3 847 memory_region_notify_iommu_one(n, &event);
832e4222
EA		 848}
849
d5291561
EA		 850/* invalidate an asid/iova range tuple in all mr's */
851static void smmuv3_inv_notifiers_iova(SMMUState *s, int asid, dma_addr_t iova,
852 uint8_t tg, uint64_t num_pages)
832e4222 853{
c6370441 854 SMMUDevice *sdev;
832e4222 855
c6370441
EA		 856 QLIST_FOREACH(sdev, &s->devices_with_notifiers, next) {
857 IOMMUMemoryRegion *mr = &sdev->iommu;
832e4222
EA		 858 IOMMUNotifier *n;
859
d5291561
EA		 860 trace_smmuv3_inv_notifiers_iova(mr->parent_obj.name, asid, iova,
861 tg, num_pages);
832e4222
EA		 862
863 IOMMU_NOTIFIER_FOREACH(n, mr) {
d5291561 864 smmuv3_notify_iova(mr, n, asid, iova, tg, num_pages);
832e4222
EA		 865 }
866 }
867}
868
c0f9ef70
EA		 869static void smmuv3_s1_range_inval(SMMUState *s, Cmd *cmd)
870{
219729cf 871 dma_addr_t end, addr = CMD_ADDR(cmd);
c0f9ef70
EA		 872 uint8_t type = CMD_TYPE(cmd);
873 uint16_t vmid = CMD_VMID(cmd);
219729cf
EA		 874 uint8_t scale = CMD_SCALE(cmd);
875 uint8_t num = CMD_NUM(cmd);
876 uint8_t ttl = CMD_TTL(cmd);
c0f9ef70 877 bool leaf = CMD_LEAF(cmd);
d5291561 878 uint8_t tg = CMD_TG(cmd);
219729cf
EA		 879 uint64_t num_pages;
880 uint8_t granule;
c0f9ef70
EA		 881 int asid = -1;
882
883 if (type == SMMU_CMD_TLBI_NH_VA) {
884 asid = CMD_ASID(cmd);
885 }
6d9cd115 886
219729cf
EA		 887 if (!tg) {
888 trace_smmuv3_s1_range_inval(vmid, asid, addr, tg, 1, ttl, leaf);
889 smmuv3_inv_notifiers_iova(s, asid, addr, tg, 1);
890 smmu_iotlb_inv_iova(s, asid, addr, tg, 1, ttl);
891 return;
892 }
893
894 /* RIL in use */
6d9cd115 895
219729cf
EA		 896 num_pages = (num + 1) * BIT_ULL(scale);
897 granule = tg * 2 + 10;
898
899 /* Split invalidations into ^2 range invalidations */
900 end = addr + (num_pages << granule) - 1;
6d9cd115 901
219729cf
EA		 902 while (addr != end + 1) {
903 uint64_t mask = dma_aligned_pow2_mask(addr, end, 64);
6d9cd115 904
219729cf
EA		 905 num_pages = (mask + 1) >> granule;
906 trace_smmuv3_s1_range_inval(vmid, asid, addr, tg, num_pages, ttl, leaf);
907 smmuv3_inv_notifiers_iova(s, asid, addr, tg, num_pages);
908 smmu_iotlb_inv_iova(s, asid, addr, tg, num_pages, ttl);
909 addr += mask + 1;
6d9cd115 910 }
c0f9ef70
EA		 911}
912
1194140b
EA		 913static gboolean
914smmuv3_invalidate_ste(gpointer key, gpointer value, gpointer user_data)
915{
916 SMMUDevice *sdev = (SMMUDevice *)key;
917 uint32_t sid = smmu_get_sid(sdev);
918 SMMUSIDRange *sid_range = (SMMUSIDRange *)user_data;
919
920 if (sid < sid_range->start || sid > sid_range->end) {
921 return false;
922 }
923 trace_smmuv3_config_cache_inv(sid);
924 return true;
925}
926
fae4be38 927static int smmuv3_cmdq_consume(SMMUv3State *s)
dadd1a08 928{
32cfd7f3 929 SMMUState *bs = ARM_SMMU(s);
dadd1a08
EA		 930 SMMUCmdError cmd_error = SMMU_CERROR_NONE;
931 SMMUQueue *q = &s->cmdq;
932 SMMUCommandType type = 0;
933
934 if (!smmuv3_cmdq_enabled(s)) {
935 return 0;
936 }
937 /*
938 * some commands depend on register values, typically CR0. In case those
939 * register values change while handling the command, spec says it
940 * is UNPREDICTABLE whether the command is interpreted under the new
941 * or old value.
942 */
943
944 while (!smmuv3_q_empty(q)) {
945 uint32_t pending = s->gerror ^ s->gerrorn;
946 Cmd cmd;
947
948 trace_smmuv3_cmdq_consume(Q_PROD(q), Q_CONS(q),
949 Q_PROD_WRAP(q), Q_CONS_WRAP(q));
950
951 if (FIELD_EX32(pending, GERROR, CMDQ_ERR)) {
952 break;
953 }
954
955 if (queue_read(q, &cmd) != MEMTX_OK) {
956 cmd_error = SMMU_CERROR_ABT;
957 break;
958 }
959
960 type = CMD_TYPE(&cmd);
961
962 trace_smmuv3_cmdq_opcode(smmu_cmd_string(type));
963
32cfd7f3 964 qemu_mutex_lock(&s->mutex);
dadd1a08
EA		 965 switch (type) {
966 case SMMU_CMD_SYNC:
967 if (CMD_SYNC_CS(&cmd) & CMD_SYNC_SIG_IRQ) {
968 smmuv3_trigger_irq(s, SMMU_IRQ_CMD_SYNC, 0);
969 }
970 break;
971 case SMMU_CMD_PREFETCH_CONFIG:
972 case SMMU_CMD_PREFETCH_ADDR:
32cfd7f3 973 break;
dadd1a08 974 case SMMU_CMD_CFGI_STE:
32cfd7f3
EA		 975 {
976 uint32_t sid = CMD_SID(&cmd);
977 IOMMUMemoryRegion *mr = smmu_iommu_mr(bs, sid);
978 SMMUDevice *sdev;
979
980 if (CMD_SSEC(&cmd)) {
981 cmd_error = SMMU_CERROR_ILL;
982 break;
983 }
984
985 if (!mr) {
986 break;
987 }
988
989 trace_smmuv3_cmdq_cfgi_ste(sid);
990 sdev = container_of(mr, SMMUDevice, iommu);
991 smmuv3_flush_config(sdev);
992
993 break;
994 }
dadd1a08 995 case SMMU_CMD_CFGI_STE_RANGE: /* same as SMMU_CMD_CFGI_ALL */
32cfd7f3 996 {
017a913a 997 uint32_t sid = CMD_SID(&cmd), mask;
32cfd7f3 998 uint8_t range = CMD_STE_RANGE(&cmd);
017a913a 999 SMMUSIDRange sid_range;
32cfd7f3
EA		 1000
1001 if (CMD_SSEC(&cmd)) {
1002 cmd_error = SMMU_CERROR_ILL;
1003 break;
1004 }
017a913a
ZY		 1005
1006 mask = (1ULL << (range + 1)) - 1;
1007 sid_range.start = sid & ~mask;
1008 sid_range.end = sid_range.start + mask;
1009
1010 trace_smmuv3_cmdq_cfgi_ste_range(sid_range.start, sid_range.end);
1194140b
EA		 1011 g_hash_table_foreach_remove(bs->configs, smmuv3_invalidate_ste,
1012 &sid_range);
32cfd7f3
EA		 1013 break;
1014 }
dadd1a08
EA		 1015 case SMMU_CMD_CFGI_CD:
1016 case SMMU_CMD_CFGI_CD_ALL:
32cfd7f3
EA		 1017 {
1018 uint32_t sid = CMD_SID(&cmd);
1019 IOMMUMemoryRegion *mr = smmu_iommu_mr(bs, sid);
1020 SMMUDevice *sdev;
1021
1022 if (CMD_SSEC(&cmd)) {
1023 cmd_error = SMMU_CERROR_ILL;
1024 break;
1025 }
1026
1027 if (!mr) {
1028 break;
1029 }
1030
1031 trace_smmuv3_cmdq_cfgi_cd(sid);
1032 sdev = container_of(mr, SMMUDevice, iommu);
1033 smmuv3_flush_config(sdev);
1034 break;
1035 }
dadd1a08 1036 case SMMU_CMD_TLBI_NH_ASID:
cc27ed81
EA		 1037 {
1038 uint16_t asid = CMD_ASID(&cmd);
1039
1040 trace_smmuv3_cmdq_tlbi_nh_asid(asid);
832e4222 1041 smmu_inv_notifiers_all(&s->smmu_state);
cc27ed81
EA		 1042 smmu_iotlb_inv_asid(bs, asid);
1043 break;
1044 }
1045 case SMMU_CMD_TLBI_NH_ALL:
1046 case SMMU_CMD_TLBI_NSNH_ALL:
1047 trace_smmuv3_cmdq_tlbi_nh();
832e4222 1048 smmu_inv_notifiers_all(&s->smmu_state);
cc27ed81
EA		 1049 smmu_iotlb_inv_all(bs);
1050 break;
dadd1a08 1051 case SMMU_CMD_TLBI_NH_VAA:
cc27ed81 1052 case SMMU_CMD_TLBI_NH_VA:
c0f9ef70 1053 smmuv3_s1_range_inval(bs, &cmd);
cc27ed81 1054 break;
dadd1a08
EA		 1055 case SMMU_CMD_TLBI_EL3_ALL:
1056 case SMMU_CMD_TLBI_EL3_VA:
1057 case SMMU_CMD_TLBI_EL2_ALL:
1058 case SMMU_CMD_TLBI_EL2_ASID:
1059 case SMMU_CMD_TLBI_EL2_VA:
1060 case SMMU_CMD_TLBI_EL2_VAA:
1061 case SMMU_CMD_TLBI_S12_VMALL:
1062 case SMMU_CMD_TLBI_S2_IPA:
dadd1a08
EA		 1063 case SMMU_CMD_ATC_INV:
1064 case SMMU_CMD_PRI_RESP:
1065 case SMMU_CMD_RESUME:
1066 case SMMU_CMD_STALL_TERM:
1067 trace_smmuv3_unhandled_cmd(type);
1068 break;
1069 default:
1070 cmd_error = SMMU_CERROR_ILL;
1071 qemu_log_mask(LOG_GUEST_ERROR,
1072 "Illegal command type: %d\n", CMD_TYPE(&cmd));
1073 break;
1074 }
32cfd7f3 1075 qemu_mutex_unlock(&s->mutex);
dadd1a08
EA		 1076 if (cmd_error) {
1077 break;
1078 }
1079 /*
1080 * We only increment the cons index after the completion of
1081 * the command. We do that because the SYNC returns immediately
1082 * and does not check the completion of previous commands
1083 */
1084 queue_cons_incr(q);
1085 }
1086
1087 if (cmd_error) {
1088 trace_smmuv3_cmdq_consume_error(smmu_cmd_string(type), cmd_error);
1089 smmu_write_cmdq_err(s, cmd_error);
1090 smmuv3_trigger_irq(s, SMMU_IRQ_GERROR, R_GERROR_CMDQ_ERR_MASK);
1091 }
1092
1093 trace_smmuv3_cmdq_consume_out(Q_PROD(q), Q_CONS(q),
1094 Q_PROD_WRAP(q), Q_CONS_WRAP(q));
1095
1096 return 0;
1097}
1098
fae4be38
EA		 1099static MemTxResult smmu_writell(SMMUv3State *s, hwaddr offset,
1100 uint64_t data, MemTxAttrs attrs)
1101{
1102 switch (offset) {
1103 case A_GERROR_IRQ_CFG0:
1104 s->gerror_irq_cfg0 = data;
1105 return MEMTX_OK;
1106 case A_STRTAB_BASE:
1107 s->strtab_base = data;
1108 return MEMTX_OK;
1109 case A_CMDQ_BASE:
1110 s->cmdq.base = data;
1111 s->cmdq.log2size = extract64(s->cmdq.base, 0, 5);
1112 if (s->cmdq.log2size > SMMU_CMDQS) {
1113 s->cmdq.log2size = SMMU_CMDQS;
1114 }
1115 return MEMTX_OK;
1116 case A_EVENTQ_BASE:
1117 s->eventq.base = data;
1118 s->eventq.log2size = extract64(s->eventq.base, 0, 5);
1119 if (s->eventq.log2size > SMMU_EVENTQS) {
1120 s->eventq.log2size = SMMU_EVENTQS;
1121 }
1122 return MEMTX_OK;
1123 case A_EVENTQ_IRQ_CFG0:
1124 s->eventq_irq_cfg0 = data;
1125 return MEMTX_OK;
1126 default:
1127 qemu_log_mask(LOG_UNIMP,
1128 "%s Unexpected 64-bit access to 0x%"PRIx64" (WI)\n",
1129 __func__, offset);
1130 return MEMTX_OK;
1131 }
1132}
1133
1134static MemTxResult smmu_writel(SMMUv3State *s, hwaddr offset,
1135 uint64_t data, MemTxAttrs attrs)
1136{
1137 switch (offset) {
1138 case A_CR0:
1139 s->cr[0] = data;
1140 s->cr0ack = data & ~SMMU_CR0_RESERVED;
1141 /* in case the command queue has been enabled */
1142 smmuv3_cmdq_consume(s);
1143 return MEMTX_OK;
1144 case A_CR1:
1145 s->cr[1] = data;
1146 return MEMTX_OK;
1147 case A_CR2:
1148 s->cr[2] = data;
1149 return MEMTX_OK;
1150 case A_IRQ_CTRL:
1151 s->irq_ctrl = data;
1152 return MEMTX_OK;
1153 case A_GERRORN:
1154 smmuv3_write_gerrorn(s, data);
1155 /*
1156 * By acknowledging the CMDQ_ERR, SW may notify cmds can
1157 * be processed again
1158 */
1159 smmuv3_cmdq_consume(s);
1160 return MEMTX_OK;
1161 case A_GERROR_IRQ_CFG0: /* 64b */
1162 s->gerror_irq_cfg0 = deposit64(s->gerror_irq_cfg0, 0, 32, data);
1163 return MEMTX_OK;
1164 case A_GERROR_IRQ_CFG0 + 4:
1165 s->gerror_irq_cfg0 = deposit64(s->gerror_irq_cfg0, 32, 32, data);
1166 return MEMTX_OK;
1167 case A_GERROR_IRQ_CFG1:
1168 s->gerror_irq_cfg1 = data;
1169 return MEMTX_OK;
1170 case A_GERROR_IRQ_CFG2:
1171 s->gerror_irq_cfg2 = data;
1172 return MEMTX_OK;
1173 case A_STRTAB_BASE: /* 64b */
1174 s->strtab_base = deposit64(s->strtab_base, 0, 32, data);
1175 return MEMTX_OK;
1176 case A_STRTAB_BASE + 4:
1177 s->strtab_base = deposit64(s->strtab_base, 32, 32, data);
1178 return MEMTX_OK;
1179 case A_STRTAB_BASE_CFG:
1180 s->strtab_base_cfg = data;
1181 if (FIELD_EX32(data, STRTAB_BASE_CFG, FMT) == 1) {
1182 s->sid_split = FIELD_EX32(data, STRTAB_BASE_CFG, SPLIT);
1183 s->features |= SMMU_FEATURE_2LVL_STE;
1184 }
1185 return MEMTX_OK;
1186 case A_CMDQ_BASE: /* 64b */
1187 s->cmdq.base = deposit64(s->cmdq.base, 0, 32, data);
1188 s->cmdq.log2size = extract64(s->cmdq.base, 0, 5);
1189 if (s->cmdq.log2size > SMMU_CMDQS) {
1190 s->cmdq.log2size = SMMU_CMDQS;
1191 }
1192 return MEMTX_OK;
1193 case A_CMDQ_BASE + 4: /* 64b */
1194 s->cmdq.base = deposit64(s->cmdq.base, 32, 32, data);
1195 return MEMTX_OK;
1196 case A_CMDQ_PROD:
1197 s->cmdq.prod = data;
1198 smmuv3_cmdq_consume(s);
1199 return MEMTX_OK;
1200 case A_CMDQ_CONS:
1201 s->cmdq.cons = data;
1202 return MEMTX_OK;
1203 case A_EVENTQ_BASE: /* 64b */
1204 s->eventq.base = deposit64(s->eventq.base, 0, 32, data);
1205 s->eventq.log2size = extract64(s->eventq.base, 0, 5);
1206 if (s->eventq.log2size > SMMU_EVENTQS) {
1207 s->eventq.log2size = SMMU_EVENTQS;
1208 }
1209 return MEMTX_OK;
1210 case A_EVENTQ_BASE + 4:
1211 s->eventq.base = deposit64(s->eventq.base, 32, 32, data);
1212 return MEMTX_OK;
1213 case A_EVENTQ_PROD:
1214 s->eventq.prod = data;
1215 return MEMTX_OK;
1216 case A_EVENTQ_CONS:
1217 s->eventq.cons = data;
1218 return MEMTX_OK;
1219 case A_EVENTQ_IRQ_CFG0: /* 64b */
1220 s->eventq_irq_cfg0 = deposit64(s->eventq_irq_cfg0, 0, 32, data);
1221 return MEMTX_OK;
1222 case A_EVENTQ_IRQ_CFG0 + 4:
1223 s->eventq_irq_cfg0 = deposit64(s->eventq_irq_cfg0, 32, 32, data);
1224 return MEMTX_OK;
1225 case A_EVENTQ_IRQ_CFG1:
1226 s->eventq_irq_cfg1 = data;
1227 return MEMTX_OK;
1228 case A_EVENTQ_IRQ_CFG2:
1229 s->eventq_irq_cfg2 = data;
1230 return MEMTX_OK;
1231 default:
1232 qemu_log_mask(LOG_UNIMP,
1233 "%s Unexpected 32-bit access to 0x%"PRIx64" (WI)\n",
1234 __func__, offset);
1235 return MEMTX_OK;
1236 }
1237}
1238
10a83cb9
PM		 1239static MemTxResult smmu_write_mmio(void *opaque, hwaddr offset, uint64_t data,
1240 unsigned size, MemTxAttrs attrs)
1241{
fae4be38
EA		 1242 SMMUState *sys = opaque;
1243 SMMUv3State *s = ARM_SMMUV3(sys);
1244 MemTxResult r;
1245
1246 /* CONSTRAINED UNPREDICTABLE choice to have page0/1 be exact aliases */
1247 offset &= ~0x10000;
1248
1249 switch (size) {
1250 case 8:
1251 r = smmu_writell(s, offset, data, attrs);
1252 break;
1253 case 4:
1254 r = smmu_writel(s, offset, data, attrs);
1255 break;
1256 default:
1257 r = MEMTX_ERROR;
1258 break;
1259 }
1260
1261 trace_smmuv3_write_mmio(offset, data, size, r);
1262 return r;
10a83cb9
PM		 1263}
1264
1265static MemTxResult smmu_readll(SMMUv3State *s, hwaddr offset,
1266 uint64_t *data, MemTxAttrs attrs)
1267{
1268 switch (offset) {
1269 case A_GERROR_IRQ_CFG0:
1270 *data = s->gerror_irq_cfg0;
1271 return MEMTX_OK;
1272 case A_STRTAB_BASE:
1273 *data = s->strtab_base;
1274 return MEMTX_OK;
1275 case A_CMDQ_BASE:
1276 *data = s->cmdq.base;
1277 return MEMTX_OK;
1278 case A_EVENTQ_BASE:
1279 *data = s->eventq.base;
1280 return MEMTX_OK;
1281 default:
1282 *data = 0;
1283 qemu_log_mask(LOG_UNIMP,
1284 "%s Unexpected 64-bit access to 0x%"PRIx64" (RAZ)\n",
1285 __func__, offset);
1286 return MEMTX_OK;
1287 }
1288}
1289
1290static MemTxResult smmu_readl(SMMUv3State *s, hwaddr offset,
1291 uint64_t *data, MemTxAttrs attrs)
1292{
1293 switch (offset) {
97fb318d 1294 case A_IDREGS ... A_IDREGS + 0x2f:
10a83cb9
PM		 1295 *data = smmuv3_idreg(offset - A_IDREGS);
1296 return MEMTX_OK;
1297 case A_IDR0 ... A_IDR5:
1298 *data = s->idr[(offset - A_IDR0) / 4];
1299 return MEMTX_OK;
1300 case A_IIDR:
1301 *data = s->iidr;
1302 return MEMTX_OK;
5888f0ad
EA		 1303 case A_AIDR:
1304 *data = s->aidr;
1305 return MEMTX_OK;
10a83cb9
PM		 1306 case A_CR0:
1307 *data = s->cr[0];
1308 return MEMTX_OK;
1309 case A_CR0ACK:
1310 *data = s->cr0ack;
1311 return MEMTX_OK;
1312 case A_CR1:
1313 *data = s->cr[1];
1314 return MEMTX_OK;
1315 case A_CR2:
1316 *data = s->cr[2];
1317 return MEMTX_OK;
1318 case A_STATUSR:
1319 *data = s->statusr;
1320 return MEMTX_OK;
1321 case A_IRQ_CTRL:
1322 case A_IRQ_CTRL_ACK:
1323 *data = s->irq_ctrl;
1324 return MEMTX_OK;
1325 case A_GERROR:
1326 *data = s->gerror;
1327 return MEMTX_OK;
1328 case A_GERRORN:
1329 *data = s->gerrorn;
1330 return MEMTX_OK;
1331 case A_GERROR_IRQ_CFG0: /* 64b */
1332 *data = extract64(s->gerror_irq_cfg0, 0, 32);
1333 return MEMTX_OK;
1334 case A_GERROR_IRQ_CFG0 + 4:
1335 *data = extract64(s->gerror_irq_cfg0, 32, 32);
1336 return MEMTX_OK;
1337 case A_GERROR_IRQ_CFG1:
1338 *data = s->gerror_irq_cfg1;
1339 return MEMTX_OK;
1340 case A_GERROR_IRQ_CFG2:
1341 *data = s->gerror_irq_cfg2;
1342 return MEMTX_OK;
1343 case A_STRTAB_BASE: /* 64b */
1344 *data = extract64(s->strtab_base, 0, 32);
1345 return MEMTX_OK;
1346 case A_STRTAB_BASE + 4: /* 64b */
1347 *data = extract64(s->strtab_base, 32, 32);
1348 return MEMTX_OK;
1349 case A_STRTAB_BASE_CFG:
1350 *data = s->strtab_base_cfg;
1351 return MEMTX_OK;
1352 case A_CMDQ_BASE: /* 64b */
1353 *data = extract64(s->cmdq.base, 0, 32);
1354 return MEMTX_OK;
1355 case A_CMDQ_BASE + 4:
1356 *data = extract64(s->cmdq.base, 32, 32);
1357 return MEMTX_OK;
1358 case A_CMDQ_PROD:
1359 *data = s->cmdq.prod;
1360 return MEMTX_OK;
1361 case A_CMDQ_CONS:
1362 *data = s->cmdq.cons;
1363 return MEMTX_OK;
1364 case A_EVENTQ_BASE: /* 64b */
1365 *data = extract64(s->eventq.base, 0, 32);
1366 return MEMTX_OK;
1367 case A_EVENTQ_BASE + 4: /* 64b */
1368 *data = extract64(s->eventq.base, 32, 32);
1369 return MEMTX_OK;
1370 case A_EVENTQ_PROD:
1371 *data = s->eventq.prod;
1372 return MEMTX_OK;
1373 case A_EVENTQ_CONS:
1374 *data = s->eventq.cons;
1375 return MEMTX_OK;
1376 default:
1377 *data = 0;
1378 qemu_log_mask(LOG_UNIMP,
1379 "%s unhandled 32-bit access at 0x%"PRIx64" (RAZ)\n",
1380 __func__, offset);
1381 return MEMTX_OK;
1382 }
1383}
1384
1385static MemTxResult smmu_read_mmio(void *opaque, hwaddr offset, uint64_t *data,
1386 unsigned size, MemTxAttrs attrs)
1387{
1388 SMMUState *sys = opaque;
1389 SMMUv3State *s = ARM_SMMUV3(sys);
1390 MemTxResult r;
1391
1392 /* CONSTRAINED UNPREDICTABLE choice to have page0/1 be exact aliases */
1393 offset &= ~0x10000;
1394
1395 switch (size) {
1396 case 8:
1397 r = smmu_readll(s, offset, data, attrs);
1398 break;
1399 case 4:
1400 r = smmu_readl(s, offset, data, attrs);
1401 break;
1402 default:
1403 r = MEMTX_ERROR;
1404 break;
1405 }
1406
1407 trace_smmuv3_read_mmio(offset, *data, size, r);
1408 return r;
1409}
1410
1411static const MemoryRegionOps smmu_mem_ops = {
1412 .read_with_attrs = smmu_read_mmio,
1413 .write_with_attrs = smmu_write_mmio,
1414 .endianness = DEVICE_LITTLE_ENDIAN,
1415 .valid = {
1416 .min_access_size = 4,
1417 .max_access_size = 8,
1418 },
1419 .impl = {
1420 .min_access_size = 4,
1421 .max_access_size = 8,
1422 },
1423};
1424
1425static void smmu_init_irq(SMMUv3State *s, SysBusDevice *dev)
1426{
1427 int i;
1428
1429 for (i = 0; i < ARRAY_SIZE(s->irq); i++) {
1430 sysbus_init_irq(dev, &s->irq[i]);
1431 }
1432}
1433
1434static void smmu_reset(DeviceState *dev)
1435{
1436 SMMUv3State *s = ARM_SMMUV3(dev);
1437 SMMUv3Class *c = ARM_SMMUV3_GET_CLASS(s);
1438
1439 c->parent_reset(dev);
1440
1441 smmuv3_init_regs(s);
1442}
1443
1444static void smmu_realize(DeviceState *d, Error **errp)
1445{
1446 SMMUState *sys = ARM_SMMU(d);
1447 SMMUv3State *s = ARM_SMMUV3(sys);
1448 SMMUv3Class *c = ARM_SMMUV3_GET_CLASS(s);
1449 SysBusDevice *dev = SYS_BUS_DEVICE(d);
1450 Error *local_err = NULL;
1451
1452 c->parent_realize(d, &local_err);
1453 if (local_err) {
1454 error_propagate(errp, local_err);
1455 return;
1456 }
1457
32cfd7f3
EA		 1458 qemu_mutex_init(&s->mutex);
1459
10a83cb9
PM		 1460 memory_region_init_io(&sys->iomem, OBJECT(s),
1461 &smmu_mem_ops, sys, TYPE_ARM_SMMUV3, 0x20000);
1462
1463 sys->mrtypename = TYPE_SMMUV3_IOMMU_MEMORY_REGION;
1464
1465 sysbus_init_mmio(dev, &sys->iomem);
1466
1467 smmu_init_irq(s, dev);
1468}
1469
1470static const VMStateDescription vmstate_smmuv3_queue = {
1471 .name = "smmuv3_queue",
1472 .version_id = 1,
1473 .minimum_version_id = 1,
1474 .fields = (VMStateField[]) {
1475 VMSTATE_UINT64(base, SMMUQueue),
1476 VMSTATE_UINT32(prod, SMMUQueue),
1477 VMSTATE_UINT32(cons, SMMUQueue),
1478 VMSTATE_UINT8(log2size, SMMUQueue),
758b71f7 1479 VMSTATE_END_OF_LIST(),
10a83cb9
PM		 1480 },
1481};
1482
1483static const VMStateDescription vmstate_smmuv3 = {
1484 .name = "smmuv3",
1485 .version_id = 1,
1486 .minimum_version_id = 1,
a55aab61 1487 .priority = MIG_PRI_IOMMU,
10a83cb9
PM		 1488 .fields = (VMStateField[]) {
1489 VMSTATE_UINT32(features, SMMUv3State),
1490 VMSTATE_UINT8(sid_size, SMMUv3State),
1491 VMSTATE_UINT8(sid_split, SMMUv3State),
1492
1493 VMSTATE_UINT32_ARRAY(cr, SMMUv3State, 3),
1494 VMSTATE_UINT32(cr0ack, SMMUv3State),
1495 VMSTATE_UINT32(statusr, SMMUv3State),
1496 VMSTATE_UINT32(irq_ctrl, SMMUv3State),
1497 VMSTATE_UINT32(gerror, SMMUv3State),
1498 VMSTATE_UINT32(gerrorn, SMMUv3State),
1499 VMSTATE_UINT64(gerror_irq_cfg0, SMMUv3State),
1500 VMSTATE_UINT32(gerror_irq_cfg1, SMMUv3State),
1501 VMSTATE_UINT32(gerror_irq_cfg2, SMMUv3State),
1502 VMSTATE_UINT64(strtab_base, SMMUv3State),
1503 VMSTATE_UINT32(strtab_base_cfg, SMMUv3State),
1504 VMSTATE_UINT64(eventq_irq_cfg0, SMMUv3State),
1505 VMSTATE_UINT32(eventq_irq_cfg1, SMMUv3State),
1506 VMSTATE_UINT32(eventq_irq_cfg2, SMMUv3State),
1507
1508 VMSTATE_STRUCT(cmdq, SMMUv3State, 0, vmstate_smmuv3_queue, SMMUQueue),
1509 VMSTATE_STRUCT(eventq, SMMUv3State, 0, vmstate_smmuv3_queue, SMMUQueue),
1510
1511 VMSTATE_END_OF_LIST(),
1512 },
1513};
1514
1515static void smmuv3_instance_init(Object *obj)
1516{
1517 /* Nothing much to do here as of now */
1518}
1519
1520static void smmuv3_class_init(ObjectClass *klass, void *data)
1521{
1522 DeviceClass *dc = DEVICE_CLASS(klass);
1523 SMMUv3Class *c = ARM_SMMUV3_CLASS(klass);
1524
1525 dc->vmsd = &vmstate_smmuv3;
1526 device_class_set_parent_reset(dc, smmu_reset, &c->parent_reset);
1527 c->parent_realize = dc->realize;
1528 dc->realize = smmu_realize;
1529}
1530
549d4005
EA		 1531static int smmuv3_notify_flag_changed(IOMMUMemoryRegion *iommu,
1532 IOMMUNotifierFlag old,
1533 IOMMUNotifierFlag new,
1534 Error **errp)
0d1ac82e 1535{
832e4222
EA		 1536 SMMUDevice *sdev = container_of(iommu, SMMUDevice, iommu);
1537 SMMUv3State *s3 = sdev->smmu;
1538 SMMUState *s = &(s3->smmu_state);
832e4222 1539
958ec334
PX		 1540 if (new & IOMMU_NOTIFIER_DEVIOTLB_UNMAP) {
1541 error_setg(errp, "SMMUv3 does not support dev-iotlb yet");
1542 return -EINVAL;
1543 }
1544
832e4222 1545 if (new & IOMMU_NOTIFIER_MAP) {
549d4005
EA		 1546 error_setg(errp,
1547 "device %02x.%02x.%x requires iommu MAP notifier which is "
1548 "not currently supported", pci_bus_num(sdev->bus),
1549 PCI_SLOT(sdev->devfn), PCI_FUNC(sdev->devfn));
1550 return -EINVAL;
832e4222
EA		 1551 }
1552
0d1ac82e 1553 if (old == IOMMU_NOTIFIER_NONE) {
832e4222 1554 trace_smmuv3_notify_flag_add(iommu->parent_obj.name);
c6370441
EA		 1555 QLIST_INSERT_HEAD(&s->devices_with_notifiers, sdev, next);
1556 } else if (new == IOMMU_NOTIFIER_NONE) {
1557 trace_smmuv3_notify_flag_del(iommu->parent_obj.name);
1558 QLIST_REMOVE(sdev, next);
0d1ac82e 1559 }
549d4005 1560 return 0;
0d1ac82e
EA		 1561}
1562
10a83cb9
PM		 1563static void smmuv3_iommu_memory_region_class_init(ObjectClass *klass,
1564 void *data)
1565{
9bde7f06
EA		 1566 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass);
1567
1568 imrc->translate = smmuv3_translate;
0d1ac82e 1569 imrc->notify_flag_changed = smmuv3_notify_flag_changed;
10a83cb9
PM		 1570}
1571
1572static const TypeInfo smmuv3_type_info = {
1573 .name = TYPE_ARM_SMMUV3,
1574 .parent = TYPE_ARM_SMMU,
1575 .instance_size = sizeof(SMMUv3State),
1576 .instance_init = smmuv3_instance_init,
1577 .class_size = sizeof(SMMUv3Class),
1578 .class_init = smmuv3_class_init,
1579};
1580
1581static const TypeInfo smmuv3_iommu_memory_region_info = {
1582 .parent = TYPE_IOMMU_MEMORY_REGION,
1583 .name = TYPE_SMMUV3_IOMMU_MEMORY_REGION,
1584 .class_init = smmuv3_iommu_memory_region_class_init,
1585};
1586
1587static void smmuv3_register_types(void)
1588{
1589 type_register(&smmuv3_type_info);
1590 type_register(&smmuv3_iommu_memory_region_info);
1591}
1592
1593type_init(smmuv3_register_types)
1594
QEMU mirror