2 * ARM SMMUv3 support - Internal API
4 * Copyright (C) 2014-2016 Broadcom Corporation
5 * Copyright (c) 2017 Red Hat, Inc.
6 * Written by Prem Mallappa, Eric Auger
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License along
18 * with this program; if not, see <http://www.gnu.org/licenses/>.
21 #ifndef HW_ARM_SMMU_V3_INTERNAL_H
22 #define HW_ARM_SMMU_V3_INTERNAL_H
24 #include "hw/arm/smmu-common.h"
29 FIELD(IDR0
, S1P
, 1 , 1)
30 FIELD(IDR0
, TTF
, 2 , 2)
31 FIELD(IDR0
, COHACC
, 4 , 1)
32 FIELD(IDR0
, ASID16
, 12, 1)
33 FIELD(IDR0
, TTENDIAN
, 21, 2)
34 FIELD(IDR0
, STALL_MODEL
, 24, 2)
35 FIELD(IDR0
, TERM_MODEL
, 26, 1)
36 FIELD(IDR0
, STLEVEL
, 27, 2)
39 FIELD(IDR1
, SIDSIZE
, 0 , 6)
40 FIELD(IDR1
, EVENTQS
, 16, 5)
41 FIELD(IDR1
, CMDQS
, 21, 5)
43 #define SMMU_IDR1_SIDSIZE 16
45 #define SMMU_EVENTQS 19
51 FIELD(IDR5
, OAS
, 0, 3);
52 FIELD(IDR5
, GRAN4K
, 4, 1);
53 FIELD(IDR5
, GRAN16K
, 5, 1);
54 FIELD(IDR5
, GRAN64K
, 6, 1);
56 #define SMMU_IDR5_OAS 4
60 FIELD(CR0
, SMMU_ENABLE
, 0, 1)
61 FIELD(CR0
, EVENTQEN
, 2, 1)
62 FIELD(CR0
, CMDQEN
, 3, 1)
64 #define SMMU_CR0_RESERVED 0xFFFFFC20
71 FIELD(IRQ_CTRL
, GERROR_IRQEN
, 0, 1)
72 FIELD(IRQ_CTRL
, PRI_IRQEN
, 1, 1)
73 FIELD(IRQ_CTRL
, EVENTQ_IRQEN
, 2, 1)
75 REG32(IRQ_CTRL_ACK
, 0x54)
77 FIELD(GERROR
, CMDQ_ERR
, 0, 1)
78 FIELD(GERROR
, EVENTQ_ABT_ERR
, 2, 1)
79 FIELD(GERROR
, PRIQ_ABT_ERR
, 3, 1)
80 FIELD(GERROR
, MSI_CMDQ_ABT_ERR
, 4, 1)
81 FIELD(GERROR
, MSI_EVENTQ_ABT_ERR
, 5, 1)
82 FIELD(GERROR
, MSI_PRIQ_ABT_ERR
, 6, 1)
83 FIELD(GERROR
, MSI_GERROR_ABT_ERR
, 7, 1)
84 FIELD(GERROR
, MSI_SFM_ERR
, 8, 1)
88 #define A_GERROR_IRQ_CFG0 0x68 /* 64b */
89 REG32(GERROR_IRQ_CFG1
, 0x70)
90 REG32(GERROR_IRQ_CFG2
, 0x74)
92 #define A_STRTAB_BASE 0x80 /* 64b */
94 #define SMMU_BASE_ADDR_MASK 0xffffffffffe0
96 REG32(STRTAB_BASE_CFG
, 0x88)
97 FIELD(STRTAB_BASE_CFG
, FMT
, 16, 2)
98 FIELD(STRTAB_BASE_CFG
, SPLIT
, 6 , 5)
99 FIELD(STRTAB_BASE_CFG
, LOG2SIZE
, 0 , 6)
101 #define A_CMDQ_BASE 0x90 /* 64b */
102 REG32(CMDQ_PROD
, 0x98)
103 REG32(CMDQ_CONS
, 0x9c)
104 FIELD(CMDQ_CONS
, ERR
, 24, 7)
106 #define A_EVENTQ_BASE 0xa0 /* 64b */
107 REG32(EVENTQ_PROD
, 0xa8)
108 REG32(EVENTQ_CONS
, 0xac)
110 #define A_EVENTQ_IRQ_CFG0 0xb0 /* 64b */
111 REG32(EVENTQ_IRQ_CFG1
, 0xb8)
112 REG32(EVENTQ_IRQ_CFG2
, 0xbc)
114 #define A_IDREGS 0xfd0
116 static inline int smmu_enabled(SMMUv3State
*s
)
118 return FIELD_EX32(s
->cr
[0], CR0
, SMMU_ENABLE
);
121 /* Command Queue Entry */
126 /* Event Queue Entry */
131 static inline uint32_t smmuv3_idreg(int regoffset
)
134 * Return the value of the Primecell/Corelink ID registers at the
135 * specified offset from the first ID register.
136 * These value indicate an ARM implementation of MMU600 p1
138 static const uint8_t smmuv3_ids
[] = {
139 0x04, 0, 0, 0, 0x84, 0xB4, 0xF0, 0x10, 0x0D, 0xF0, 0x05, 0xB1
141 return smmuv3_ids
[regoffset
/ 4];
144 static inline bool smmuv3_eventq_irq_enabled(SMMUv3State
*s
)
146 return FIELD_EX32(s
->irq_ctrl
, IRQ_CTRL
, EVENTQ_IRQEN
);
149 static inline bool smmuv3_gerror_irq_enabled(SMMUv3State
*s
)
151 return FIELD_EX32(s
->irq_ctrl
, IRQ_CTRL
, GERROR_IRQEN
);
156 #define Q_BASE(q) ((q)->base & SMMU_BASE_ADDR_MASK)
157 #define WRAP_MASK(q) (1 << (q)->log2size)
158 #define INDEX_MASK(q) (((1 << (q)->log2size)) - 1)
159 #define WRAP_INDEX_MASK(q) ((1 << ((q)->log2size + 1)) - 1)
161 #define Q_CONS(q) ((q)->cons & INDEX_MASK(q))
162 #define Q_PROD(q) ((q)->prod & INDEX_MASK(q))
164 #define Q_CONS_ENTRY(q) (Q_BASE(q) + (q)->entry_size * Q_CONS(q))
165 #define Q_PROD_ENTRY(q) (Q_BASE(q) + (q)->entry_size * Q_PROD(q))
167 #define Q_CONS_WRAP(q) (((q)->cons & WRAP_MASK(q)) >> (q)->log2size)
168 #define Q_PROD_WRAP(q) (((q)->prod & WRAP_MASK(q)) >> (q)->log2size)
170 static inline bool smmuv3_q_full(SMMUQueue
*q
)
172 return ((q
->cons
^ q
->prod
) & WRAP_INDEX_MASK(q
)) == WRAP_MASK(q
);
175 static inline bool smmuv3_q_empty(SMMUQueue
*q
)
177 return (q
->cons
& WRAP_INDEX_MASK(q
)) == (q
->prod
& WRAP_INDEX_MASK(q
));
180 static inline void queue_prod_incr(SMMUQueue
*q
)
182 q
->prod
= (q
->prod
+ 1) & WRAP_INDEX_MASK(q
);
185 static inline void queue_cons_incr(SMMUQueue
*q
)
188 * We have to use deposit for the CONS registers to preserve
189 * the ERR field in the high bits.
191 q
->cons
= deposit32(q
->cons
, 0, q
->log2size
+ 1, q
->cons
+ 1);
194 static inline bool smmuv3_cmdq_enabled(SMMUv3State
*s
)
196 return FIELD_EX32(s
->cr
[0], CR0
, CMDQEN
);
199 static inline bool smmuv3_eventq_enabled(SMMUv3State
*s
)
201 return FIELD_EX32(s
->cr
[0], CR0
, EVENTQEN
);
204 static inline void smmu_write_cmdq_err(SMMUv3State
*s
, uint32_t err_type
)
206 s
->cmdq
.cons
= FIELD_DP32(s
->cmdq
.cons
, CMDQ_CONS
, ERR
, err_type
);
211 typedef enum SMMUCommandType
{
212 SMMU_CMD_NONE
= 0x00,
213 SMMU_CMD_PREFETCH_CONFIG
,
214 SMMU_CMD_PREFETCH_ADDR
,
216 SMMU_CMD_CFGI_STE_RANGE
,
218 SMMU_CMD_CFGI_CD_ALL
,
220 SMMU_CMD_TLBI_NH_ALL
= 0x10,
221 SMMU_CMD_TLBI_NH_ASID
,
223 SMMU_CMD_TLBI_NH_VAA
,
224 SMMU_CMD_TLBI_EL3_ALL
= 0x18,
225 SMMU_CMD_TLBI_EL3_VA
= 0x1a,
226 SMMU_CMD_TLBI_EL2_ALL
= 0x20,
227 SMMU_CMD_TLBI_EL2_ASID
,
228 SMMU_CMD_TLBI_EL2_VA
,
229 SMMU_CMD_TLBI_EL2_VAA
,
230 SMMU_CMD_TLBI_S12_VMALL
= 0x28,
231 SMMU_CMD_TLBI_S2_IPA
= 0x2a,
232 SMMU_CMD_TLBI_NSNH_ALL
= 0x30,
233 SMMU_CMD_ATC_INV
= 0x40,
235 SMMU_CMD_RESUME
= 0x44,
240 static const char *cmd_stringify
[] = {
241 [SMMU_CMD_PREFETCH_CONFIG
] = "SMMU_CMD_PREFETCH_CONFIG",
242 [SMMU_CMD_PREFETCH_ADDR
] = "SMMU_CMD_PREFETCH_ADDR",
243 [SMMU_CMD_CFGI_STE
] = "SMMU_CMD_CFGI_STE",
244 [SMMU_CMD_CFGI_STE_RANGE
] = "SMMU_CMD_CFGI_STE_RANGE",
245 [SMMU_CMD_CFGI_CD
] = "SMMU_CMD_CFGI_CD",
246 [SMMU_CMD_CFGI_CD_ALL
] = "SMMU_CMD_CFGI_CD_ALL",
247 [SMMU_CMD_CFGI_ALL
] = "SMMU_CMD_CFGI_ALL",
248 [SMMU_CMD_TLBI_NH_ALL
] = "SMMU_CMD_TLBI_NH_ALL",
249 [SMMU_CMD_TLBI_NH_ASID
] = "SMMU_CMD_TLBI_NH_ASID",
250 [SMMU_CMD_TLBI_NH_VA
] = "SMMU_CMD_TLBI_NH_VA",
251 [SMMU_CMD_TLBI_NH_VAA
] = "SMMU_CMD_TLBI_NH_VAA",
252 [SMMU_CMD_TLBI_EL3_ALL
] = "SMMU_CMD_TLBI_EL3_ALL",
253 [SMMU_CMD_TLBI_EL3_VA
] = "SMMU_CMD_TLBI_EL3_VA",
254 [SMMU_CMD_TLBI_EL2_ALL
] = "SMMU_CMD_TLBI_EL2_ALL",
255 [SMMU_CMD_TLBI_EL2_ASID
] = "SMMU_CMD_TLBI_EL2_ASID",
256 [SMMU_CMD_TLBI_EL2_VA
] = "SMMU_CMD_TLBI_EL2_VA",
257 [SMMU_CMD_TLBI_EL2_VAA
] = "SMMU_CMD_TLBI_EL2_VAA",
258 [SMMU_CMD_TLBI_S12_VMALL
] = "SMMU_CMD_TLBI_S12_VMALL",
259 [SMMU_CMD_TLBI_S2_IPA
] = "SMMU_CMD_TLBI_S2_IPA",
260 [SMMU_CMD_TLBI_NSNH_ALL
] = "SMMU_CMD_TLBI_NSNH_ALL",
261 [SMMU_CMD_ATC_INV
] = "SMMU_CMD_ATC_INV",
262 [SMMU_CMD_PRI_RESP
] = "SMMU_CMD_PRI_RESP",
263 [SMMU_CMD_RESUME
] = "SMMU_CMD_RESUME",
264 [SMMU_CMD_STALL_TERM
] = "SMMU_CMD_STALL_TERM",
265 [SMMU_CMD_SYNC
] = "SMMU_CMD_SYNC",
268 static inline const char *smmu_cmd_string(SMMUCommandType type
)
270 if (type
> SMMU_CMD_NONE
&& type
< ARRAY_SIZE(cmd_stringify
)) {
271 return cmd_stringify
[type
] ? cmd_stringify
[type
] : "UNKNOWN";
280 SMMU_CERROR_NONE
= 0,
283 SMMU_CERROR_ATC_INV_SYNC
,
286 enum { /* Command completion notification */
292 #define CMD_TYPE(x) extract32((x)->word[0], 0 , 8)
293 #define CMD_SSEC(x) extract32((x)->word[0], 10, 1)
294 #define CMD_SSV(x) extract32((x)->word[0], 11, 1)
295 #define CMD_RESUME_AC(x) extract32((x)->word[0], 12, 1)
296 #define CMD_RESUME_AB(x) extract32((x)->word[0], 13, 1)
297 #define CMD_SYNC_CS(x) extract32((x)->word[0], 12, 2)
298 #define CMD_SSID(x) extract32((x)->word[0], 12, 20)
299 #define CMD_SID(x) ((x)->word[1])
300 #define CMD_VMID(x) extract32((x)->word[1], 0 , 16)
301 #define CMD_ASID(x) extract32((x)->word[1], 16, 16)
302 #define CMD_RESUME_STAG(x) extract32((x)->word[2], 0 , 16)
303 #define CMD_RESP(x) extract32((x)->word[2], 11, 2)
304 #define CMD_LEAF(x) extract32((x)->word[2], 0 , 1)
305 #define CMD_STE_RANGE(x) extract32((x)->word[2], 0 , 5)
306 #define CMD_ADDR(x) ({ \
307 uint64_t high = (uint64_t)(x)->word[3]; \
308 uint64_t low = extract32((x)->word[2], 12, 20); \
309 uint64_t addr = high << 32 | (low << 12); \
313 #define SMMU_FEATURE_2LVL_STE (1 << 0)
317 typedef enum SMMUEventType
{
320 SMMU_EVT_C_BAD_STREAMID
,
321 SMMU_EVT_F_STE_FETCH
,
323 SMMU_EVT_F_BAD_ATS_TREQ
,
324 SMMU_EVT_F_STREAM_DISABLED
,
325 SMMU_EVT_F_TRANS_FORBIDDEN
,
326 SMMU_EVT_C_BAD_SUBSTREAMID
,
327 SMMU_EVT_F_CD_FETCH
,
329 SMMU_EVT_F_WALK_EABT
,
330 SMMU_EVT_F_TRANSLATION
= 0x10,
331 SMMU_EVT_F_ADDR_SIZE
,
333 SMMU_EVT_F_PERMISSION
,
334 SMMU_EVT_F_TLB_CONFLICT
= 0x20,
335 SMMU_EVT_F_CFG_CONFLICT
,
336 SMMU_EVT_E_PAGE_REQ
= 0x24,
339 static const char *event_stringify
[] = {
340 [SMMU_EVT_OK
] = "SMMU_EVT_OK",
341 [SMMU_EVT_F_UUT
] = "SMMU_EVT_F_UUT",
342 [SMMU_EVT_C_BAD_STREAMID
] = "SMMU_EVT_C_BAD_STREAMID",
343 [SMMU_EVT_F_STE_FETCH
] = "SMMU_EVT_F_STE_FETCH",
344 [SMMU_EVT_C_BAD_STE
] = "SMMU_EVT_C_BAD_STE",
345 [SMMU_EVT_F_BAD_ATS_TREQ
] = "SMMU_EVT_F_BAD_ATS_TREQ",
346 [SMMU_EVT_F_STREAM_DISABLED
] = "SMMU_EVT_F_STREAM_DISABLED",
347 [SMMU_EVT_F_TRANS_FORBIDDEN
] = "SMMU_EVT_F_TRANS_FORBIDDEN",
348 [SMMU_EVT_C_BAD_SUBSTREAMID
] = "SMMU_EVT_C_BAD_SUBSTREAMID",
349 [SMMU_EVT_F_CD_FETCH
] = "SMMU_EVT_F_CD_FETCH",
350 [SMMU_EVT_C_BAD_CD
] = "SMMU_EVT_C_BAD_CD",
351 [SMMU_EVT_F_WALK_EABT
] = "SMMU_EVT_F_WALK_EABT",
352 [SMMU_EVT_F_TRANSLATION
] = "SMMU_EVT_F_TRANSLATION",
353 [SMMU_EVT_F_ADDR_SIZE
] = "SMMU_EVT_F_ADDR_SIZE",
354 [SMMU_EVT_F_ACCESS
] = "SMMU_EVT_F_ACCESS",
355 [SMMU_EVT_F_PERMISSION
] = "SMMU_EVT_F_PERMISSION",
356 [SMMU_EVT_F_TLB_CONFLICT
] = "SMMU_EVT_F_TLB_CONFLICT",
357 [SMMU_EVT_F_CFG_CONFLICT
] = "SMMU_EVT_F_CFG_CONFLICT",
358 [SMMU_EVT_E_PAGE_REQ
] = "SMMU_EVT_E_PAGE_REQ",
361 static inline const char *smmu_event_string(SMMUEventType type
)
363 if (type
< ARRAY_SIZE(event_stringify
)) {
364 return event_stringify
[type
] ? event_stringify
[type
] : "UNKNOWN";
370 /* Encode an event record */
371 typedef struct SMMUEventInfo
{
375 bool record_trans_faults
;
389 struct SSIDAddrInfo
{
394 struct SSIDInfo c_bad_ste
;
398 } f_transl_forbidden
;
402 struct SSIDAddrInfo f_cd_fetch
;
403 struct SSIDInfo c_bad_cd
;
417 struct FullInfo f_translation
;
418 struct FullInfo f_addr_size
;
419 struct FullInfo f_access
;
420 struct FullInfo f_permission
;
421 struct SSIDInfo f_cfg_conflict
;
435 #define EVT_Q_OVERFLOW (1 << 31)
437 #define EVT_SET_TYPE(x, v) deposit32((x)->word[0], 0 , 8 , v)
438 #define EVT_SET_SSV(x, v) deposit32((x)->word[0], 11, 1 , v)
439 #define EVT_SET_SSID(x, v) deposit32((x)->word[0], 12, 20, v)
440 #define EVT_SET_SID(x, v) ((x)->word[1] = v)
441 #define EVT_SET_STAG(x, v) deposit32((x)->word[2], 0 , 16, v)
442 #define EVT_SET_STALL(x, v) deposit32((x)->word[2], 31, 1 , v)
443 #define EVT_SET_PNU(x, v) deposit32((x)->word[3], 1 , 1 , v)
444 #define EVT_SET_IND(x, v) deposit32((x)->word[3], 2 , 1 , v)
445 #define EVT_SET_RNW(x, v) deposit32((x)->word[3], 3 , 1 , v)
446 #define EVT_SET_S2(x, v) deposit32((x)->word[3], 7 , 1 , v)
447 #define EVT_SET_CLASS(x, v) deposit32((x)->word[3], 8 , 2 , v)
448 #define EVT_SET_ADDR(x, addr) \
450 (x)->word[5] = (uint32_t)(addr >> 32); \
451 (x)->word[4] = (uint32_t)(addr & 0xffffffff); \
453 #define EVT_SET_ADDR2(x, addr) \
455 deposit32((x)->word[7], 3, 29, addr >> 16); \
456 deposit32((x)->word[7], 0, 16, addr & 0xffff);\
459 void smmuv3_record_event(SMMUv3State
*s
, SMMUEventInfo
*event
);
461 /* Configuration Data */
463 /* STE Level 1 Descriptor */
464 typedef struct STEDesc
{
468 /* CD Level 1 Descriptor */
469 typedef struct CDDesc
{
473 /* Stream Table Entry(STE) */
478 /* Context Descriptor(CD) */
485 #define STE_VALID(x) extract32((x)->word[0], 0, 1)
487 #define STE_CONFIG(x) extract32((x)->word[0], 1, 3)
488 #define STE_CFG_S1_ENABLED(config) (config & 0x1)
489 #define STE_CFG_S2_ENABLED(config) (config & 0x2)
490 #define STE_CFG_ABORT(config) (!(config & 0x4))
491 #define STE_CFG_BYPASS(config) (config == 0x4)
493 #define STE_S1FMT(x) extract32((x)->word[0], 4 , 2)
494 #define STE_S1CDMAX(x) extract32((x)->word[1], 27, 5)
495 #define STE_S1STALLD(x) extract32((x)->word[2], 27, 1)
496 #define STE_EATS(x) extract32((x)->word[2], 28, 2)
497 #define STE_STRW(x) extract32((x)->word[2], 30, 2)
498 #define STE_S2VMID(x) extract32((x)->word[4], 0 , 16)
499 #define STE_S2T0SZ(x) extract32((x)->word[5], 0 , 6)
500 #define STE_S2SL0(x) extract32((x)->word[5], 6 , 2)
501 #define STE_S2TG(x) extract32((x)->word[5], 14, 2)
502 #define STE_S2PS(x) extract32((x)->word[5], 16, 3)
503 #define STE_S2AA64(x) extract32((x)->word[5], 19, 1)
504 #define STE_S2HD(x) extract32((x)->word[5], 24, 1)
505 #define STE_S2HA(x) extract32((x)->word[5], 25, 1)
506 #define STE_S2S(x) extract32((x)->word[5], 26, 1)
507 #define STE_CTXPTR(x) \
509 unsigned long addr; \
510 addr = (uint64_t)extract32((x)->word[1], 0, 16) << 32; \
511 addr |= (uint64_t)((x)->word[0] & 0xffffffc0); \
515 #define STE_S2TTB(x) \
517 unsigned long addr; \
518 addr = (uint64_t)extract32((x)->word[7], 0, 16) << 32; \
519 addr |= (uint64_t)((x)->word[6] & 0xfffffff0); \
523 static inline int oas2bits(int oas_field
)
542 static inline int pa_range(STE
*ste
)
544 int oas_field
= MIN(STE_S2PS(ste
), SMMU_IDR5_OAS
);
546 if (!STE_S2AA64(ste
)) {
550 return oas2bits(oas_field
);
553 #define MAX_PA(ste) ((1 << pa_range(ste)) - 1)
557 #define CD_VALID(x) extract32((x)->word[0], 30, 1)
558 #define CD_ASID(x) extract32((x)->word[1], 16, 16)
559 #define CD_TTB(x, sel) \
562 hi = extract32((x)->word[(sel) * 2 + 3], 0, 19); \
564 lo = (x)->word[(sel) * 2 + 2] & ~0xfULL; \
568 #define CD_TSZ(x, sel) extract32((x)->word[0], (16 * (sel)) + 0, 6)
569 #define CD_TG(x, sel) extract32((x)->word[0], (16 * (sel)) + 6, 2)
570 #define CD_EPD(x, sel) extract32((x)->word[0], (16 * (sel)) + 14, 1)
571 #define CD_ENDI(x) extract32((x)->word[0], 15, 1)
572 #define CD_IPS(x) extract32((x)->word[1], 0 , 3)
573 #define CD_TBI(x) extract32((x)->word[1], 6 , 2)
574 #define CD_HD(x) extract32((x)->word[1], 10 , 1)
575 #define CD_HA(x) extract32((x)->word[1], 11 , 1)
576 #define CD_S(x) extract32((x)->word[1], 12, 1)
577 #define CD_R(x) extract32((x)->word[1], 13, 1)
578 #define CD_A(x) extract32((x)->word[1], 14, 1)
579 #define CD_AARCH64(x) extract32((x)->word[1], 9 , 1)
581 #define CDM_VALID(x) ((x)->word[0] & 0x1)
583 static inline int is_cd_valid(SMMUv3State
*s
, STE
*ste
, CD
*cd
)
589 * tg2granule - Decodes the CD translation granule size field according
591 * @bits: TG0/1 fields
592 * @ttbr: ttbr index in use
594 static inline int tg2granule(int bits
, int ttbr
)
598 return ttbr
? 0 : 12;
600 return ttbr
? 14 : 16;
602 return ttbr
? 12 : 14;
604 return ttbr
? 16 : 0;
610 static inline uint64_t l1std_l2ptr(STEDesc
*desc
)
615 lo
= desc
->word
[0] & ~0x1fULL
;
616 return hi
<< 32 | lo
;
619 #define L1STD_SPAN(stm) (extract32((stm)->word[0], 0, 4))