1 #------------------------------------------------------------------------------
3 # Copyright (c) 2008 - 2010, Apple Inc. All rights reserved.<BR>
4 # Copyright (c) 2011 - 2017, ARM Limited. All rights reserved.
5 # Copyright (c) 2016, Linaro Limited. All rights reserved.
6 # Copyright (c) 2020, NUVIA Inc. All rights reserved.
8 # SPDX-License-Identifier: BSD-2-Clause-Patent
10 #------------------------------------------------------------------------------
12 #include <Chipset/AArch64.h>
13 #include <AsmMacroIoLibV8.h>
15 .set CTRL_M_BIT, (1 << 0)
16 .set CTRL_A_BIT, (1 << 1)
17 .set CTRL_C_BIT, (1 << 2)
18 .set CTRL_SA_BIT, (1 << 3)
19 .set CTRL_I_BIT, (1 << 12)
20 .set CTRL_V_BIT, (1 << 12)
21 .set CPACR_VFP_BITS, (3 << 20)
23 ASM_FUNC(ArmInvalidateDataCacheEntryByMVA)
24 dc ivac, x0 // Invalidate single data cache line
28 ASM_FUNC(ArmCleanDataCacheEntryByMVA)
29 dc cvac, x0 // Clean single data cache line
33 ASM_FUNC(ArmCleanDataCacheEntryToPoUByMVA)
34 dc cvau, x0 // Clean single data cache line to PoU
37 ASM_FUNC(ArmInvalidateInstructionCacheEntryToPoUByMVA)
38 ic ivau, x0 // Invalidate single instruction cache line to PoU
42 ASM_FUNC(ArmCleanInvalidateDataCacheEntryByMVA)
43 dc civac, x0 // Clean and invalidate single data cache line
47 ASM_FUNC(ArmInvalidateDataCacheEntryBySetWay)
48 dc isw, x0 // Invalidate this line
52 ASM_FUNC(ArmCleanInvalidateDataCacheEntryBySetWay)
53 dc cisw, x0 // Clean and Invalidate this line
57 ASM_FUNC(ArmCleanDataCacheEntryBySetWay)
58 dc csw, x0 // Clean this line
62 ASM_FUNC(ArmInvalidateInstructionCache)
63 ic iallu // Invalidate entire instruction cache
69 ASM_FUNC(ArmEnableMmu)
71 1: mrs x0, sctlr_el1 // Read System control register EL1
73 2: mrs x0, sctlr_el2 // Read System control register EL2
75 3: mrs x0, sctlr_el3 // Read System control register EL3
76 4: orr x0, x0, #CTRL_M_BIT // Set MMU enable bit
81 msr sctlr_el1, x0 // Write back
86 msr sctlr_el2, x0 // Write back
91 msr sctlr_el3, x0 // Write back
96 ASM_FUNC(ArmDisableMmu)
98 1: mrs x0, sctlr_el1 // Read System Control Register EL1
100 2: mrs x0, sctlr_el2 // Read System Control Register EL2
102 3: mrs x0, sctlr_el3 // Read System Control Register EL3
103 4: and x0, x0, #~CTRL_M_BIT // Clear MMU enable bit
104 EL1_OR_EL2_OR_EL3(x1)
105 1: msr sctlr_el1, x0 // Write back
108 2: msr sctlr_el2, x0 // Write back
111 3: msr sctlr_el3, x0 // Write back
118 ASM_FUNC(ArmDisableCachesAndMmu)
119 EL1_OR_EL2_OR_EL3(x1)
120 1: mrs x0, sctlr_el1 // Get control register EL1
122 2: mrs x0, sctlr_el2 // Get control register EL2
124 3: mrs x0, sctlr_el3 // Get control register EL3
125 4: mov x1, #~(CTRL_M_BIT | CTRL_C_BIT | CTRL_I_BIT) // Disable MMU, D & I caches
127 EL1_OR_EL2_OR_EL3(x1)
128 1: msr sctlr_el1, x0 // Write back control register
130 2: msr sctlr_el2, x0 // Write back control register
132 3: msr sctlr_el3, x0 // Write back control register
138 ASM_FUNC(ArmMmuEnabled)
139 EL1_OR_EL2_OR_EL3(x1)
140 1: mrs x0, sctlr_el1 // Get control register EL1
142 2: mrs x0, sctlr_el2 // Get control register EL2
144 3: mrs x0, sctlr_el3 // Get control register EL3
145 4: and x0, x0, #CTRL_M_BIT
149 ASM_FUNC(ArmEnableDataCache)
150 EL1_OR_EL2_OR_EL3(x1)
151 1: mrs x0, sctlr_el1 // Get control register EL1
153 2: mrs x0, sctlr_el2 // Get control register EL2
155 3: mrs x0, sctlr_el3 // Get control register EL3
156 4: orr x0, x0, #CTRL_C_BIT // Set C bit
157 EL1_OR_EL2_OR_EL3(x1)
158 1: msr sctlr_el1, x0 // Write back control register
160 2: msr sctlr_el2, x0 // Write back control register
162 3: msr sctlr_el3, x0 // Write back control register
168 ASM_FUNC(ArmDisableDataCache)
169 EL1_OR_EL2_OR_EL3(x1)
170 1: mrs x0, sctlr_el1 // Get control register EL1
172 2: mrs x0, sctlr_el2 // Get control register EL2
174 3: mrs x0, sctlr_el3 // Get control register EL3
175 4: and x0, x0, #~CTRL_C_BIT // Clear C bit
176 EL1_OR_EL2_OR_EL3(x1)
177 1: msr sctlr_el1, x0 // Write back control register
179 2: msr sctlr_el2, x0 // Write back control register
181 3: msr sctlr_el3, x0 // Write back control register
187 ASM_FUNC(ArmEnableInstructionCache)
188 EL1_OR_EL2_OR_EL3(x1)
189 1: mrs x0, sctlr_el1 // Get control register EL1
191 2: mrs x0, sctlr_el2 // Get control register EL2
193 3: mrs x0, sctlr_el3 // Get control register EL3
194 4: orr x0, x0, #CTRL_I_BIT // Set I bit
195 EL1_OR_EL2_OR_EL3(x1)
196 1: msr sctlr_el1, x0 // Write back control register
198 2: msr sctlr_el2, x0 // Write back control register
200 3: msr sctlr_el3, x0 // Write back control register
206 ASM_FUNC(ArmDisableInstructionCache)
207 EL1_OR_EL2_OR_EL3(x1)
208 1: mrs x0, sctlr_el1 // Get control register EL1
210 2: mrs x0, sctlr_el2 // Get control register EL2
212 3: mrs x0, sctlr_el3 // Get control register EL3
213 4: and x0, x0, #~CTRL_I_BIT // Clear I bit
214 EL1_OR_EL2_OR_EL3(x1)
215 1: msr sctlr_el1, x0 // Write back control register
217 2: msr sctlr_el2, x0 // Write back control register
219 3: msr sctlr_el3, x0 // Write back control register
225 ASM_FUNC(ArmEnableAlignmentCheck)
227 1: mrs x0, sctlr_el1 // Get control register EL1
229 2: mrs x0, sctlr_el2 // Get control register EL2
230 3: orr x0, x0, #CTRL_A_BIT // Set A (alignment check) bit
232 1: msr sctlr_el1, x0 // Write back control register
234 2: msr sctlr_el2, x0 // Write back control register
240 ASM_FUNC(ArmDisableAlignmentCheck)
241 EL1_OR_EL2_OR_EL3(x1)
242 1: mrs x0, sctlr_el1 // Get control register EL1
244 2: mrs x0, sctlr_el2 // Get control register EL2
246 3: mrs x0, sctlr_el3 // Get control register EL3
247 4: and x0, x0, #~CTRL_A_BIT // Clear A (alignment check) bit
248 EL1_OR_EL2_OR_EL3(x1)
249 1: msr sctlr_el1, x0 // Write back control register
251 2: msr sctlr_el2, x0 // Write back control register
253 3: msr sctlr_el3, x0 // Write back control register
258 ASM_FUNC(ArmEnableStackAlignmentCheck)
260 1: mrs x0, sctlr_el1 // Get control register EL1
262 2: mrs x0, sctlr_el2 // Get control register EL2
263 3: orr x0, x0, #CTRL_SA_BIT // Set SA (stack alignment check) bit
265 1: msr sctlr_el1, x0 // Write back control register
267 2: msr sctlr_el2, x0 // Write back control register
273 ASM_FUNC(ArmDisableStackAlignmentCheck)
274 EL1_OR_EL2_OR_EL3(x1)
275 1: mrs x0, sctlr_el1 // Get control register EL1
277 2: mrs x0, sctlr_el2 // Get control register EL2
279 3: mrs x0, sctlr_el3 // Get control register EL3
280 4: bic x0, x0, #CTRL_SA_BIT // Clear SA (stack alignment check) bit
281 EL1_OR_EL2_OR_EL3(x1)
282 1: msr sctlr_el1, x0 // Write back control register
284 2: msr sctlr_el2, x0 // Write back control register
286 3: msr sctlr_el3, x0 // Write back control register
292 // Always turned on in AArch64. Else implementation specific. Leave in for C compatibility for now
293 ASM_FUNC(ArmEnableBranchPrediction)
297 // Always turned on in AArch64. Else implementation specific. Leave in for C compatibility for now.
298 ASM_FUNC(ArmDisableBranchPrediction)
302 ASM_FUNC(AArch64AllDataCachesOperation)
303 // We can use regs 0-7 and 9-15 without having to save/restore.
304 // Save our link register on the stack. - The stack must always be quad-word aligned
305 stp x29, x30, [sp, #-16]!
307 mov x1, x0 // Save Function call in x1
308 mrs x6, clidr_el1 // Read EL1 CLIDR
309 and x3, x6, #0x7000000 // Mask out all but Level of Coherency (LoC)
310 lsr x3, x3, #23 // Left align cache level value - the level is shifted by 1 to the
311 // right to ease the access to CSSELR and the Set/Way operation.
312 cbz x3, L_Finished // No need to clean if LoC is 0
313 mov x10, #0 // Start clean at cache level 0
316 add x2, x10, x10, lsr #1 // Work out 3x cachelevel for cache info
317 lsr x12, x6, x2 // bottom 3 bits are the Cache type for this level
318 and x12, x12, #7 // get those 3 bits alone
319 cmp x12, #2 // what cache at this level?
320 b.lt L_Skip // no cache or only instruction cache at this level
321 msr csselr_el1, x10 // write the Cache Size selection register with current level (CSSELR)
322 isb // isb to sync the change to the CacheSizeID reg
323 mrs x12, ccsidr_el1 // reads current Cache Size ID register (CCSIDR)
324 and x2, x12, #0x7 // extract the line length field
325 add x2, x2, #4 // add 4 for the line length offset (log2 16 bytes)
328 and x4, x4, x12, lsr #3 // x4 is the max number on the way size (right aligned)
329 clz w5, w4 // w5 is the bit position of the way size increment
332 and x7, x7, x12, lsr #13 // x7 is the max number of the index size (right aligned)
335 mov x9, x4 // x9 working copy of the max way size (right aligned)
339 orr x0, x10, x11 // factor in the way number and cache number
341 orr x0, x0, x11 // factor in the index number
343 blr x1 // Goto requested cache operation
345 subs x9, x9, #1 // decrement the way number
347 subs x7, x7, #1 // decrement the index
350 add x10, x10, #2 // increment the cache number
357 ldp x29, x30, [sp], #0x10
361 ASM_FUNC(ArmDataMemoryBarrier)
366 ASM_FUNC(ArmDataSynchronizationBarrier)
371 ASM_FUNC(ArmInstructionSynchronizationBarrier)
376 ASM_FUNC(ArmWriteVBar)
377 EL1_OR_EL2_OR_EL3(x1)
378 1: msr vbar_el1, x0 // Set the Address of the EL1 Vector Table in the VBAR register
380 2: msr vbar_el2, x0 // Set the Address of the EL2 Vector Table in the VBAR register
382 3: msr vbar_el3, x0 // Set the Address of the EL3 Vector Table in the VBAR register
386 ASM_FUNC(ArmReadVBar)
387 EL1_OR_EL2_OR_EL3(x1)
388 1: mrs x0, vbar_el1 // Set the Address of the EL1 Vector Table in the VBAR register
390 2: mrs x0, vbar_el2 // Set the Address of the EL2 Vector Table in the VBAR register
392 3: mrs x0, vbar_el3 // Set the Address of the EL3 Vector Table in the VBAR register
396 ASM_FUNC(ArmEnableVFP)
397 // Check whether floating-point is implemented in the processor.
398 mov x1, x30 // Save LR
399 bl ArmReadIdAA64Pfr0 // Read EL1 Processor Feature Register (PFR0)
400 mov x30, x1 // Restore LR
401 ubfx x0, x0, #16, #4 // Extract the FP bits 16:19
402 cmp x0, #0xF // Check if FP bits are '1111b',
403 // i.e. Floating Point not implemented
404 b.eq 4f // Exit when VFP is not implemented.
406 // FVP is implemented.
407 // Make sure VFP exceptions are not trapped (to any exception level).
408 mrs x0, cpacr_el1 // Read EL1 Coprocessor Access Control Register (CPACR)
409 orr x0, x0, #CPACR_VFP_BITS // Disable FVP traps to EL1
410 msr cpacr_el1, x0 // Write back EL1 Coprocessor Access Control Register (CPACR)
411 mov x1, #AARCH64_CPTR_TFP // TFP Bit for trapping VFP Exceptions
412 EL1_OR_EL2_OR_EL3(x2)
413 1:ret // Not configurable in EL1
414 2:mrs x0, cptr_el2 // Disable VFP traps to EL2
418 3:mrs x0, cptr_el3 // Disable VFP traps to EL3
428 ASM_FUNC(ArmReadIdAA64Mmfr2)
429 mrs x0, ID_AA64MMFR2_EL1 // read EL1 MMFR2
432 ASM_FUNC(ArmReadMpidr)
433 mrs x0, mpidr_el1 // read EL1 MPIDR
437 // Keep old function names for C compatibility for now. Change later?
438 ASM_FUNC(ArmReadTpidrurw)
439 mrs x0, tpidr_el0 // read tpidr_el0 (v7 TPIDRURW) -> (v8 TPIDR_EL0)
443 // Keep old function names for C compatibility for now. Change later?
444 ASM_FUNC(ArmWriteTpidrurw)
445 msr tpidr_el0, x0 // write tpidr_el0 (v7 TPIDRURW) -> (v8 TPIDR_EL0)
449 // Arch timers are mandatory on AArch64
450 ASM_FUNC(ArmIsArchTimerImplemented)
455 ASM_FUNC(ArmReadIdAA64Pfr0)
456 mrs x0, id_aa64pfr0_el1 // Read ID_AA64PFR0 Register
460 // VOID ArmWriteHcr(UINTN Hcr)
461 ASM_FUNC(ArmWriteHcr)
462 msr hcr_el2, x0 // Write the passed HCR value
465 // UINTN ArmReadHcr(VOID)
470 // UINTN ArmReadCurrentEL(VOID)
471 ASM_FUNC(ArmReadCurrentEL)
475 // UINT32 ArmReadCntHctl(VOID)
476 ASM_FUNC(ArmReadCntHctl)
480 // VOID ArmWriteCntHctl(UINT32 CntHctl)
481 ASM_FUNC(ArmWriteCntHctl)
485 ASM_FUNCTION_REMOVE_IF_UNREFERENCED