]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - arch/arm64/kernel/cpufeature.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
arm64: cpufeature: Change read_cpuid() to use sysreg's mrs_s macro
[mirror_ubuntu-artful-kernel.git] / arch / arm64 / kernel / cpufeature.c
CommitLineData
359b7064
MZ		 1/*
2 * Contains CPU feature definitions
3 *
4 * Copyright (C) 2015 ARM Ltd.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 */
18
9cdf8ec4 19#define pr_fmt(fmt) "CPU features: " fmt
359b7064 20
3c739b57
SP		 21#include <linux/bsearch.h>
22#include <linux/sort.h>
359b7064
MZ		 23#include <linux/types.h>
24#include <asm/cpu.h>
25#include <asm/cpufeature.h>
dbb4e152 26#include <asm/cpu_ops.h>
338d4f49 27#include <asm/processor.h>
cdcf817b 28#include <asm/sysreg.h>
359b7064 29
9cdf8ec4
SP		 30unsigned long elf_hwcap __read_mostly;
31EXPORT_SYMBOL_GPL(elf_hwcap);
32
33#ifdef CONFIG_COMPAT
34#define COMPAT_ELF_HWCAP_DEFAULT \
35 (COMPAT_HWCAP_HALF|COMPAT_HWCAP_THUMB|\
36 COMPAT_HWCAP_FAST_MULT|COMPAT_HWCAP_EDSP|\
37 COMPAT_HWCAP_TLS|COMPAT_HWCAP_VFP|\
38 COMPAT_HWCAP_VFPv3|COMPAT_HWCAP_VFPv4|\
39 COMPAT_HWCAP_NEON|COMPAT_HWCAP_IDIV|\
40 COMPAT_HWCAP_LPAE)
41unsigned int compat_elf_hwcap __read_mostly = COMPAT_ELF_HWCAP_DEFAULT;
42unsigned int compat_elf_hwcap2 __read_mostly;
43#endif
44
45DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
46
4f0a606b 47#define __ARM64_FTR_BITS(SIGNED, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
3c739b57 48 { \
4f0a606b 49 .sign = SIGNED, \
3c739b57
SP		 50 .strict = STRICT, \
51 .type = TYPE, \
52 .shift = SHIFT, \
53 .width = WIDTH, \
54 .safe_val = SAFE_VAL, \
55 }
56
4f0a606b
SP		 57/* Define a feature with signed values */
58#define ARM64_FTR_BITS(STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
59 __ARM64_FTR_BITS(FTR_SIGNED, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL)
60
61/* Define a feature with unsigned value */
62#define U_ARM64_FTR_BITS(STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
63 __ARM64_FTR_BITS(FTR_UNSIGNED, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL)
64
3c739b57
SP		 65#define ARM64_FTR_END \
66 { \
67 .width = 0, \
68 }
69
70static struct arm64_ftr_bits ftr_id_aa64isar0[] = {
71 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 32, 32, 0),
72 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64ISAR0_RDM_SHIFT, 4, 0),
73 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 24, 4, 0),
74 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_ATOMICS_SHIFT, 4, 0),
75 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_CRC32_SHIFT, 4, 0),
76 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SHA2_SHIFT, 4, 0),
77 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SHA1_SHIFT, 4, 0),
78 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_AES_SHIFT, 4, 0),
79 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 0, 4, 0), /* RAZ */
80 ARM64_FTR_END,
81};
82
83static struct arm64_ftr_bits ftr_id_aa64pfr0[] = {
84 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 32, 32, 0),
85 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 28, 4, 0),
86 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64PFR0_GIC_SHIFT, 4, 0),
87 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_ASIMD_SHIFT, 4, ID_AA64PFR0_ASIMD_NI),
88 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_FP_SHIFT, 4, ID_AA64PFR0_FP_NI),
89 /* Linux doesn't care about the EL3 */
90 ARM64_FTR_BITS(FTR_NONSTRICT, FTR_EXACT, ID_AA64PFR0_EL3_SHIFT, 4, 0),
91 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64PFR0_EL2_SHIFT, 4, 0),
92 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64PFR0_EL1_SHIFT, 4, ID_AA64PFR0_EL1_64BIT_ONLY),
93 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64PFR0_EL0_SHIFT, 4, ID_AA64PFR0_EL0_64BIT_ONLY),
94 ARM64_FTR_END,
95};
96
97static struct arm64_ftr_bits ftr_id_aa64mmfr0[] = {
98 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 32, 32, 0),
99 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_TGRAN4_SHIFT, 4, ID_AA64MMFR0_TGRAN4_NI),
100 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_TGRAN64_SHIFT, 4, ID_AA64MMFR0_TGRAN64_NI),
101 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_TGRAN16_SHIFT, 4, ID_AA64MMFR0_TGRAN16_NI),
102 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_BIGENDEL0_SHIFT, 4, 0),
103 /* Linux shouldn't care about secure memory */
104 ARM64_FTR_BITS(FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_SNSMEM_SHIFT, 4, 0),
105 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_BIGENDEL_SHIFT, 4, 0),
106 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_ASID_SHIFT, 4, 0),
107 /*
108 * Differing PARange is fine as long as all peripherals and memory are mapped
109 * within the minimum PARange of all CPUs
110 */
4f0a606b 111 U_ARM64_FTR_BITS(FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_PARANGE_SHIFT, 4, 0),
3c739b57
SP		 112 ARM64_FTR_END,
113};
114
115static struct arm64_ftr_bits ftr_id_aa64mmfr1[] = {
116 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 32, 32, 0),
117 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_PAN_SHIFT, 4, 0),
118 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_LOR_SHIFT, 4, 0),
119 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_HPD_SHIFT, 4, 0),
120 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_VHE_SHIFT, 4, 0),
121 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_VMIDBITS_SHIFT, 4, 0),
122 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_HADBS_SHIFT, 4, 0),
123 ARM64_FTR_END,
124};
125
126static struct arm64_ftr_bits ftr_ctr[] = {
4f0a606b 127 U_ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 31, 1, 1), /* RAO */
3c739b57 128 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 28, 3, 0),
4f0a606b
SP		 129 U_ARM64_FTR_BITS(FTR_STRICT, FTR_HIGHER_SAFE, 24, 4, 0), /* CWG */
130 U_ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 20, 4, 0), /* ERG */
131 U_ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 1), /* DminLine */
3c739b57
SP		 132 /*
133 * Linux can handle differing I-cache policies. Userspace JITs will
134 * make use of *minLine
135 */
4f0a606b 136 U_ARM64_FTR_BITS(FTR_NONSTRICT, FTR_EXACT, 14, 2, 0), /* L1Ip */
3c739b57 137 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 4, 10, 0), /* RAZ */
4f0a606b 138 U_ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0), /* IminLine */
3c739b57
SP		 139 ARM64_FTR_END,
140};
141
142static struct arm64_ftr_bits ftr_id_mmfr0[] = {
143 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 28, 4, 0), /* InnerShr */
144 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 24, 4, 0), /* FCSE */
145 ARM64_FTR_BITS(FTR_NONSTRICT, FTR_LOWER_SAFE, 20, 4, 0), /* AuxReg */
146 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 16, 4, 0), /* TCM */
147 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 12, 4, 0), /* ShareLvl */
148 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 8, 4, 0), /* OuterShr */
149 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 4, 4, 0), /* PMSA */
150 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 0, 4, 0), /* VMSA */
151 ARM64_FTR_END,
152};
153
154static struct arm64_ftr_bits ftr_id_aa64dfr0[] = {
155 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 32, 32, 0),
4f0a606b
SP		 156 U_ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_CTX_CMPS_SHIFT, 4, 0),
157 U_ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_WRPS_SHIFT, 4, 0),
158 U_ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_BRPS_SHIFT, 4, 0),
159 U_ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64DFR0_PMUVER_SHIFT, 4, 0),
160 U_ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64DFR0_TRACEVER_SHIFT, 4, 0),
161 U_ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64DFR0_DEBUGVER_SHIFT, 4, 0x6),
3c739b57
SP		 162 ARM64_FTR_END,
163};
164
165static struct arm64_ftr_bits ftr_mvfr2[] = {
166 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 8, 24, 0), /* RAZ */
167 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 4, 4, 0), /* FPMisc */
168 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 0, 4, 0), /* SIMDMisc */
169 ARM64_FTR_END,
170};
171
172static struct arm64_ftr_bits ftr_dczid[] = {
173 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 5, 27, 0), /* RAZ */
174 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 4, 1, 1), /* DZP */
175 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0), /* BS */
176 ARM64_FTR_END,
177};
178
179
180static struct arm64_ftr_bits ftr_id_isar5[] = {
181 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_ISAR5_RDM_SHIFT, 4, 0),
182 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 20, 4, 0), /* RAZ */
183 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_ISAR5_CRC32_SHIFT, 4, 0),
184 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_ISAR5_SHA2_SHIFT, 4, 0),
185 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_ISAR5_SHA1_SHIFT, 4, 0),
186 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_ISAR5_AES_SHIFT, 4, 0),
187 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_ISAR5_SEVL_SHIFT, 4, 0),
188 ARM64_FTR_END,
189};
190
191static struct arm64_ftr_bits ftr_id_mmfr4[] = {
192 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 8, 24, 0), /* RAZ */
193 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 4, 4, 0), /* ac2 */
194 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 0, 4, 0), /* RAZ */
195 ARM64_FTR_END,
196};
197
198static struct arm64_ftr_bits ftr_id_pfr0[] = {
199 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 16, 16, 0), /* RAZ */
200 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 12, 4, 0), /* State3 */
201 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 8, 4, 0), /* State2 */
202 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 4, 4, 0), /* State1 */
203 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 0, 4, 0), /* State0 */
204 ARM64_FTR_END,
205};
206
207/*
208 * Common ftr bits for a 32bit register with all hidden, strict
209 * attributes, with 4bit feature fields and a default safe value of
210 * 0. Covers the following 32bit registers:
211 * id_isar[0-4], id_mmfr[1-3], id_pfr1, mvfr[0-1]
212 */
213static struct arm64_ftr_bits ftr_generic_32bits[] = {
214 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 28, 4, 0),
215 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 24, 4, 0),
216 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 20, 4, 0),
217 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 0),
218 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 12, 4, 0),
219 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 8, 4, 0),
220 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0),
221 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0),
222 ARM64_FTR_END,
223};
224
225static struct arm64_ftr_bits ftr_generic[] = {
226 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 0, 64, 0),
227 ARM64_FTR_END,
228};
229
230static struct arm64_ftr_bits ftr_generic32[] = {
231 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 0, 32, 0),
232 ARM64_FTR_END,
233};
234
235static struct arm64_ftr_bits ftr_aa64raz[] = {
236 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 0, 64, 0),
237 ARM64_FTR_END,
238};
239
240#define ARM64_FTR_REG(id, table) \
241 { \
242 .sys_id = id, \
243 .name = #id, \
244 .ftr_bits = &((table)[0]), \
245 }
246
247static struct arm64_ftr_reg arm64_ftr_regs[] = {
248
249 /* Op1 = 0, CRn = 0, CRm = 1 */
250 ARM64_FTR_REG(SYS_ID_PFR0_EL1, ftr_id_pfr0),
251 ARM64_FTR_REG(SYS_ID_PFR1_EL1, ftr_generic_32bits),
252 ARM64_FTR_REG(SYS_ID_DFR0_EL1, ftr_generic_32bits),
253 ARM64_FTR_REG(SYS_ID_MMFR0_EL1, ftr_id_mmfr0),
254 ARM64_FTR_REG(SYS_ID_MMFR1_EL1, ftr_generic_32bits),
255 ARM64_FTR_REG(SYS_ID_MMFR2_EL1, ftr_generic_32bits),
256 ARM64_FTR_REG(SYS_ID_MMFR3_EL1, ftr_generic_32bits),
257
258 /* Op1 = 0, CRn = 0, CRm = 2 */
259 ARM64_FTR_REG(SYS_ID_ISAR0_EL1, ftr_generic_32bits),
260 ARM64_FTR_REG(SYS_ID_ISAR1_EL1, ftr_generic_32bits),
261 ARM64_FTR_REG(SYS_ID_ISAR2_EL1, ftr_generic_32bits),
262 ARM64_FTR_REG(SYS_ID_ISAR3_EL1, ftr_generic_32bits),
263 ARM64_FTR_REG(SYS_ID_ISAR4_EL1, ftr_generic_32bits),
264 ARM64_FTR_REG(SYS_ID_ISAR5_EL1, ftr_id_isar5),
265 ARM64_FTR_REG(SYS_ID_MMFR4_EL1, ftr_id_mmfr4),
266
267 /* Op1 = 0, CRn = 0, CRm = 3 */
268 ARM64_FTR_REG(SYS_MVFR0_EL1, ftr_generic_32bits),
269 ARM64_FTR_REG(SYS_MVFR1_EL1, ftr_generic_32bits),
270 ARM64_FTR_REG(SYS_MVFR2_EL1, ftr_mvfr2),
271
272 /* Op1 = 0, CRn = 0, CRm = 4 */
273 ARM64_FTR_REG(SYS_ID_AA64PFR0_EL1, ftr_id_aa64pfr0),
274 ARM64_FTR_REG(SYS_ID_AA64PFR1_EL1, ftr_aa64raz),
275
276 /* Op1 = 0, CRn = 0, CRm = 5 */
277 ARM64_FTR_REG(SYS_ID_AA64DFR0_EL1, ftr_id_aa64dfr0),
278 ARM64_FTR_REG(SYS_ID_AA64DFR1_EL1, ftr_generic),
279
280 /* Op1 = 0, CRn = 0, CRm = 6 */
281 ARM64_FTR_REG(SYS_ID_AA64ISAR0_EL1, ftr_id_aa64isar0),
282 ARM64_FTR_REG(SYS_ID_AA64ISAR1_EL1, ftr_aa64raz),
283
284 /* Op1 = 0, CRn = 0, CRm = 7 */
285 ARM64_FTR_REG(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0),
286 ARM64_FTR_REG(SYS_ID_AA64MMFR1_EL1, ftr_id_aa64mmfr1),
287
288 /* Op1 = 3, CRn = 0, CRm = 0 */
289 ARM64_FTR_REG(SYS_CTR_EL0, ftr_ctr),
290 ARM64_FTR_REG(SYS_DCZID_EL0, ftr_dczid),
291
292 /* Op1 = 3, CRn = 14, CRm = 0 */
293 ARM64_FTR_REG(SYS_CNTFRQ_EL0, ftr_generic32),
294};
295
296static int search_cmp_ftr_reg(const void *id, const void *regp)
297{
298 return (int)(unsigned long)id - (int)((const struct arm64_ftr_reg *)regp)->sys_id;
299}
300
301/*
302 * get_arm64_ftr_reg - Lookup a feature register entry using its
303 * sys_reg() encoding. With the array arm64_ftr_regs sorted in the
304 * ascending order of sys_id , we use binary search to find a matching
305 * entry.
306 *
307 * returns - Upon success, matching ftr_reg entry for id.
308 * - NULL on failure. It is upto the caller to decide
309 * the impact of a failure.
310 */
311static struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id)
312{
313 return bsearch((const void *)(unsigned long)sys_id,
314 arm64_ftr_regs,
315 ARRAY_SIZE(arm64_ftr_regs),
316 sizeof(arm64_ftr_regs[0]),
317 search_cmp_ftr_reg);
318}
319
320static u64 arm64_ftr_set_value(struct arm64_ftr_bits *ftrp, s64 reg, s64 ftr_val)
321{
322 u64 mask = arm64_ftr_mask(ftrp);
323
324 reg &= ~mask;
325 reg |= (ftr_val << ftrp->shift) & mask;
326 return reg;
327}
328
329static s64 arm64_ftr_safe_value(struct arm64_ftr_bits *ftrp, s64 new, s64 cur)
330{
331 s64 ret = 0;
332
333 switch (ftrp->type) {
334 case FTR_EXACT:
335 ret = ftrp->safe_val;
336 break;
337 case FTR_LOWER_SAFE:
338 ret = new < cur ? new : cur;
339 break;
340 case FTR_HIGHER_SAFE:
341 ret = new > cur ? new : cur;
342 break;
343 default:
344 BUG();
345 }
346
347 return ret;
348}
349
350static int __init sort_cmp_ftr_regs(const void *a, const void *b)
351{
352 return ((const struct arm64_ftr_reg *)a)->sys_id -
353 ((const struct arm64_ftr_reg *)b)->sys_id;
354}
355
356static void __init swap_ftr_regs(void *a, void *b, int size)
357{
358 struct arm64_ftr_reg tmp = *(struct arm64_ftr_reg *)a;
359 *(struct arm64_ftr_reg *)a = *(struct arm64_ftr_reg *)b;
360 *(struct arm64_ftr_reg *)b = tmp;
361}
362
363static void __init sort_ftr_regs(void)
364{
365 /* Keep the array sorted so that we can do the binary search */
366 sort(arm64_ftr_regs,
367 ARRAY_SIZE(arm64_ftr_regs),
368 sizeof(arm64_ftr_regs[0]),
369 sort_cmp_ftr_regs,
370 swap_ftr_regs);
371}
372
373/*
374 * Initialise the CPU feature register from Boot CPU values.
375 * Also initiliases the strict_mask for the register.
376 */
377static void __init init_cpu_ftr_reg(u32 sys_reg, u64 new)
378{
379 u64 val = 0;
380 u64 strict_mask = ~0x0ULL;
381 struct arm64_ftr_bits *ftrp;
382 struct arm64_ftr_reg *reg = get_arm64_ftr_reg(sys_reg);
383
384 BUG_ON(!reg);
385
386 for (ftrp = reg->ftr_bits; ftrp->width; ftrp++) {
387 s64 ftr_new = arm64_ftr_value(ftrp, new);
388
389 val = arm64_ftr_set_value(ftrp, val, ftr_new);
390 if (!ftrp->strict)
391 strict_mask &= ~arm64_ftr_mask(ftrp);
392 }
393 reg->sys_val = val;
394 reg->strict_mask = strict_mask;
395}
396
397void __init init_cpu_features(struct cpuinfo_arm64 *info)
398{
399 /* Before we start using the tables, make sure it is sorted */
400 sort_ftr_regs();
401
402 init_cpu_ftr_reg(SYS_CTR_EL0, info->reg_ctr);
403 init_cpu_ftr_reg(SYS_DCZID_EL0, info->reg_dczid);
404 init_cpu_ftr_reg(SYS_CNTFRQ_EL0, info->reg_cntfrq);
405 init_cpu_ftr_reg(SYS_ID_AA64DFR0_EL1, info->reg_id_aa64dfr0);
406 init_cpu_ftr_reg(SYS_ID_AA64DFR1_EL1, info->reg_id_aa64dfr1);
407 init_cpu_ftr_reg(SYS_ID_AA64ISAR0_EL1, info->reg_id_aa64isar0);
408 init_cpu_ftr_reg(SYS_ID_AA64ISAR1_EL1, info->reg_id_aa64isar1);
409 init_cpu_ftr_reg(SYS_ID_AA64MMFR0_EL1, info->reg_id_aa64mmfr0);
410 init_cpu_ftr_reg(SYS_ID_AA64MMFR1_EL1, info->reg_id_aa64mmfr1);
411 init_cpu_ftr_reg(SYS_ID_AA64PFR0_EL1, info->reg_id_aa64pfr0);
412 init_cpu_ftr_reg(SYS_ID_AA64PFR1_EL1, info->reg_id_aa64pfr1);
413 init_cpu_ftr_reg(SYS_ID_DFR0_EL1, info->reg_id_dfr0);
414 init_cpu_ftr_reg(SYS_ID_ISAR0_EL1, info->reg_id_isar0);
415 init_cpu_ftr_reg(SYS_ID_ISAR1_EL1, info->reg_id_isar1);
416 init_cpu_ftr_reg(SYS_ID_ISAR2_EL1, info->reg_id_isar2);
417 init_cpu_ftr_reg(SYS_ID_ISAR3_EL1, info->reg_id_isar3);
418 init_cpu_ftr_reg(SYS_ID_ISAR4_EL1, info->reg_id_isar4);
419 init_cpu_ftr_reg(SYS_ID_ISAR5_EL1, info->reg_id_isar5);
420 init_cpu_ftr_reg(SYS_ID_MMFR0_EL1, info->reg_id_mmfr0);
421 init_cpu_ftr_reg(SYS_ID_MMFR1_EL1, info->reg_id_mmfr1);
422 init_cpu_ftr_reg(SYS_ID_MMFR2_EL1, info->reg_id_mmfr2);
423 init_cpu_ftr_reg(SYS_ID_MMFR3_EL1, info->reg_id_mmfr3);
424 init_cpu_ftr_reg(SYS_ID_PFR0_EL1, info->reg_id_pfr0);
425 init_cpu_ftr_reg(SYS_ID_PFR1_EL1, info->reg_id_pfr1);
426 init_cpu_ftr_reg(SYS_MVFR0_EL1, info->reg_mvfr0);
427 init_cpu_ftr_reg(SYS_MVFR1_EL1, info->reg_mvfr1);
428 init_cpu_ftr_reg(SYS_MVFR2_EL1, info->reg_mvfr2);
3c739b57
SP		 429}
430
3086d391 431static void update_cpu_ftr_reg(struct arm64_ftr_reg *reg, u64 new)
3c739b57
SP		 432{
433 struct arm64_ftr_bits *ftrp;
3c739b57
SP		 434
435 for (ftrp = reg->ftr_bits; ftrp->width; ftrp++) {
436 s64 ftr_cur = arm64_ftr_value(ftrp, reg->sys_val);
437 s64 ftr_new = arm64_ftr_value(ftrp, new);
438
439 if (ftr_cur == ftr_new)
440 continue;
441 /* Find a safe value */
442 ftr_new = arm64_ftr_safe_value(ftrp, ftr_new, ftr_cur);
443 reg->sys_val = arm64_ftr_set_value(ftrp, reg->sys_val, ftr_new);
444 }
445
446}
447
3086d391 448static int check_update_ftr_reg(u32 sys_id, int cpu, u64 val, u64 boot)
cdcf817b 449{
3086d391
SP		 450 struct arm64_ftr_reg *regp = get_arm64_ftr_reg(sys_id);
451
452 BUG_ON(!regp);
453 update_cpu_ftr_reg(regp, val);
454 if ((boot & regp->strict_mask) == (val & regp->strict_mask))
455 return 0;
456 pr_warn("SANITY CHECK: Unexpected variation in %s. Boot CPU: %#016llx, CPU%d: %#016llx\n",
457 regp->name, boot, cpu, val);
458 return 1;
459}
460
461/*
462 * Update system wide CPU feature registers with the values from a
463 * non-boot CPU. Also performs SANITY checks to make sure that there
464 * aren't any insane variations from that of the boot CPU.
465 */
466void update_cpu_features(int cpu,
467 struct cpuinfo_arm64 *info,
468 struct cpuinfo_arm64 *boot)
469{
470 int taint = 0;
471
472 /*
473 * The kernel can handle differing I-cache policies, but otherwise
474 * caches should look identical. Userspace JITs will make use of
475 * *minLine.
476 */
477 taint |= check_update_ftr_reg(SYS_CTR_EL0, cpu,
478 info->reg_ctr, boot->reg_ctr);
479
480 /*
481 * Userspace may perform DC ZVA instructions. Mismatched block sizes
482 * could result in too much or too little memory being zeroed if a
483 * process is preempted and migrated between CPUs.
484 */
485 taint |= check_update_ftr_reg(SYS_DCZID_EL0, cpu,
486 info->reg_dczid, boot->reg_dczid);
487
488 /* If different, timekeeping will be broken (especially with KVM) */
489 taint |= check_update_ftr_reg(SYS_CNTFRQ_EL0, cpu,
490 info->reg_cntfrq, boot->reg_cntfrq);
491
492 /*
493 * The kernel uses self-hosted debug features and expects CPUs to
494 * support identical debug features. We presently need CTX_CMPs, WRPs,
495 * and BRPs to be identical.
496 * ID_AA64DFR1 is currently RES0.
497 */
498 taint |= check_update_ftr_reg(SYS_ID_AA64DFR0_EL1, cpu,
499 info->reg_id_aa64dfr0, boot->reg_id_aa64dfr0);
500 taint |= check_update_ftr_reg(SYS_ID_AA64DFR1_EL1, cpu,
501 info->reg_id_aa64dfr1, boot->reg_id_aa64dfr1);
502 /*
503 * Even in big.LITTLE, processors should be identical instruction-set
504 * wise.
505 */
506 taint |= check_update_ftr_reg(SYS_ID_AA64ISAR0_EL1, cpu,
507 info->reg_id_aa64isar0, boot->reg_id_aa64isar0);
508 taint |= check_update_ftr_reg(SYS_ID_AA64ISAR1_EL1, cpu,
509 info->reg_id_aa64isar1, boot->reg_id_aa64isar1);
510
511 /*
512 * Differing PARange support is fine as long as all peripherals and
513 * memory are mapped within the minimum PARange of all CPUs.
514 * Linux should not care about secure memory.
515 */
516 taint |= check_update_ftr_reg(SYS_ID_AA64MMFR0_EL1, cpu,
517 info->reg_id_aa64mmfr0, boot->reg_id_aa64mmfr0);
518 taint |= check_update_ftr_reg(SYS_ID_AA64MMFR1_EL1, cpu,
519 info->reg_id_aa64mmfr1, boot->reg_id_aa64mmfr1);
520
521 /*
522 * EL3 is not our concern.
523 * ID_AA64PFR1 is currently RES0.
524 */
525 taint |= check_update_ftr_reg(SYS_ID_AA64PFR0_EL1, cpu,
526 info->reg_id_aa64pfr0, boot->reg_id_aa64pfr0);
527 taint |= check_update_ftr_reg(SYS_ID_AA64PFR1_EL1, cpu,
528 info->reg_id_aa64pfr1, boot->reg_id_aa64pfr1);
529
530 /*
531 * If we have AArch32, we care about 32-bit features for compat. These
532 * registers should be RES0 otherwise.
533 */
534 taint |= check_update_ftr_reg(SYS_ID_DFR0_EL1, cpu,
535 info->reg_id_dfr0, boot->reg_id_dfr0);
536 taint |= check_update_ftr_reg(SYS_ID_ISAR0_EL1, cpu,
537 info->reg_id_isar0, boot->reg_id_isar0);
538 taint |= check_update_ftr_reg(SYS_ID_ISAR1_EL1, cpu,
539 info->reg_id_isar1, boot->reg_id_isar1);
540 taint |= check_update_ftr_reg(SYS_ID_ISAR2_EL1, cpu,
541 info->reg_id_isar2, boot->reg_id_isar2);
542 taint |= check_update_ftr_reg(SYS_ID_ISAR3_EL1, cpu,
543 info->reg_id_isar3, boot->reg_id_isar3);
544 taint |= check_update_ftr_reg(SYS_ID_ISAR4_EL1, cpu,
545 info->reg_id_isar4, boot->reg_id_isar4);
546 taint |= check_update_ftr_reg(SYS_ID_ISAR5_EL1, cpu,
547 info->reg_id_isar5, boot->reg_id_isar5);
548
549 /*
550 * Regardless of the value of the AuxReg field, the AIFSR, ADFSR, and
551 * ACTLR formats could differ across CPUs and therefore would have to
552 * be trapped for virtualization anyway.
553 */
554 taint |= check_update_ftr_reg(SYS_ID_MMFR0_EL1, cpu,
555 info->reg_id_mmfr0, boot->reg_id_mmfr0);
556 taint |= check_update_ftr_reg(SYS_ID_MMFR1_EL1, cpu,
557 info->reg_id_mmfr1, boot->reg_id_mmfr1);
558 taint |= check_update_ftr_reg(SYS_ID_MMFR2_EL1, cpu,
559 info->reg_id_mmfr2, boot->reg_id_mmfr2);
560 taint |= check_update_ftr_reg(SYS_ID_MMFR3_EL1, cpu,
561 info->reg_id_mmfr3, boot->reg_id_mmfr3);
562 taint |= check_update_ftr_reg(SYS_ID_PFR0_EL1, cpu,
563 info->reg_id_pfr0, boot->reg_id_pfr0);
564 taint |= check_update_ftr_reg(SYS_ID_PFR1_EL1, cpu,
565 info->reg_id_pfr1, boot->reg_id_pfr1);
566 taint |= check_update_ftr_reg(SYS_MVFR0_EL1, cpu,
567 info->reg_mvfr0, boot->reg_mvfr0);
568 taint |= check_update_ftr_reg(SYS_MVFR1_EL1, cpu,
569 info->reg_mvfr1, boot->reg_mvfr1);
570 taint |= check_update_ftr_reg(SYS_MVFR2_EL1, cpu,
571 info->reg_mvfr2, boot->reg_mvfr2);
572
573 /*
574 * Mismatched CPU features are a recipe for disaster. Don't even
575 * pretend to support them.
576 */
577 WARN_TAINT_ONCE(taint, TAINT_CPU_OUT_OF_SPEC,
578 "Unsupported CPU feature variation.\n");
cdcf817b
SP		 579}
580
b3f15378
SP		 581u64 read_system_reg(u32 id)
582{
583 struct arm64_ftr_reg *regp = get_arm64_ftr_reg(id);
584
585 /* We shouldn't get a request for an unsupported register */
586 BUG_ON(!regp);
587 return regp->sys_val;
588}
359b7064 589
963fcd40
MZ		 590#include <linux/irqchip/arm-gic-v3.h>
591
18ffa046
JM		 592static bool
593feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry)
594{
595 int val = cpuid_feature_extract_field(reg, entry->field_pos);
596
597 return val >= entry->min_field_value;
598}
599
da8d02d1
SP		 600static bool
601has_cpuid_feature(const struct arm64_cpu_capabilities *entry)
602{
603 u64 val;
94a9e04a 604
da8d02d1
SP		 605 val = read_system_reg(entry->sys_reg);
606 return feature_matches(val, entry);
607}
338d4f49 608
963fcd40
MZ		 609static bool has_useable_gicv3_cpuif(const struct arm64_cpu_capabilities *entry)
610{
611 bool has_sre;
612
2dc10ad8 613 if (!has_cpuid_feature(entry))
963fcd40
MZ		 614 return false;
615
616 has_sre = gic_enable_sre();
617 if (!has_sre)
618 pr_warn_once("%s present but disabled by higher exception level\n",
619 entry->desc);
620
621 return has_sre;
622}
623
d5370f75
WD		 624static bool has_no_hw_prefetch(const struct arm64_cpu_capabilities *entry)
625{
626 u32 midr = read_cpuid_id();
627 u32 rv_min, rv_max;
628
629 /* Cavium ThunderX pass 1.x and 2.x */
630 rv_min = 0;
631 rv_max = (1 << MIDR_VARIANT_SHIFT) | MIDR_REVISION_MASK;
632
633 return MIDR_IS_CPU_MODEL_RANGE(midr, MIDR_THUNDERX, rv_min, rv_max);
634}
635
359b7064 636static const struct arm64_cpu_capabilities arm64_features[] = {
94a9e04a
MZ		 637 {
638 .desc = "GIC system register CPU interface",
639 .capability = ARM64_HAS_SYSREG_GIC_CPUIF,
963fcd40 640 .matches = has_useable_gicv3_cpuif,
da8d02d1
SP		 641 .sys_reg = SYS_ID_AA64PFR0_EL1,
642 .field_pos = ID_AA64PFR0_GIC_SHIFT,
18ffa046 643 .min_field_value = 1,
94a9e04a 644 },
338d4f49
JM		 645#ifdef CONFIG_ARM64_PAN
646 {
647 .desc = "Privileged Access Never",
648 .capability = ARM64_HAS_PAN,
da8d02d1
SP		 649 .matches = has_cpuid_feature,
650 .sys_reg = SYS_ID_AA64MMFR1_EL1,
651 .field_pos = ID_AA64MMFR1_PAN_SHIFT,
338d4f49
JM		 652 .min_field_value = 1,
653 .enable = cpu_enable_pan,
654 },
655#endif /* CONFIG_ARM64_PAN */
2e94da13
WD		 656#if defined(CONFIG_AS_LSE) && defined(CONFIG_ARM64_LSE_ATOMICS)
657 {
658 .desc = "LSE atomic instructions",
659 .capability = ARM64_HAS_LSE_ATOMICS,
da8d02d1
SP		 660 .matches = has_cpuid_feature,
661 .sys_reg = SYS_ID_AA64ISAR0_EL1,
662 .field_pos = ID_AA64ISAR0_ATOMICS_SHIFT,
2e94da13
WD		 663 .min_field_value = 2,
664 },
665#endif /* CONFIG_AS_LSE && CONFIG_ARM64_LSE_ATOMICS */
d5370f75
WD		 666 {
667 .desc = "Software prefetching using PRFM",
668 .capability = ARM64_HAS_NO_HW_PREFETCH,
669 .matches = has_no_hw_prefetch,
670 },
359b7064
MZ		 671 {},
672};
673
37b01d53
SP		 674#define HWCAP_CAP(reg, field, min_value, type, cap) \
675 { \
676 .desc = #cap, \
677 .matches = has_cpuid_feature, \
678 .sys_reg = reg, \
679 .field_pos = field, \
680 .min_field_value = min_value, \
681 .hwcap_type = type, \
682 .hwcap = cap, \
683 }
684
685static const struct arm64_cpu_capabilities arm64_hwcaps[] = {
686 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_AES_SHIFT, 2, CAP_HWCAP, HWCAP_PMULL),
687 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_AES_SHIFT, 1, CAP_HWCAP, HWCAP_AES),
688 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA1_SHIFT, 1, CAP_HWCAP, HWCAP_SHA1),
689 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA2_SHIFT, 1, CAP_HWCAP, HWCAP_SHA2),
690 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_CRC32_SHIFT, 1, CAP_HWCAP, HWCAP_CRC32),
691 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_ATOMICS_SHIFT, 2, CAP_HWCAP, HWCAP_ATOMICS),
fe80f9f2
SP		 692 HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, 0, CAP_HWCAP, HWCAP_FP),
693 HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, 0, CAP_HWCAP, HWCAP_ASIMD),
37b01d53
SP		 694#ifdef CONFIG_COMPAT
695 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_AES_SHIFT, 2, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_PMULL),
696 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_AES_SHIFT, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_AES),
697 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_SHA1_SHIFT, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA1),
698 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_SHA2_SHIFT, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA2),
699 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_CRC32_SHIFT, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_CRC32),
700#endif
701 {},
702};
703
a7c61a34 704static void __init cap_set_hwcap(const struct arm64_cpu_capabilities *cap)
37b01d53
SP		 705{
706 switch (cap->hwcap_type) {
707 case CAP_HWCAP:
708 elf_hwcap |= cap->hwcap;
709 break;
710#ifdef CONFIG_COMPAT
711 case CAP_COMPAT_HWCAP:
712 compat_elf_hwcap |= (u32)cap->hwcap;
713 break;
714 case CAP_COMPAT_HWCAP2:
715 compat_elf_hwcap2 |= (u32)cap->hwcap;
716 break;
717#endif
718 default:
719 WARN_ON(1);
720 break;
721 }
722}
723
724/* Check if we have a particular HWCAP enabled */
3d6d1035 725static bool __maybe_unused cpus_have_hwcap(const struct arm64_cpu_capabilities *cap)
37b01d53
SP		 726{
727 bool rc;
728
729 switch (cap->hwcap_type) {
730 case CAP_HWCAP:
731 rc = (elf_hwcap & cap->hwcap) != 0;
732 break;
733#ifdef CONFIG_COMPAT
734 case CAP_COMPAT_HWCAP:
735 rc = (compat_elf_hwcap & (u32)cap->hwcap) != 0;
736 break;
737 case CAP_COMPAT_HWCAP2:
738 rc = (compat_elf_hwcap2 & (u32)cap->hwcap) != 0;
739 break;
740#endif
741 default:
742 WARN_ON(1);
743 rc = false;
744 }
745
746 return rc;
747}
748
a7c61a34 749static void __init setup_cpu_hwcaps(void)
37b01d53
SP		 750{
751 int i;
752 const struct arm64_cpu_capabilities *hwcaps = arm64_hwcaps;
753
754 for (i = 0; hwcaps[i].desc; i++)
755 if (hwcaps[i].matches(&hwcaps[i]))
756 cap_set_hwcap(&hwcaps[i]);
757}
758
ce8b602c 759void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps,
359b7064
MZ		 760 const char *info)
761{
762 int i;
763
764 for (i = 0; caps[i].desc; i++) {
765 if (!caps[i].matches(&caps[i]))
766 continue;
767
768 if (!cpus_have_cap(caps[i].capability))
769 pr_info("%s %s\n", info, caps[i].desc);
770 cpus_set_cap(caps[i].capability);
771 }
ce8b602c
SP		 772}
773
774/*
dbb4e152
SP		 775 * Run through the enabled capabilities and enable() it on all active
776 * CPUs
ce8b602c 777 */
a7c61a34
JZ		 778static void __init
779enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps)
ce8b602c
SP		 780{
781 int i;
1c076303 782
dbb4e152
SP		 783 for (i = 0; caps[i].desc; i++)
784 if (caps[i].enable && cpus_have_cap(caps[i].capability))
785 on_each_cpu(caps[i].enable, NULL, true);
786}
787
788#ifdef CONFIG_HOTPLUG_CPU
789
790/*
791 * Flag to indicate if we have computed the system wide
792 * capabilities based on the boot time active CPUs. This
793 * will be used to determine if a new booting CPU should
794 * go through the verification process to make sure that it
795 * supports the system capabilities, without using a hotplug
796 * notifier.
797 */
798static bool sys_caps_initialised;
799
800static inline void set_sys_caps_initialised(void)
801{
802 sys_caps_initialised = true;
803}
804
da8d02d1
SP		 805/*
806 * __raw_read_system_reg() - Used by a STARTING cpu before cpuinfo is populated.
807 */
808static u64 __raw_read_system_reg(u32 sys_id)
809{
810 switch (sys_id) {
0f54b14e
JM		 811 case SYS_ID_PFR0_EL1: return read_cpuid(SYS_ID_PFR0_EL1);
812 case SYS_ID_PFR1_EL1: return read_cpuid(SYS_ID_PFR1_EL1);
813 case SYS_ID_DFR0_EL1: return read_cpuid(SYS_ID_DFR0_EL1);
814 case SYS_ID_MMFR0_EL1: return read_cpuid(SYS_ID_MMFR0_EL1);
815 case SYS_ID_MMFR1_EL1: return read_cpuid(SYS_ID_MMFR1_EL1);
816 case SYS_ID_MMFR2_EL1: return read_cpuid(SYS_ID_MMFR2_EL1);
817 case SYS_ID_MMFR3_EL1: return read_cpuid(SYS_ID_MMFR3_EL1);
818 case SYS_ID_ISAR0_EL1: return read_cpuid(SYS_ID_ISAR0_EL1);
819 case SYS_ID_ISAR1_EL1: return read_cpuid(SYS_ID_ISAR1_EL1);
820 case SYS_ID_ISAR2_EL1: return read_cpuid(SYS_ID_ISAR2_EL1);
821 case SYS_ID_ISAR3_EL1: return read_cpuid(SYS_ID_ISAR3_EL1);
822 case SYS_ID_ISAR4_EL1: return read_cpuid(SYS_ID_ISAR4_EL1);
823 case SYS_ID_ISAR5_EL1: return read_cpuid(SYS_ID_ISAR4_EL1);
824 case SYS_MVFR0_EL1: return read_cpuid(SYS_MVFR0_EL1);
825 case SYS_MVFR1_EL1: return read_cpuid(SYS_MVFR1_EL1);
826 case SYS_MVFR2_EL1: return read_cpuid(SYS_MVFR2_EL1);
827
828 case SYS_ID_AA64PFR0_EL1: return read_cpuid(SYS_ID_AA64PFR0_EL1);
829 case SYS_ID_AA64PFR1_EL1: return read_cpuid(SYS_ID_AA64PFR0_EL1);
830 case SYS_ID_AA64DFR0_EL1: return read_cpuid(SYS_ID_AA64DFR0_EL1);
831 case SYS_ID_AA64DFR1_EL1: return read_cpuid(SYS_ID_AA64DFR0_EL1);
832 case SYS_ID_AA64MMFR0_EL1: return read_cpuid(SYS_ID_AA64MMFR0_EL1);
833 case SYS_ID_AA64MMFR1_EL1: return read_cpuid(SYS_ID_AA64MMFR1_EL1);
834 case SYS_ID_AA64ISAR0_EL1: return read_cpuid(SYS_ID_AA64ISAR0_EL1);
835 case SYS_ID_AA64ISAR1_EL1: return read_cpuid(SYS_ID_AA64ISAR1_EL1);
836
837 case SYS_CNTFRQ_EL0: return read_cpuid(SYS_CNTFRQ_EL0);
838 case SYS_CTR_EL0: return read_cpuid(SYS_CTR_EL0);
839 case SYS_DCZID_EL0: return read_cpuid(SYS_DCZID_EL0);
da8d02d1
SP		 840 default:
841 BUG();
842 return 0;
843 }
844}
845
dbb4e152
SP		 846/*
847 * Park the CPU which doesn't have the capability as advertised
848 * by the system.
849 */
850static void fail_incapable_cpu(char *cap_type,
851 const struct arm64_cpu_capabilities *cap)
852{
853 int cpu = smp_processor_id();
854
855 pr_crit("CPU%d: missing %s : %s\n", cpu, cap_type, cap->desc);
856 /* Mark this CPU absent */
857 set_cpu_present(cpu, 0);
858
859 /* Check if we can park ourselves */
860 if (cpu_ops[cpu] && cpu_ops[cpu]->cpu_die)
861 cpu_ops[cpu]->cpu_die(cpu);
862 asm(
863 "1: wfe\n"
864 " wfi\n"
865 " b 1b");
866}
1c076303 867
dbb4e152
SP		 868/*
869 * Run through the enabled system capabilities and enable() it on this CPU.
870 * The capabilities were decided based on the available CPUs at the boot time.
871 * Any new CPU should match the system wide status of the capability. If the
872 * new CPU doesn't have a capability which the system now has enabled, we
873 * cannot do anything to fix it up and could cause unexpected failures. So
874 * we park the CPU.
875 */
876void verify_local_cpu_capabilities(void)
877{
878 int i;
879 const struct arm64_cpu_capabilities *caps;
880
881 /*
882 * If we haven't computed the system capabilities, there is nothing
883 * to verify.
884 */
885 if (!sys_caps_initialised)
886 return;
887
888 caps = arm64_features;
1c076303 889 for (i = 0; caps[i].desc; i++) {
da8d02d1 890 if (!cpus_have_cap(caps[i].capability) || !caps[i].sys_reg)
dbb4e152
SP		 891 continue;
892 /*
893 * If the new CPU misses an advertised feature, we cannot proceed
894 * further, park the cpu.
895 */
da8d02d1 896 if (!feature_matches(__raw_read_system_reg(caps[i].sys_reg), &caps[i]))
dbb4e152
SP		 897 fail_incapable_cpu("arm64_features", &caps[i]);
898 if (caps[i].enable)
899 caps[i].enable(NULL);
1c076303 900 }
37b01d53
SP		 901
902 for (i = 0, caps = arm64_hwcaps; caps[i].desc; i++) {
903 if (!cpus_have_hwcap(&caps[i]))
904 continue;
905 if (!feature_matches(__raw_read_system_reg(caps[i].sys_reg), &caps[i]))
906 fail_incapable_cpu("arm64_hwcaps", &caps[i]);
907 }
359b7064
MZ		 908}
909
dbb4e152
SP		 910#else /* !CONFIG_HOTPLUG_CPU */
911
912static inline void set_sys_caps_initialised(void)
913{
914}
915
916#endif /* CONFIG_HOTPLUG_CPU */
917
a7c61a34 918static void __init setup_feature_capabilities(void)
359b7064 919{
ce8b602c
SP		 920 update_cpu_capabilities(arm64_features, "detected feature:");
921 enable_cpu_capabilities(arm64_features);
359b7064
MZ		 922}
923
9cdf8ec4 924void __init setup_cpu_features(void)
359b7064 925{
9cdf8ec4
SP		 926 u32 cwg;
927 int cls;
928
dbb4e152
SP		 929 /* Set the CPU feature capabilies */
930 setup_feature_capabilities();
37b01d53 931 setup_cpu_hwcaps();
dbb4e152
SP		 932
933 /* Advertise that we have computed the system capabilities */
934 set_sys_caps_initialised();
935
9cdf8ec4
SP		 936 /*
937 * Check for sane CTR_EL0.CWG value.
938 */
939 cwg = cache_type_cwg();
940 cls = cache_line_size();
941 if (!cwg)
942 pr_warn("No Cache Writeback Granule information, assuming cache line size %d\n",
943 cls);
944 if (L1_CACHE_BYTES < cls)
945 pr_warn("L1_CACHE_BYTES smaller than the Cache Writeback Granule (%d < %d)\n",
946 L1_CACHE_BYTES, cls);
359b7064 947}
Ubuntu Artful kernel mirror