1 // SPDX-License-Identifier: GPL-2.0+
3 * PowerPC Memory Protection Keys management
5 * Copyright 2017, Ram Pai, IBM Corporation.
9 #include <asm/mmu_context.h>
10 #include <asm/setup.h>
11 #include <linux/pkeys.h>
12 #include <linux/of_device.h>
14 DEFINE_STATIC_KEY_TRUE(pkey_disabled
);
15 int pkeys_total
; /* Total pkeys as per device tree */
16 u32 initial_allocation_mask
; /* Bits set for the initially allocated keys */
17 u32 reserved_allocation_mask
; /* Bits set for reserved keys */
18 static bool pkey_execute_disable_supported
;
19 static bool pkeys_devtree_defined
; /* property exported by device tree */
20 static u64 pkey_amr_mask
; /* Bits in AMR not to be touched */
21 static u64 pkey_iamr_mask
; /* Bits in AMR not to be touched */
22 static u64 pkey_uamor_mask
; /* Bits in UMOR not to be touched */
23 static int execute_only_key
= 2;
25 #define AMR_BITS_PER_PKEY 2
26 #define AMR_RD_BIT 0x1UL
27 #define AMR_WR_BIT 0x2UL
28 #define IAMR_EX_BIT 0x1UL
29 #define PKEY_REG_BITS (sizeof(u64)*8)
30 #define pkeyshift(pkey) (PKEY_REG_BITS - ((pkey+1) * AMR_BITS_PER_PKEY))
32 static void scan_pkey_feature(void)
35 struct device_node
*cpu
;
37 cpu
= of_find_node_by_type(NULL
, "cpu");
41 if (of_property_read_u32_array(cpu
,
42 "ibm,processor-storage-keys", vals
, 2))
46 * Since any pkey can be used for data or execute, we will just treat
47 * all keys as equal and track them as one entity.
49 pkeys_total
= vals
[0];
50 pkeys_devtree_defined
= true;
53 static inline bool pkey_mmu_enabled(void)
55 if (firmware_has_feature(FW_FEATURE_LPAR
))
58 return cpu_has_feature(CPU_FTR_PKEY
);
61 static int pkey_initialize(void)
66 * We define PKEY_DISABLE_EXECUTE in addition to the arch-neutral
67 * generic defines for PKEY_DISABLE_ACCESS and PKEY_DISABLE_WRITE.
68 * Ensure that the bits a distinct.
70 BUILD_BUG_ON(PKEY_DISABLE_EXECUTE
&
71 (PKEY_DISABLE_ACCESS
| PKEY_DISABLE_WRITE
));
74 * pkey_to_vmflag_bits() assumes that the pkey bits are contiguous
75 * in the vmaflag. Make sure that is really the case.
77 BUILD_BUG_ON(__builtin_clzl(ARCH_VM_PKEY_FLAGS
>> VM_PKEY_SHIFT
) +
78 __builtin_popcountl(ARCH_VM_PKEY_FLAGS
>> VM_PKEY_SHIFT
)
79 != (sizeof(u64
) * BITS_PER_BYTE
));
81 /* scan the device tree for pkey feature */
85 * Let's assume 32 pkeys on P8 bare metal, if its not defined by device
86 * tree. We make this exception since skiboot forgot to expose this
89 if (!pkeys_devtree_defined
&& !firmware_has_feature(FW_FEATURE_LPAR
) &&
90 cpu_has_feature(CPU_FTRS_POWER8
))
94 * Adjust the upper limit, based on the number of bits supported by
97 pkeys_total
= min_t(int, pkeys_total
,
98 ((ARCH_VM_PKEY_FLAGS
>> VM_PKEY_SHIFT
)+1));
100 if (!pkey_mmu_enabled() || radix_enabled() || !pkeys_total
)
101 static_branch_enable(&pkey_disabled
);
103 static_branch_disable(&pkey_disabled
);
105 if (static_branch_likely(&pkey_disabled
))
109 * The device tree cannot be relied to indicate support for
110 * execute_disable support. Instead we use a PVR check.
112 if (pvr_version_is(PVR_POWER7
) || pvr_version_is(PVR_POWER7p
))
113 pkey_execute_disable_supported
= false;
115 pkey_execute_disable_supported
= true;
117 #ifdef CONFIG_PPC_4K_PAGES
119 * The OS can manage only 8 pkeys due to its inability to represent them
120 * in the Linux 4K PTE.
122 os_reserved
= pkeys_total
- 8;
126 /* Bits are in LE format. */
127 reserved_allocation_mask
= (0x1 << 1) | (0x1 << execute_only_key
);
129 /* register mask is in BE format */
130 pkey_amr_mask
= ~0x0ul
;
131 pkey_amr_mask
&= ~(0x3ul
<< pkeyshift(0));
133 pkey_iamr_mask
= ~0x0ul
;
134 pkey_iamr_mask
&= ~(0x3ul
<< pkeyshift(0));
135 pkey_iamr_mask
&= ~(0x3ul
<< pkeyshift(execute_only_key
));
137 pkey_uamor_mask
= ~0x0ul
;
138 pkey_uamor_mask
&= ~(0x3ul
<< pkeyshift(0));
139 pkey_uamor_mask
&= ~(0x3ul
<< pkeyshift(execute_only_key
));
141 /* mark the rest of the keys as reserved and hence unavailable */
142 for (i
= (pkeys_total
- os_reserved
); i
< pkeys_total
; i
++) {
143 reserved_allocation_mask
|= (0x1 << i
);
144 pkey_uamor_mask
&= ~(0x3ul
<< pkeyshift(i
));
146 initial_allocation_mask
= reserved_allocation_mask
| (0x1 << 0);
148 if (unlikely((pkeys_total
- os_reserved
) <= execute_only_key
)) {
150 * Insufficient number of keys to support
151 * execute only key. Mark it unavailable.
152 * Any AMR, UAMOR, IAMR bit set for
153 * this key is irrelevant since this key
154 * can never be allocated.
156 execute_only_key
= -1;
162 arch_initcall(pkey_initialize
);
164 void pkey_mm_init(struct mm_struct
*mm
)
166 if (static_branch_likely(&pkey_disabled
))
168 mm_pkey_allocation_map(mm
) = initial_allocation_mask
;
169 mm
->context
.execute_only_pkey
= execute_only_key
;
172 static inline u64
read_amr(void)
174 return mfspr(SPRN_AMR
);
177 static inline void write_amr(u64 value
)
179 mtspr(SPRN_AMR
, value
);
182 static inline u64
read_iamr(void)
184 if (!likely(pkey_execute_disable_supported
))
187 return mfspr(SPRN_IAMR
);
190 static inline void write_iamr(u64 value
)
192 if (!likely(pkey_execute_disable_supported
))
195 mtspr(SPRN_IAMR
, value
);
198 static inline u64
read_uamor(void)
200 return mfspr(SPRN_UAMOR
);
203 static inline void write_uamor(u64 value
)
205 mtspr(SPRN_UAMOR
, value
);
208 static bool is_pkey_enabled(int pkey
)
210 u64 uamor
= read_uamor();
211 u64 pkey_bits
= 0x3ul
<< pkeyshift(pkey
);
212 u64 uamor_pkey_bits
= (uamor
& pkey_bits
);
215 * Both the bits in UAMOR corresponding to the key should be set or
218 WARN_ON(uamor_pkey_bits
&& (uamor_pkey_bits
!= pkey_bits
));
219 return !!(uamor_pkey_bits
);
222 static inline void init_amr(int pkey
, u8 init_bits
)
224 u64 new_amr_bits
= (((u64
)init_bits
& 0x3UL
) << pkeyshift(pkey
));
225 u64 old_amr
= read_amr() & ~((u64
)(0x3ul
) << pkeyshift(pkey
));
227 write_amr(old_amr
| new_amr_bits
);
230 static inline void init_iamr(int pkey
, u8 init_bits
)
232 u64 new_iamr_bits
= (((u64
)init_bits
& 0x1UL
) << pkeyshift(pkey
));
233 u64 old_iamr
= read_iamr() & ~((u64
)(0x1ul
) << pkeyshift(pkey
));
235 write_iamr(old_iamr
| new_iamr_bits
);
239 * Set the access rights in AMR IAMR and UAMOR registers for @pkey to that
240 * specified in @init_val.
242 int __arch_set_user_pkey_access(struct task_struct
*tsk
, int pkey
,
243 unsigned long init_val
)
245 u64 new_amr_bits
= 0x0ul
;
246 u64 new_iamr_bits
= 0x0ul
;
248 if (!is_pkey_enabled(pkey
))
251 if (init_val
& PKEY_DISABLE_EXECUTE
) {
252 if (!pkey_execute_disable_supported
)
254 new_iamr_bits
|= IAMR_EX_BIT
;
256 init_iamr(pkey
, new_iamr_bits
);
258 /* Set the bits we need in AMR: */
259 if (init_val
& PKEY_DISABLE_ACCESS
)
260 new_amr_bits
|= AMR_RD_BIT
| AMR_WR_BIT
;
261 else if (init_val
& PKEY_DISABLE_WRITE
)
262 new_amr_bits
|= AMR_WR_BIT
;
264 init_amr(pkey
, new_amr_bits
);
268 void thread_pkey_regs_save(struct thread_struct
*thread
)
270 if (static_branch_likely(&pkey_disabled
))
274 * TODO: Skip saving registers if @thread hasn't used any keys yet.
276 thread
->amr
= read_amr();
277 thread
->iamr
= read_iamr();
278 thread
->uamor
= read_uamor();
281 void thread_pkey_regs_restore(struct thread_struct
*new_thread
,
282 struct thread_struct
*old_thread
)
284 if (static_branch_likely(&pkey_disabled
))
287 if (old_thread
->amr
!= new_thread
->amr
)
288 write_amr(new_thread
->amr
);
289 if (old_thread
->iamr
!= new_thread
->iamr
)
290 write_iamr(new_thread
->iamr
);
291 if (old_thread
->uamor
!= new_thread
->uamor
)
292 write_uamor(new_thread
->uamor
);
295 void thread_pkey_regs_init(struct thread_struct
*thread
)
297 if (static_branch_likely(&pkey_disabled
))
300 thread
->amr
= pkey_amr_mask
;
301 thread
->iamr
= pkey_iamr_mask
;
302 thread
->uamor
= pkey_uamor_mask
;
304 write_uamor(pkey_uamor_mask
);
305 write_amr(pkey_amr_mask
);
306 write_iamr(pkey_iamr_mask
);
309 static inline bool pkey_allows_readwrite(int pkey
)
311 int pkey_shift
= pkeyshift(pkey
);
313 if (!is_pkey_enabled(pkey
))
316 return !(read_amr() & ((AMR_RD_BIT
|AMR_WR_BIT
) << pkey_shift
));
319 int __execute_only_pkey(struct mm_struct
*mm
)
321 return mm
->context
.execute_only_pkey
;
324 static inline bool vma_is_pkey_exec_only(struct vm_area_struct
*vma
)
326 /* Do this check first since the vm_flags should be hot */
327 if ((vma
->vm_flags
& (VM_READ
| VM_WRITE
| VM_EXEC
)) != VM_EXEC
)
330 return (vma_pkey(vma
) == vma
->vm_mm
->context
.execute_only_pkey
);
334 * This should only be called for *plain* mprotect calls.
336 int __arch_override_mprotect_pkey(struct vm_area_struct
*vma
, int prot
,
340 * If the currently associated pkey is execute-only, but the requested
341 * protection is not execute-only, move it back to the default pkey.
343 if (vma_is_pkey_exec_only(vma
) && (prot
!= PROT_EXEC
))
347 * The requested protection is execute-only. Hence let's use an
350 if (prot
== PROT_EXEC
) {
351 pkey
= execute_only_pkey(vma
->vm_mm
);
356 /* Nothing to override. */
357 return vma_pkey(vma
);
360 static bool pkey_access_permitted(int pkey
, bool write
, bool execute
)
365 if (!is_pkey_enabled(pkey
))
368 pkey_shift
= pkeyshift(pkey
);
369 if (execute
&& !(read_iamr() & (IAMR_EX_BIT
<< pkey_shift
)))
372 amr
= read_amr(); /* Delay reading amr until absolutely needed */
373 return ((!write
&& !(amr
& (AMR_RD_BIT
<< pkey_shift
))) ||
374 (write
&& !(amr
& (AMR_WR_BIT
<< pkey_shift
))));
377 bool arch_pte_access_permitted(u64 pte
, bool write
, bool execute
)
379 if (static_branch_likely(&pkey_disabled
))
382 return pkey_access_permitted(pte_to_pkey_bits(pte
), write
, execute
);
386 * We only want to enforce protection keys on the current thread because we
387 * effectively have no access to AMR/IAMR for other threads or any way to tell
388 * which AMR/IAMR in a threaded process we could use.
390 * So do not enforce things if the VMA is not from the current mm, or if we are
391 * in a kernel thread.
393 static inline bool vma_is_foreign(struct vm_area_struct
*vma
)
398 /* if it is not our ->mm, it has to be foreign */
399 if (current
->mm
!= vma
->vm_mm
)
405 bool arch_vma_access_permitted(struct vm_area_struct
*vma
, bool write
,
406 bool execute
, bool foreign
)
408 if (static_branch_likely(&pkey_disabled
))
411 * Do not enforce our key-permissions on a foreign vma.
413 if (foreign
|| vma_is_foreign(vma
))
416 return pkey_access_permitted(vma_pkey(vma
), write
, execute
);
419 void arch_dup_pkeys(struct mm_struct
*oldmm
, struct mm_struct
*mm
)
421 if (static_branch_likely(&pkey_disabled
))
424 /* Duplicate the oldmm pkey state in mm: */
425 mm_pkey_allocation_map(mm
) = mm_pkey_allocation_map(oldmm
);
426 mm
->context
.execute_only_pkey
= oldmm
->context
.execute_only_pkey
;