]>
Commit | Line | Data |
---|---|---|
caab277b | 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
4f04d8f0 CM |
2 | /* |
3 | * Based on arch/arm/include/asm/memory.h | |
4 | * | |
5 | * Copyright (C) 2000-2002 Russell King | |
6 | * Copyright (C) 2012 ARM Ltd. | |
7 | * | |
4f04d8f0 CM |
8 | * Note: this file should not be included by non-asm/.h files |
9 | */ | |
10 | #ifndef __ASM_MEMORY_H | |
11 | #define __ASM_MEMORY_H | |
12 | ||
4f04d8f0 | 13 | #include <linux/const.h> |
d0b3c32e | 14 | #include <linux/sizes.h> |
b6531456 | 15 | #include <asm/page-def.h> |
4f04d8f0 | 16 | |
aa03c428 MR |
17 | /* |
18 | * Size of the PCI I/O space. This must remain a power of two so that | |
19 | * IO_SPACE_LIMIT acts as a mask for the low bits of I/O addresses. | |
20 | */ | |
21 | #define PCI_IO_SIZE SZ_16M | |
22 | ||
3e1907d5 AB |
23 | /* |
24 | * VMEMMAP_SIZE - allows the whole linear region to be covered by | |
25 | * a struct page array | |
ce3aaed8 SC |
26 | * |
27 | * If we are configured with a 52-bit kernel VA then our VMEMMAP_SIZE | |
77ad4ce6 MR |
28 | * needs to cover the memory region from the beginning of the 52-bit |
29 | * PAGE_OFFSET all the way to PAGE_END for 48-bit. This allows us to | |
ce3aaed8 SC |
30 | * keep a constant PAGE_OFFSET and "fallback" to using the higher end |
31 | * of the VMEMMAP where 52-bit support is not available in hardware. | |
3e1907d5 | 32 | */ |
8c96400d AB |
33 | #define VMEMMAP_SHIFT (PAGE_SHIFT - STRUCT_PAGE_MAX_SHIFT) |
34 | #define VMEMMAP_SIZE ((_PAGE_END(VA_BITS_MIN) - PAGE_OFFSET) >> VMEMMAP_SHIFT) | |
3e1907d5 | 35 | |
4f04d8f0 | 36 | /* |
77ad4ce6 MR |
37 | * PAGE_OFFSET - the virtual address of the start of the linear map, at the |
38 | * start of the TTBR1 address space. | |
39 | * PAGE_END - the end of the linear map, where all other kernel mappings begin. | |
40 | * KIMAGE_VADDR - the virtual address of the start of the kernel image. | |
4f04d8f0 | 41 | * VA_BITS - the maximum number of bits for virtual addresses. |
4f04d8f0 | 42 | */ |
e41ceed0 | 43 | #define VA_BITS (CONFIG_ARM64_VA_BITS) |
a5ac40f5 | 44 | #define _PAGE_OFFSET(va) (-(UL(1) << (va))) |
b6d00d47 | 45 | #define PAGE_OFFSET (_PAGE_OFFSET(VA_BITS)) |
f9040773 | 46 | #define KIMAGE_VADDR (MODULES_END) |
f4693c27 | 47 | #define BPF_JIT_REGION_START (_PAGE_END(VA_BITS_MIN)) |
91fc957c AB |
48 | #define BPF_JIT_REGION_SIZE (SZ_128M) |
49 | #define BPF_JIT_REGION_END (BPF_JIT_REGION_START + BPF_JIT_REGION_SIZE) | |
f9040773 | 50 | #define MODULES_END (MODULES_VADDR + MODULES_VSIZE) |
91fc957c | 51 | #define MODULES_VADDR (BPF_JIT_REGION_END) |
f80fb3a3 | 52 | #define MODULES_VSIZE (SZ_128M) |
8c96400d | 53 | #define VMEMMAP_START (-(UL(1) << (VA_BITS - VMEMMAP_SHIFT))) |
bbd6ec60 | 54 | #define VMEMMAP_END (VMEMMAP_START + VMEMMAP_SIZE) |
9ad7c6d5 | 55 | #define PCI_IO_END (VMEMMAP_START - SZ_8M) |
aa03c428 | 56 | #define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE) |
9ad7c6d5 | 57 | #define FIXADDR_TOP (VMEMMAP_START - SZ_32M) |
4f04d8f0 | 58 | |
90ec95cd SC |
59 | #if VA_BITS > 48 |
60 | #define VA_BITS_MIN (48) | |
61 | #else | |
62 | #define VA_BITS_MIN (VA_BITS) | |
63 | #endif | |
68933aa9 | 64 | |
77ad4ce6 | 65 | #define _PAGE_END(va) (-(UL(1) << ((va) - 1))) |
28c72583 | 66 | |
d0b3c32e WD |
67 | #define KERNEL_START _text |
68 | #define KERNEL_END _end | |
28c72583 | 69 | |
f9040773 | 70 | /* |
b2f557ea AK |
71 | * Generic and tag-based KASAN require 1/8th and 1/16th of the kernel virtual |
72 | * address space for the shadow region respectively. They can bloat the stack | |
73 | * significantly, so double the (minimum) stack size when they are in use. | |
f9040773 AB |
74 | */ |
75 | #ifdef CONFIG_KASAN | |
6bd1d0be SC |
76 | #define KASAN_SHADOW_OFFSET _AC(CONFIG_KASAN_SHADOW_OFFSET, UL) |
77 | #define KASAN_SHADOW_END ((UL(1) << (64 - KASAN_SHADOW_SCALE_SHIFT)) \ | |
78 | + KASAN_SHADOW_OFFSET) | |
f4693c27 | 79 | #define PAGE_END (KASAN_SHADOW_END - (1UL << (vabits_actual - KASAN_SHADOW_SCALE_SHIFT))) |
b02faed1 | 80 | #define KASAN_THREAD_SHIFT 1 |
f9040773 | 81 | #else |
b02faed1 | 82 | #define KASAN_THREAD_SHIFT 0 |
f4693c27 | 83 | #define PAGE_END (_PAGE_END(VA_BITS_MIN)) |
68933aa9 | 84 | #endif /* CONFIG_KASAN */ |
f9040773 | 85 | |
b02faed1 | 86 | #define MIN_THREAD_SHIFT (14 + KASAN_THREAD_SHIFT) |
e3067861 MR |
87 | |
88 | /* | |
89 | * VMAP'd stacks are allocated at page granularity, so we must ensure that such | |
90 | * stacks are a multiple of page size. | |
91 | */ | |
92 | #if defined(CONFIG_VMAP_STACK) && (MIN_THREAD_SHIFT < PAGE_SHIFT) | |
93 | #define THREAD_SHIFT PAGE_SHIFT | |
94 | #else | |
95 | #define THREAD_SHIFT MIN_THREAD_SHIFT | |
96 | #endif | |
dbc9344a MR |
97 | |
98 | #if THREAD_SHIFT >= PAGE_SHIFT | |
99 | #define THREAD_SIZE_ORDER (THREAD_SHIFT - PAGE_SHIFT) | |
100 | #endif | |
101 | ||
102 | #define THREAD_SIZE (UL(1) << THREAD_SHIFT) | |
103 | ||
e3067861 MR |
104 | /* |
105 | * By aligning VMAP'd stacks to 2 * THREAD_SIZE, we can detect overflow by | |
106 | * checking sp & (1 << THREAD_SHIFT), which we can do cheaply in the entry | |
107 | * assembly. | |
108 | */ | |
109 | #ifdef CONFIG_VMAP_STACK | |
110 | #define THREAD_ALIGN (2 * THREAD_SIZE) | |
111 | #else | |
112 | #define THREAD_ALIGN THREAD_SIZE | |
113 | #endif | |
114 | ||
f60ad4ed MR |
115 | #define IRQ_STACK_SIZE THREAD_SIZE |
116 | ||
872d8327 MR |
117 | #define OVERFLOW_STACK_SIZE SZ_4K |
118 | ||
8018ba4e MR |
119 | /* |
120 | * Alignment of kernel segments (e.g. .text, .data). | |
e16e65a0 | 121 | * |
8018ba4e MR |
122 | * 4 KB granule: 16 level 3 entries, with contiguous bit |
123 | * 16 KB granule: 4 level 3 entries, without contiguous bit | |
124 | * 64 KB granule: 1 level 3 entry | |
125 | */ | |
d0b3c32e | 126 | #define SEGMENT_ALIGN SZ_64K |
8018ba4e | 127 | |
4f04d8f0 CM |
128 | /* |
129 | * Memory types available. | |
9f341931 CM |
130 | * |
131 | * IMPORTANT: MT_NORMAL must be index 0 since vm_get_page_prot() may 'or' in | |
132 | * the MT_NORMAL_TAGGED memory type for PROT_MTE mappings. Note | |
133 | * that protection_map[] only contains MT_NORMAL attributes. | |
4f04d8f0 | 134 | */ |
9f341931 CM |
135 | #define MT_NORMAL 0 |
136 | #define MT_NORMAL_TAGGED 1 | |
137 | #define MT_NORMAL_NC 2 | |
138 | #define MT_NORMAL_WT 3 | |
139 | #define MT_DEVICE_nGnRnE 4 | |
140 | #define MT_DEVICE_nGnRE 5 | |
141 | #define MT_DEVICE_GRE 6 | |
4f04d8f0 | 142 | |
36311607 MZ |
143 | /* |
144 | * Memory types for Stage-2 translation | |
145 | */ | |
146 | #define MT_S2_NORMAL 0xf | |
147 | #define MT_S2_DEVICE_nGnRE 0x1 | |
148 | ||
e48d53a9 MZ |
149 | /* |
150 | * Memory types for Stage-2 translation when ID_AA64MMFR2_EL1.FWB is 0001 | |
151 | * Stage-2 enforces Normal-WB and Device-nGnRE | |
152 | */ | |
153 | #define MT_S2_FWB_NORMAL 6 | |
154 | #define MT_S2_FWB_DEVICE_nGnRE 1 | |
155 | ||
324420bf AB |
156 | #ifdef CONFIG_ARM64_4K_PAGES |
157 | #define IOREMAP_MAX_ORDER (PUD_SHIFT) | |
158 | #else | |
159 | #define IOREMAP_MAX_ORDER (PMD_SHIFT) | |
160 | #endif | |
161 | ||
4f04d8f0 CM |
162 | #ifndef __ASSEMBLY__ |
163 | ||
8439e62a | 164 | #include <linux/bitops.h> |
5f1f7f6c | 165 | #include <linux/compiler.h> |
a92405f0 | 166 | #include <linux/mmdebug.h> |
5f1f7f6c WD |
167 | #include <linux/types.h> |
168 | #include <asm/bug.h> | |
169 | ||
170 | extern u64 vabits_actual; | |
a92405f0 | 171 | |
020d044f | 172 | extern s64 memstart_addr; |
4f04d8f0 | 173 | /* PHYS_OFFSET - the physical address of the start of memory. */ |
a92405f0 | 174 | #define PHYS_OFFSET ({ VM_BUG_ON(memstart_addr & 1); memstart_addr; }) |
a7f8de16 | 175 | |
120dc60d | 176 | /* the virtual base of the kernel image */ |
f80fb3a3 AB |
177 | extern u64 kimage_vaddr; |
178 | ||
a7f8de16 AB |
179 | /* the offset between the kernel virtual and physical mappings */ |
180 | extern u64 kimage_voffset; | |
4f04d8f0 | 181 | |
7ede8665 AP |
182 | static inline unsigned long kaslr_offset(void) |
183 | { | |
184 | return kimage_vaddr - KIMAGE_VADDR; | |
185 | } | |
186 | ||
34ba2c42 | 187 | /* |
a7f8de16 | 188 | * Allow all memory at the discovery stage. We will clip it later. |
34ba2c42 | 189 | */ |
a7f8de16 AB |
190 | #define MIN_MEMBLOCK_ADDR 0 |
191 | #define MAX_MEMBLOCK_ADDR U64_MAX | |
34ba2c42 | 192 | |
4f04d8f0 CM |
193 | /* |
194 | * PFNs are used to describe any physical page; this means | |
195 | * PFN 0 == physical address 0. | |
196 | * | |
197 | * This is the PFN of the first RAM page in the kernel | |
198 | * direct-mapped view. We assume this is the first page | |
199 | * of RAM in the mem_map as well. | |
200 | */ | |
201 | #define PHYS_PFN_OFFSET (PHYS_OFFSET >> PAGE_SHIFT) | |
202 | ||
9c23f847 AK |
203 | /* |
204 | * When dealing with data aborts, watchpoints, or instruction traps we may end | |
205 | * up with a tagged userland pointer. Clear the tag to get a sane pointer to | |
206 | * pass on to access_ok(), for instance. | |
207 | */ | |
597399d0 | 208 | #define __untagged_addr(addr) \ |
9c1cac42 | 209 | ((__force __typeof__(addr))sign_extend64((__force u64)(addr), 55)) |
9c23f847 | 210 | |
597399d0 | 211 | #define untagged_addr(addr) ({ \ |
d0022c0e | 212 | u64 __addr = (__force u64)(addr); \ |
597399d0 WD |
213 | __addr &= __untagged_addr(__addr); \ |
214 | (__force __typeof__(addr))__addr; \ | |
215 | }) | |
216 | ||
3c9e3aa1 AK |
217 | #ifdef CONFIG_KASAN_SW_TAGS |
218 | #define __tag_shifted(tag) ((u64)(tag) << 56) | |
597399d0 | 219 | #define __tag_reset(addr) __untagged_addr(addr) |
3c9e3aa1 AK |
220 | #define __tag_get(addr) (__u8)((u64)(addr) >> 56) |
221 | #else | |
6bbd497f | 222 | #define __tag_shifted(tag) 0UL |
3c9e3aa1 AK |
223 | #define __tag_reset(addr) (addr) |
224 | #define __tag_get(addr) 0 | |
68933aa9 | 225 | #endif /* CONFIG_KASAN_SW_TAGS */ |
3c9e3aa1 | 226 | |
7732d20a QC |
227 | static inline const void *__tag_set(const void *addr, u8 tag) |
228 | { | |
6bbd497f WD |
229 | u64 __addr = (u64)addr & ~__tag_shifted(0xff); |
230 | return (const void *)(__addr | __tag_shifted(tag)); | |
7732d20a QC |
231 | } |
232 | ||
ccbe2aab AK |
233 | #ifdef CONFIG_KASAN_HW_TAGS |
234 | #define arch_enable_tagging() mte_enable_kernel() | |
235 | #define arch_init_tags(max_tag) mte_init_tags(max_tag) | |
236 | #define arch_get_random_tag() mte_get_random_tag() | |
237 | #define arch_get_mem_tag(addr) mte_get_mem_tag(addr) | |
238 | #define arch_set_mem_tag_range(addr, size, tag) \ | |
239 | mte_set_mem_tag_range((addr), (size), (tag)) | |
240 | #endif /* CONFIG_KASAN_HW_TAGS */ | |
241 | ||
9e22eb61 LA |
242 | /* |
243 | * Physical vs virtual RAM address space conversion. These are | |
244 | * private definitions which should NOT be used outside memory.h | |
245 | * files. Use virt_to_phys/phys_to_virt/__pa/__va instead. | |
246 | */ | |
ec6d06ef LA |
247 | |
248 | ||
249 | /* | |
f4693c27 | 250 | * The linear kernel range starts at the bottom of the virtual address space. |
ec6d06ef | 251 | */ |
f4693c27 | 252 | #define __is_lm_address(addr) (((u64)(addr) & ~PAGE_OFFSET) < (PAGE_END - PAGE_OFFSET)) |
ec6d06ef | 253 | |
7bc1a0f9 | 254 | #define __lm_to_phys(addr) (((addr) & ~PAGE_OFFSET) + PHYS_OFFSET) |
ec6d06ef LA |
255 | #define __kimg_to_phys(addr) ((addr) - kimage_voffset) |
256 | ||
257 | #define __virt_to_phys_nodebug(x) ({ \ | |
577c2b35 | 258 | phys_addr_t __x = (phys_addr_t)(__tag_reset(x)); \ |
d0b3c32e | 259 | __is_lm_address(__x) ? __lm_to_phys(__x) : __kimg_to_phys(__x); \ |
ec6d06ef LA |
260 | }) |
261 | ||
262 | #define __pa_symbol_nodebug(x) __kimg_to_phys((phys_addr_t)(x)) | |
263 | ||
264 | #ifdef CONFIG_DEBUG_VIRTUAL | |
265 | extern phys_addr_t __virt_to_phys(unsigned long x); | |
266 | extern phys_addr_t __phys_addr_symbol(unsigned long x); | |
267 | #else | |
268 | #define __virt_to_phys(x) __virt_to_phys_nodebug(x) | |
269 | #define __phys_addr_symbol(x) __pa_symbol_nodebug(x) | |
68933aa9 | 270 | #endif /* CONFIG_DEBUG_VIRTUAL */ |
9e22eb61 | 271 | |
7bc1a0f9 | 272 | #define __phys_to_virt(x) ((unsigned long)((x) - PHYS_OFFSET) | PAGE_OFFSET) |
9e22eb61 LA |
273 | #define __phys_to_kimg(x) ((unsigned long)((x) + kimage_voffset)) |
274 | ||
275 | /* | |
276 | * Convert a page to/from a physical address | |
277 | */ | |
278 | #define page_to_phys(page) (__pfn_to_phys(page_to_pfn(page))) | |
279 | #define phys_to_page(phys) (pfn_to_page(__phys_to_pfn(phys))) | |
280 | ||
4f04d8f0 CM |
281 | /* |
282 | * Note: Drivers should NOT use these. They are the wrong | |
283 | * translation for translating DMA addresses. Use the driver | |
284 | * DMA support - see dma-mapping.h. | |
285 | */ | |
09a57239 | 286 | #define virt_to_phys virt_to_phys |
4f04d8f0 CM |
287 | static inline phys_addr_t virt_to_phys(const volatile void *x) |
288 | { | |
289 | return __virt_to_phys((unsigned long)(x)); | |
290 | } | |
291 | ||
09a57239 | 292 | #define phys_to_virt phys_to_virt |
4f04d8f0 CM |
293 | static inline void *phys_to_virt(phys_addr_t x) |
294 | { | |
295 | return (void *)(__phys_to_virt(x)); | |
296 | } | |
297 | ||
298 | /* | |
299 | * Drivers should NOT use these either. | |
300 | */ | |
301 | #define __pa(x) __virt_to_phys((unsigned long)(x)) | |
ec6d06ef LA |
302 | #define __pa_symbol(x) __phys_addr_symbol(RELOC_HIDE((unsigned long)(x), 0)) |
303 | #define __pa_nodebug(x) __virt_to_phys_nodebug((unsigned long)(x)) | |
4f04d8f0 CM |
304 | #define __va(x) ((void *)__phys_to_virt((phys_addr_t)(x))) |
305 | #define pfn_to_kaddr(pfn) __va((pfn) << PAGE_SHIFT) | |
d0b3c32e WD |
306 | #define virt_to_pfn(x) __phys_to_pfn(__virt_to_phys((unsigned long)(x))) |
307 | #define sym_to_pfn(x) __phys_to_pfn(__pa_symbol(x)) | |
4f04d8f0 CM |
308 | |
309 | /* | |
d0b3c32e WD |
310 | * virt_to_page(x) convert a _valid_ virtual address to struct page * |
311 | * virt_addr_valid(x) indicates whether a virtual address is valid | |
4f04d8f0 | 312 | */ |
5fd6690c | 313 | #define ARCH_PFN_OFFSET ((unsigned long)PHYS_PFN_OFFSET) |
4f04d8f0 | 314 | |
eea1bb22 | 315 | #if !defined(CONFIG_SPARSEMEM_VMEMMAP) || defined(CONFIG_DEBUG_VIRTUAL) |
d0b3c32e | 316 | #define virt_to_page(x) pfn_to_page(virt_to_pfn(x)) |
9f287591 | 317 | #else |
96628f0f WD |
318 | #define page_to_virt(x) ({ \ |
319 | __typeof__(x) __page = x; \ | |
c1090bb1 AB |
320 | u64 __idx = ((u64)__page - VMEMMAP_START) / sizeof(struct page);\ |
321 | u64 __addr = PAGE_OFFSET + (__idx * PAGE_SIZE); \ | |
96628f0f | 322 | (void *)__tag_set((const void *)__addr, page_kasan_tag(__page));\ |
2813b9c0 AK |
323 | }) |
324 | ||
96628f0f | 325 | #define virt_to_page(x) ({ \ |
c1090bb1 AB |
326 | u64 __idx = (__tag_reset((u64)x) - PAGE_OFFSET) / PAGE_SIZE; \ |
327 | u64 __addr = VMEMMAP_START + (__idx * sizeof(struct page)); \ | |
328 | (struct page *)__addr; \ | |
96628f0f | 329 | }) |
68933aa9 | 330 | #endif /* !CONFIG_SPARSEMEM_VMEMMAP || CONFIG_DEBUG_VIRTUAL */ |
4f04d8f0 | 331 | |
68dd8ef3 WD |
332 | #define virt_addr_valid(addr) ({ \ |
333 | __typeof__(addr) __addr = addr; \ | |
334 | __is_lm_address(__addr) && pfn_valid(virt_to_pfn(__addr)); \ | |
335 | }) | |
4f04d8f0 | 336 | |
638d5031 | 337 | void dump_mem_limit(void); |
68933aa9 | 338 | #endif /* !ASSEMBLY */ |
ca219452 | 339 | |
8a5b403d AB |
340 | /* |
341 | * Given that the GIC architecture permits ITS implementations that can only be | |
342 | * configured with a LPI table address once, GICv3 systems with many CPUs may | |
343 | * end up reserving a lot of different regions after a kexec for their LPI | |
344 | * tables (one per CPU), as we are forced to reuse the same memory after kexec | |
345 | * (and thus reserve it persistently with EFI beforehand) | |
346 | */ | |
347 | #if defined(CONFIG_EFI) && defined(CONFIG_ARM_GIC_V3_ITS) | |
348 | # define INIT_MEMBLOCK_RESERVED_REGIONS (INIT_MEMBLOCK_REGIONS + NR_CPUS + 1) | |
349 | #endif | |
350 | ||
4f04d8f0 CM |
351 | #include <asm-generic/memory_model.h> |
352 | ||
68933aa9 | 353 | #endif /* __ASM_MEMORY_H */ |