]>
Commit | Line | Data |
---|---|---|
caab277b | 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
4f04d8f0 CM |
2 | /* |
3 | * Based on arch/arm/include/asm/memory.h | |
4 | * | |
5 | * Copyright (C) 2000-2002 Russell King | |
6 | * Copyright (C) 2012 ARM Ltd. | |
7 | * | |
4f04d8f0 CM |
8 | * Note: this file should not be included by non-asm/.h files |
9 | */ | |
10 | #ifndef __ASM_MEMORY_H | |
11 | #define __ASM_MEMORY_H | |
12 | ||
13 | #include <linux/compiler.h> | |
14 | #include <linux/const.h> | |
15 | #include <linux/types.h> | |
a7f8de16 | 16 | #include <asm/bug.h> |
b6531456 | 17 | #include <asm/page-def.h> |
87dfb311 | 18 | #include <linux/sizes.h> |
4f04d8f0 | 19 | |
aa03c428 MR |
20 | /* |
21 | * Size of the PCI I/O space. This must remain a power of two so that | |
22 | * IO_SPACE_LIMIT acts as a mask for the low bits of I/O addresses. | |
23 | */ | |
24 | #define PCI_IO_SIZE SZ_16M | |
25 | ||
3e1907d5 AB |
26 | /* |
27 | * VMEMMAP_SIZE - allows the whole linear region to be covered by | |
28 | * a struct page array | |
29 | */ | |
30 | #define VMEMMAP_SIZE (UL(1) << (VA_BITS - PAGE_SHIFT - 1 + STRUCT_PAGE_MAX_SHIFT)) | |
31 | ||
4f04d8f0 | 32 | /* |
a13e3a5b | 33 | * PAGE_OFFSET - the virtual address of the start of the linear map (top |
847264fb | 34 | * (VA_BITS - 1)) |
a13e3a5b | 35 | * KIMAGE_VADDR - the virtual address of the start of the kernel image |
4f04d8f0 | 36 | * VA_BITS - the maximum number of bits for virtual addresses. |
127db024 | 37 | * VA_START - the first kernel virtual address. |
4f04d8f0 | 38 | */ |
e41ceed0 | 39 | #define VA_BITS (CONFIG_ARM64_VA_BITS) |
82cd5880 ND |
40 | #define VA_START (UL(0xffffffffffffffff) - \ |
41 | (UL(1) << VA_BITS) + 1) | |
42 | #define PAGE_OFFSET (UL(0xffffffffffffffff) - \ | |
43 | (UL(1) << (VA_BITS - 1)) + 1) | |
f9040773 | 44 | #define KIMAGE_VADDR (MODULES_END) |
91fc957c AB |
45 | #define BPF_JIT_REGION_START (VA_START + KASAN_SHADOW_SIZE) |
46 | #define BPF_JIT_REGION_SIZE (SZ_128M) | |
47 | #define BPF_JIT_REGION_END (BPF_JIT_REGION_START + BPF_JIT_REGION_SIZE) | |
f9040773 | 48 | #define MODULES_END (MODULES_VADDR + MODULES_VSIZE) |
91fc957c | 49 | #define MODULES_VADDR (BPF_JIT_REGION_END) |
f80fb3a3 | 50 | #define MODULES_VSIZE (SZ_128M) |
3e1907d5 AB |
51 | #define VMEMMAP_START (PAGE_OFFSET - VMEMMAP_SIZE) |
52 | #define PCI_IO_END (VMEMMAP_START - SZ_2M) | |
aa03c428 MR |
53 | #define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE) |
54 | #define FIXADDR_TOP (PCI_IO_START - SZ_2M) | |
4f04d8f0 | 55 | |
28c72583 JM |
56 | #define KERNEL_START _text |
57 | #define KERNEL_END _end | |
58 | ||
9b31cf49 WD |
59 | #ifdef CONFIG_ARM64_USER_VA_BITS_52 |
60 | #define MAX_USER_VA_BITS 52 | |
61 | #else | |
62 | #define MAX_USER_VA_BITS VA_BITS | |
63 | #endif | |
64 | ||
f9040773 | 65 | /* |
b2f557ea AK |
66 | * Generic and tag-based KASAN require 1/8th and 1/16th of the kernel virtual |
67 | * address space for the shadow region respectively. They can bloat the stack | |
68 | * significantly, so double the (minimum) stack size when they are in use. | |
f9040773 AB |
69 | */ |
70 | #ifdef CONFIG_KASAN | |
917538e2 | 71 | #define KASAN_SHADOW_SIZE (UL(1) << (VA_BITS - KASAN_SHADOW_SCALE_SHIFT)) |
b02faed1 | 72 | #define KASAN_THREAD_SHIFT 1 |
f9040773 AB |
73 | #else |
74 | #define KASAN_SHADOW_SIZE (0) | |
b02faed1 | 75 | #define KASAN_THREAD_SHIFT 0 |
f9040773 AB |
76 | #endif |
77 | ||
b02faed1 | 78 | #define MIN_THREAD_SHIFT (14 + KASAN_THREAD_SHIFT) |
e3067861 MR |
79 | |
80 | /* | |
81 | * VMAP'd stacks are allocated at page granularity, so we must ensure that such | |
82 | * stacks are a multiple of page size. | |
83 | */ | |
84 | #if defined(CONFIG_VMAP_STACK) && (MIN_THREAD_SHIFT < PAGE_SHIFT) | |
85 | #define THREAD_SHIFT PAGE_SHIFT | |
86 | #else | |
87 | #define THREAD_SHIFT MIN_THREAD_SHIFT | |
88 | #endif | |
dbc9344a MR |
89 | |
90 | #if THREAD_SHIFT >= PAGE_SHIFT | |
91 | #define THREAD_SIZE_ORDER (THREAD_SHIFT - PAGE_SHIFT) | |
92 | #endif | |
93 | ||
94 | #define THREAD_SIZE (UL(1) << THREAD_SHIFT) | |
95 | ||
e3067861 MR |
96 | /* |
97 | * By aligning VMAP'd stacks to 2 * THREAD_SIZE, we can detect overflow by | |
98 | * checking sp & (1 << THREAD_SHIFT), which we can do cheaply in the entry | |
99 | * assembly. | |
100 | */ | |
101 | #ifdef CONFIG_VMAP_STACK | |
102 | #define THREAD_ALIGN (2 * THREAD_SIZE) | |
103 | #else | |
104 | #define THREAD_ALIGN THREAD_SIZE | |
105 | #endif | |
106 | ||
f60ad4ed MR |
107 | #define IRQ_STACK_SIZE THREAD_SIZE |
108 | ||
872d8327 MR |
109 | #define OVERFLOW_STACK_SIZE SZ_4K |
110 | ||
8018ba4e MR |
111 | /* |
112 | * Alignment of kernel segments (e.g. .text, .data). | |
113 | */ | |
114 | #if defined(CONFIG_DEBUG_ALIGN_RODATA) | |
115 | /* | |
116 | * 4 KB granule: 1 level 2 entry | |
117 | * 16 KB granule: 128 level 3 entries, with contiguous bit | |
118 | * 64 KB granule: 32 level 3 entries, with contiguous bit | |
119 | */ | |
120 | #define SEGMENT_ALIGN SZ_2M | |
121 | #else | |
122 | /* | |
123 | * 4 KB granule: 16 level 3 entries, with contiguous bit | |
124 | * 16 KB granule: 4 level 3 entries, without contiguous bit | |
125 | * 64 KB granule: 1 level 3 entry | |
126 | */ | |
127 | #define SEGMENT_ALIGN SZ_64K | |
128 | #endif | |
129 | ||
4f04d8f0 CM |
130 | /* |
131 | * Memory types available. | |
132 | */ | |
133 | #define MT_DEVICE_nGnRnE 0 | |
134 | #define MT_DEVICE_nGnRE 1 | |
135 | #define MT_DEVICE_GRE 2 | |
136 | #define MT_NORMAL_NC 3 | |
137 | #define MT_NORMAL 4 | |
8d446c86 | 138 | #define MT_NORMAL_WT 5 |
4f04d8f0 | 139 | |
36311607 MZ |
140 | /* |
141 | * Memory types for Stage-2 translation | |
142 | */ | |
143 | #define MT_S2_NORMAL 0xf | |
144 | #define MT_S2_DEVICE_nGnRE 0x1 | |
145 | ||
e48d53a9 MZ |
146 | /* |
147 | * Memory types for Stage-2 translation when ID_AA64MMFR2_EL1.FWB is 0001 | |
148 | * Stage-2 enforces Normal-WB and Device-nGnRE | |
149 | */ | |
150 | #define MT_S2_FWB_NORMAL 6 | |
151 | #define MT_S2_FWB_DEVICE_nGnRE 1 | |
152 | ||
324420bf AB |
153 | #ifdef CONFIG_ARM64_4K_PAGES |
154 | #define IOREMAP_MAX_ORDER (PUD_SHIFT) | |
155 | #else | |
156 | #define IOREMAP_MAX_ORDER (PMD_SHIFT) | |
157 | #endif | |
158 | ||
4f04d8f0 CM |
159 | #ifndef __ASSEMBLY__ |
160 | ||
8439e62a | 161 | #include <linux/bitops.h> |
a92405f0 AB |
162 | #include <linux/mmdebug.h> |
163 | ||
020d044f | 164 | extern s64 memstart_addr; |
4f04d8f0 | 165 | /* PHYS_OFFSET - the physical address of the start of memory. */ |
a92405f0 | 166 | #define PHYS_OFFSET ({ VM_BUG_ON(memstart_addr & 1); memstart_addr; }) |
a7f8de16 | 167 | |
f80fb3a3 AB |
168 | /* the virtual base of the kernel image (minus TEXT_OFFSET) */ |
169 | extern u64 kimage_vaddr; | |
170 | ||
a7f8de16 AB |
171 | /* the offset between the kernel virtual and physical mappings */ |
172 | extern u64 kimage_voffset; | |
4f04d8f0 | 173 | |
7ede8665 AP |
174 | static inline unsigned long kaslr_offset(void) |
175 | { | |
176 | return kimage_vaddr - KIMAGE_VADDR; | |
177 | } | |
178 | ||
ec6e822d MR |
179 | /* the actual size of a user virtual address */ |
180 | extern u64 vabits_user; | |
181 | ||
34ba2c42 | 182 | /* |
a7f8de16 | 183 | * Allow all memory at the discovery stage. We will clip it later. |
34ba2c42 | 184 | */ |
a7f8de16 AB |
185 | #define MIN_MEMBLOCK_ADDR 0 |
186 | #define MAX_MEMBLOCK_ADDR U64_MAX | |
34ba2c42 | 187 | |
4f04d8f0 CM |
188 | /* |
189 | * PFNs are used to describe any physical page; this means | |
190 | * PFN 0 == physical address 0. | |
191 | * | |
192 | * This is the PFN of the first RAM page in the kernel | |
193 | * direct-mapped view. We assume this is the first page | |
194 | * of RAM in the mem_map as well. | |
195 | */ | |
196 | #define PHYS_PFN_OFFSET (PHYS_OFFSET >> PAGE_SHIFT) | |
197 | ||
9c23f847 AK |
198 | /* |
199 | * When dealing with data aborts, watchpoints, or instruction traps we may end | |
200 | * up with a tagged userland pointer. Clear the tag to get a sane pointer to | |
201 | * pass on to access_ok(), for instance. | |
202 | */ | |
203 | #define untagged_addr(addr) \ | |
204 | ((__typeof__(addr))sign_extend64((u64)(addr), 55)) | |
205 | ||
3c9e3aa1 AK |
206 | #ifdef CONFIG_KASAN_SW_TAGS |
207 | #define __tag_shifted(tag) ((u64)(tag) << 56) | |
208 | #define __tag_set(addr, tag) (__typeof__(addr))( \ | |
209 | ((u64)(addr) & ~__tag_shifted(0xff)) | __tag_shifted(tag)) | |
210 | #define __tag_reset(addr) untagged_addr(addr) | |
211 | #define __tag_get(addr) (__u8)((u64)(addr) >> 56) | |
212 | #else | |
213 | #define __tag_set(addr, tag) (addr) | |
214 | #define __tag_reset(addr) (addr) | |
215 | #define __tag_get(addr) 0 | |
216 | #endif | |
217 | ||
9e22eb61 LA |
218 | /* |
219 | * Physical vs virtual RAM address space conversion. These are | |
220 | * private definitions which should NOT be used outside memory.h | |
221 | * files. Use virt_to_phys/phys_to_virt/__pa/__va instead. | |
222 | */ | |
ec6d06ef LA |
223 | |
224 | ||
225 | /* | |
226 | * The linear kernel range starts in the middle of the virtual adddress | |
227 | * space. Testing the top bit for the start of the region is a | |
228 | * sufficient check. | |
229 | */ | |
230 | #define __is_lm_address(addr) (!!((addr) & BIT(VA_BITS - 1))) | |
231 | ||
232 | #define __lm_to_phys(addr) (((addr) & ~PAGE_OFFSET) + PHYS_OFFSET) | |
233 | #define __kimg_to_phys(addr) ((addr) - kimage_voffset) | |
234 | ||
235 | #define __virt_to_phys_nodebug(x) ({ \ | |
9e22eb61 | 236 | phys_addr_t __x = (phys_addr_t)(x); \ |
ec6d06ef LA |
237 | __is_lm_address(__x) ? __lm_to_phys(__x) : \ |
238 | __kimg_to_phys(__x); \ | |
239 | }) | |
240 | ||
241 | #define __pa_symbol_nodebug(x) __kimg_to_phys((phys_addr_t)(x)) | |
242 | ||
243 | #ifdef CONFIG_DEBUG_VIRTUAL | |
244 | extern phys_addr_t __virt_to_phys(unsigned long x); | |
245 | extern phys_addr_t __phys_addr_symbol(unsigned long x); | |
246 | #else | |
247 | #define __virt_to_phys(x) __virt_to_phys_nodebug(x) | |
248 | #define __phys_addr_symbol(x) __pa_symbol_nodebug(x) | |
249 | #endif | |
9e22eb61 LA |
250 | |
251 | #define __phys_to_virt(x) ((unsigned long)((x) - PHYS_OFFSET) | PAGE_OFFSET) | |
252 | #define __phys_to_kimg(x) ((unsigned long)((x) + kimage_voffset)) | |
253 | ||
254 | /* | |
255 | * Convert a page to/from a physical address | |
256 | */ | |
257 | #define page_to_phys(page) (__pfn_to_phys(page_to_pfn(page))) | |
258 | #define phys_to_page(phys) (pfn_to_page(__phys_to_pfn(phys))) | |
259 | ||
4f04d8f0 CM |
260 | /* |
261 | * Note: Drivers should NOT use these. They are the wrong | |
262 | * translation for translating DMA addresses. Use the driver | |
263 | * DMA support - see dma-mapping.h. | |
264 | */ | |
09a57239 | 265 | #define virt_to_phys virt_to_phys |
4f04d8f0 CM |
266 | static inline phys_addr_t virt_to_phys(const volatile void *x) |
267 | { | |
268 | return __virt_to_phys((unsigned long)(x)); | |
269 | } | |
270 | ||
09a57239 | 271 | #define phys_to_virt phys_to_virt |
4f04d8f0 CM |
272 | static inline void *phys_to_virt(phys_addr_t x) |
273 | { | |
274 | return (void *)(__phys_to_virt(x)); | |
275 | } | |
276 | ||
277 | /* | |
278 | * Drivers should NOT use these either. | |
279 | */ | |
280 | #define __pa(x) __virt_to_phys((unsigned long)(x)) | |
ec6d06ef LA |
281 | #define __pa_symbol(x) __phys_addr_symbol(RELOC_HIDE((unsigned long)(x), 0)) |
282 | #define __pa_nodebug(x) __virt_to_phys_nodebug((unsigned long)(x)) | |
4f04d8f0 CM |
283 | #define __va(x) ((void *)__phys_to_virt((phys_addr_t)(x))) |
284 | #define pfn_to_kaddr(pfn) __va((pfn) << PAGE_SHIFT) | |
869dcfd1 | 285 | #define virt_to_pfn(x) __phys_to_pfn(__virt_to_phys((unsigned long)(x))) |
2077be67 | 286 | #define sym_to_pfn(x) __phys_to_pfn(__pa_symbol(x)) |
4f04d8f0 CM |
287 | |
288 | /* | |
289 | * virt_to_page(k) convert a _valid_ virtual address to struct page * | |
290 | * virt_addr_valid(k) indicates whether a virtual address is valid | |
291 | */ | |
5fd6690c | 292 | #define ARCH_PFN_OFFSET ((unsigned long)PHYS_PFN_OFFSET) |
4f04d8f0 | 293 | |
eea1bb22 | 294 | #if !defined(CONFIG_SPARSEMEM_VMEMMAP) || defined(CONFIG_DEBUG_VIRTUAL) |
4f04d8f0 | 295 | #define virt_to_page(kaddr) pfn_to_page(__pa(kaddr) >> PAGE_SHIFT) |
ca219452 | 296 | #define _virt_addr_valid(kaddr) pfn_valid(__pa(kaddr) >> PAGE_SHIFT) |
9f287591 AB |
297 | #else |
298 | #define __virt_to_pgoff(kaddr) (((u64)(kaddr) & ~PAGE_OFFSET) / PAGE_SIZE * sizeof(struct page)) | |
1c8a946b | 299 | #define __page_to_voff(kaddr) (((u64)(kaddr) & ~VMEMMAP_START) * PAGE_SIZE / sizeof(struct page)) |
9f287591 | 300 | |
2813b9c0 AK |
301 | #define page_to_virt(page) ({ \ |
302 | unsigned long __addr = \ | |
303 | ((__page_to_voff(page)) | PAGE_OFFSET); \ | |
366e37e4 AB |
304 | unsigned long __addr_tag = \ |
305 | __tag_set(__addr, page_kasan_tag(page)); \ | |
306 | ((void *)__addr_tag); \ | |
2813b9c0 AK |
307 | }) |
308 | ||
9f287591 | 309 | #define virt_to_page(vaddr) ((struct page *)((__virt_to_pgoff(vaddr)) | VMEMMAP_START)) |
4f04d8f0 | 310 | |
ca219452 | 311 | #define _virt_addr_valid(kaddr) pfn_valid((((u64)(kaddr) & ~PAGE_OFFSET) \ |
9f287591 AB |
312 | + PHYS_OFFSET) >> PAGE_SHIFT) |
313 | #endif | |
4f04d8f0 CM |
314 | #endif |
315 | ||
e71fe3f9 AK |
316 | #define _virt_addr_is_linear(kaddr) \ |
317 | (__tag_reset((u64)(kaddr)) >= PAGE_OFFSET) | |
318 | #define virt_addr_valid(kaddr) \ | |
319 | (_virt_addr_is_linear(kaddr) && _virt_addr_valid(kaddr)) | |
ca219452 | 320 | |
8a5b403d AB |
321 | /* |
322 | * Given that the GIC architecture permits ITS implementations that can only be | |
323 | * configured with a LPI table address once, GICv3 systems with many CPUs may | |
324 | * end up reserving a lot of different regions after a kexec for their LPI | |
325 | * tables (one per CPU), as we are forced to reuse the same memory after kexec | |
326 | * (and thus reserve it persistently with EFI beforehand) | |
327 | */ | |
328 | #if defined(CONFIG_EFI) && defined(CONFIG_ARM_GIC_V3_ITS) | |
329 | # define INIT_MEMBLOCK_RESERVED_REGIONS (INIT_MEMBLOCK_REGIONS + NR_CPUS + 1) | |
330 | #endif | |
331 | ||
4f04d8f0 CM |
332 | #include <asm-generic/memory_model.h> |
333 | ||
334 | #endif |