1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (c) 2014, The Linux Foundation. All rights reserved.
5 #include <linux/kernel.h>
7 #include <linux/module.h>
8 #include <linux/sched.h>
9 #include <linux/vmalloc.h>
11 #include <asm/cacheflush.h>
12 #include <asm/set_memory.h>
13 #include <asm/tlbflush.h>
15 struct page_change_data
{
20 bool rodata_full __ro_after_init
= IS_ENABLED(CONFIG_RODATA_FULL_DEFAULT_ENABLED
);
22 static int change_page_range(pte_t
*ptep
, unsigned long addr
, void *data
)
24 struct page_change_data
*cdata
= data
;
25 pte_t pte
= READ_ONCE(*ptep
);
27 pte
= clear_pte_bit(pte
, cdata
->clear_mask
);
28 pte
= set_pte_bit(pte
, cdata
->set_mask
);
35 * This function assumes that the range is mapped with PAGE_SIZE pages.
37 static int __change_memory_common(unsigned long start
, unsigned long size
,
38 pgprot_t set_mask
, pgprot_t clear_mask
)
40 struct page_change_data data
;
43 data
.set_mask
= set_mask
;
44 data
.clear_mask
= clear_mask
;
46 ret
= apply_to_page_range(&init_mm
, start
, size
, change_page_range
,
49 flush_tlb_kernel_range(start
, start
+ size
);
53 static int change_memory_common(unsigned long addr
, int numpages
,
54 pgprot_t set_mask
, pgprot_t clear_mask
)
56 unsigned long start
= addr
;
57 unsigned long size
= PAGE_SIZE
* numpages
;
58 unsigned long end
= start
+ size
;
59 struct vm_struct
*area
;
62 if (!PAGE_ALIGNED(addr
)) {
69 * Kernel VA mappings are always live, and splitting live section
70 * mappings into page mappings may cause TLB conflicts. This means
71 * we have to ensure that changing the permission bits of the range
72 * we are operating on does not result in such splitting.
74 * Let's restrict ourselves to mappings created by vmalloc (or vmap).
75 * Those are guaranteed to consist entirely of page mappings, and
76 * splitting is never needed.
78 * So check whether the [addr, addr + size) interval is entirely
79 * covered by precisely one VM area that has the VM_ALLOC flag set.
81 area
= find_vm_area((void *)addr
);
83 end
> (unsigned long)area
->addr
+ area
->size
||
84 !(area
->flags
& VM_ALLOC
))
91 * If we are manipulating read-only permissions, apply the same
92 * change to the linear mapping of the pages that back this VM area.
94 if (rodata_full
&& (pgprot_val(set_mask
) == PTE_RDONLY
||
95 pgprot_val(clear_mask
) == PTE_RDONLY
)) {
96 for (i
= 0; i
< area
->nr_pages
; i
++) {
97 __change_memory_common((u64
)page_address(area
->pages
[i
]),
98 PAGE_SIZE
, set_mask
, clear_mask
);
103 * Get rid of potentially aliasing lazily unmapped vm areas that may
104 * have permissions set that deviate from the ones we are setting here.
108 return __change_memory_common(start
, size
, set_mask
, clear_mask
);
111 int set_memory_ro(unsigned long addr
, int numpages
)
113 return change_memory_common(addr
, numpages
,
114 __pgprot(PTE_RDONLY
),
115 __pgprot(PTE_WRITE
));
118 int set_memory_rw(unsigned long addr
, int numpages
)
120 return change_memory_common(addr
, numpages
,
122 __pgprot(PTE_RDONLY
));
125 int set_memory_nx(unsigned long addr
, int numpages
)
127 return change_memory_common(addr
, numpages
,
129 __pgprot(PTE_MAYBE_GP
));
132 int set_memory_x(unsigned long addr
, int numpages
)
134 return change_memory_common(addr
, numpages
,
135 __pgprot(PTE_MAYBE_GP
),
139 int set_memory_valid(unsigned long addr
, int numpages
, int enable
)
142 return __change_memory_common(addr
, PAGE_SIZE
* numpages
,
146 return __change_memory_common(addr
, PAGE_SIZE
* numpages
,
148 __pgprot(PTE_VALID
));
151 int set_direct_map_invalid_noflush(struct page
*page
)
153 struct page_change_data data
= {
154 .set_mask
= __pgprot(0),
155 .clear_mask
= __pgprot(PTE_VALID
),
158 if (!debug_pagealloc_enabled() && !rodata_full
)
161 return apply_to_page_range(&init_mm
,
162 (unsigned long)page_address(page
),
163 PAGE_SIZE
, change_page_range
, &data
);
166 int set_direct_map_default_noflush(struct page
*page
)
168 struct page_change_data data
= {
169 .set_mask
= __pgprot(PTE_VALID
| PTE_WRITE
),
170 .clear_mask
= __pgprot(PTE_RDONLY
),
173 if (!debug_pagealloc_enabled() && !rodata_full
)
176 return apply_to_page_range(&init_mm
,
177 (unsigned long)page_address(page
),
178 PAGE_SIZE
, change_page_range
, &data
);
181 #ifdef CONFIG_DEBUG_PAGEALLOC
182 void __kernel_map_pages(struct page
*page
, int numpages
, int enable
)
184 if (!debug_pagealloc_enabled() && !rodata_full
)
187 set_memory_valid((unsigned long)page_address(page
), numpages
, enable
);
189 #endif /* CONFIG_DEBUG_PAGEALLOC */
192 * This function is used to determine if a linear map page has been marked as
193 * not-valid. Walk the page table and check the PTE_VALID bit. This is based
194 * on kern_addr_valid(), which almost does what we need.
196 * Because this is only called on the kernel linear map, p?d_sect() implies
197 * p?d_present(). When debug_pagealloc is enabled, sections mappings are
200 bool kernel_page_present(struct page
*page
)
207 unsigned long addr
= (unsigned long)page_address(page
);
209 if (!debug_pagealloc_enabled() && !rodata_full
)
212 pgdp
= pgd_offset_k(addr
);
213 if (pgd_none(READ_ONCE(*pgdp
)))
216 p4dp
= p4d_offset(pgdp
, addr
);
217 if (p4d_none(READ_ONCE(*p4dp
)))
220 pudp
= pud_offset(p4dp
, addr
);
221 pud
= READ_ONCE(*pudp
);
227 pmdp
= pmd_offset(pudp
, addr
);
228 pmd
= READ_ONCE(*pmdp
);
234 ptep
= pte_offset_kernel(pmdp
, addr
);
235 return pte_valid(READ_ONCE(*ptep
));