]>
Commit | Line | Data |
---|---|---|
9f4c815c IM |
1 | /* |
2 | * Copyright 2002 Andi Kleen, SuSE Labs. | |
1da177e4 | 3 | * Thanks to Ben LaHaise for precious feedback. |
9f4c815c | 4 | */ |
1da177e4 | 5 | |
1da177e4 LT |
6 | #include <linux/highmem.h> |
7 | #include <linux/module.h> | |
9f4c815c | 8 | #include <linux/sched.h> |
1da177e4 | 9 | #include <linux/slab.h> |
9f4c815c IM |
10 | #include <linux/mm.h> |
11 | ||
1da177e4 LT |
12 | #include <asm/processor.h> |
13 | #include <asm/tlbflush.h> | |
f8af095d | 14 | #include <asm/sections.h> |
9f4c815c IM |
15 | #include <asm/uaccess.h> |
16 | #include <asm/pgalloc.h> | |
1da177e4 | 17 | |
f0646e43 | 18 | pte_t *lookup_address(unsigned long address, int *level) |
9f4c815c | 19 | { |
1da177e4 LT |
20 | pgd_t *pgd = pgd_offset_k(address); |
21 | pud_t *pud; | |
22 | pmd_t *pmd; | |
9f4c815c | 23 | |
1da177e4 LT |
24 | if (pgd_none(*pgd)) |
25 | return NULL; | |
26 | pud = pud_offset(pgd, address); | |
27 | if (pud_none(*pud)) | |
28 | return NULL; | |
29 | pmd = pmd_offset(pud, address); | |
30 | if (pmd_none(*pmd)) | |
31 | return NULL; | |
f0646e43 | 32 | *level = 2; |
1da177e4 LT |
33 | if (pmd_large(*pmd)) |
34 | return (pte_t *)pmd; | |
f0646e43 | 35 | *level = 3; |
1da177e4 | 36 | |
9f4c815c IM |
37 | return pte_offset_kernel(pmd, address); |
38 | } | |
39 | ||
40 | static struct page * | |
97f99fed | 41 | split_large_page(unsigned long address, pgprot_t ref_prot) |
9f4c815c | 42 | { |
1da177e4 LT |
43 | unsigned long addr; |
44 | struct page *base; | |
45 | pte_t *pbase; | |
9f4c815c | 46 | int i; |
1da177e4 | 47 | |
1da177e4 | 48 | base = alloc_pages(GFP_KERNEL, 0); |
9f4c815c | 49 | if (!base) |
1da177e4 LT |
50 | return NULL; |
51 | ||
84d1c054 NP |
52 | /* |
53 | * page_private is used to track the number of entries in | |
54 | * the page table page that have non standard attributes. | |
55 | */ | |
1da177e4 | 56 | address = __pa(address); |
9f4c815c | 57 | addr = address & LARGE_PAGE_MASK; |
1da177e4 | 58 | pbase = (pte_t *)page_address(base); |
fdb4c338 | 59 | paravirt_alloc_pt(&init_mm, page_to_pfn(base)); |
9f4c815c | 60 | |
97f99fed IM |
61 | for (i = 0; i < PTRS_PER_PTE; i++, addr += PAGE_SIZE) |
62 | set_pte(&pbase[i], pfn_pte(addr >> PAGE_SHIFT, ref_prot)); | |
63 | ||
1da177e4 | 64 | return base; |
9f4c815c | 65 | } |
1da177e4 | 66 | |
9f4c815c IM |
67 | static void set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte) |
68 | { | |
1da177e4 | 69 | unsigned long flags; |
9f4c815c | 70 | struct page *page; |
1da177e4 | 71 | |
9f4c815c IM |
72 | /* change init_mm */ |
73 | set_pte_atomic(kpte, pte); | |
5311ab62 | 74 | if (SHARED_KERNEL_PMD) |
1da177e4 LT |
75 | return; |
76 | ||
77 | spin_lock_irqsave(&pgd_lock, flags); | |
78 | for (page = pgd_list; page; page = (struct page *)page->index) { | |
79 | pgd_t *pgd; | |
80 | pud_t *pud; | |
81 | pmd_t *pmd; | |
9f4c815c | 82 | |
1da177e4 LT |
83 | pgd = (pgd_t *)page_address(page) + pgd_index(address); |
84 | pud = pud_offset(pgd, address); | |
85 | pmd = pmd_offset(pud, address); | |
86 | set_pte_atomic((pte_t *)pmd, pte); | |
87 | } | |
88 | spin_unlock_irqrestore(&pgd_lock, flags); | |
89 | } | |
90 | ||
9f4c815c IM |
91 | static int __change_page_attr(struct page *page, pgprot_t prot) |
92 | { | |
78c94aba | 93 | pgprot_t ref_prot = PAGE_KERNEL; |
1da177e4 | 94 | struct page *kpte_page; |
9f4c815c | 95 | unsigned long address; |
78c94aba | 96 | pgprot_t oldprot; |
9f4c815c | 97 | pte_t *kpte; |
f0646e43 | 98 | int level; |
1da177e4 LT |
99 | |
100 | BUG_ON(PageHighMem(page)); | |
101 | address = (unsigned long)page_address(page); | |
102 | ||
97f99fed | 103 | repeat: |
f0646e43 | 104 | kpte = lookup_address(address, &level); |
1da177e4 LT |
105 | if (!kpte) |
106 | return -EINVAL; | |
9f4c815c | 107 | |
78c94aba | 108 | oldprot = pte_pgprot(*kpte); |
1da177e4 | 109 | kpte_page = virt_to_page(kpte); |
65d2f0bc AK |
110 | BUG_ON(PageLRU(kpte_page)); |
111 | BUG_ON(PageCompound(kpte_page)); | |
112 | ||
1da177e4 | 113 | /* |
78c94aba IM |
114 | * Better fail early if someone sets the kernel text to NX. |
115 | * Does not cover __inittext | |
1da177e4 | 116 | */ |
78c94aba IM |
117 | BUG_ON(address >= (unsigned long)&_text && |
118 | address < (unsigned long)&_etext && | |
119 | (pgprot_val(prot) & _PAGE_NX)); | |
65d2f0bc | 120 | |
78c94aba IM |
121 | if ((address & LARGE_PAGE_MASK) < (unsigned long)&_etext) |
122 | ref_prot = PAGE_KERNEL_EXEC; | |
123 | ||
124 | ref_prot = canon_pgprot(ref_prot); | |
125 | prot = canon_pgprot(prot); | |
126 | ||
127 | if (level == 3) { | |
128 | set_pte_atomic(kpte, mk_pte(page, prot)); | |
129 | } else { | |
130 | struct page *split; | |
97f99fed IM |
131 | |
132 | split = split_large_page(address, ref_prot); | |
78c94aba IM |
133 | if (!split) |
134 | return -ENOMEM; | |
135 | ||
136 | /* | |
137 | * There's a small window here to waste a bit of RAM: | |
138 | */ | |
139 | set_pmd_pte(kpte, address, mk_pte(split, ref_prot)); | |
97f99fed | 140 | goto repeat; |
1da177e4 LT |
141 | } |
142 | return 0; | |
9f4c815c | 143 | } |
1da177e4 | 144 | |
1da177e4 LT |
145 | /* |
146 | * Change the page attributes of an page in the linear mapping. | |
147 | * | |
148 | * This should be used when a page is mapped with a different caching policy | |
149 | * than write-back somewhere - some CPUs do not like it when mappings with | |
150 | * different caching policies exist. This changes the page attributes of the | |
151 | * in kernel linear mapping too. | |
9f4c815c | 152 | * |
1da177e4 LT |
153 | * The caller needs to ensure that there are no conflicting mappings elsewhere. |
154 | * This function only deals with the kernel linear map. | |
9f4c815c | 155 | * |
1da177e4 LT |
156 | * Caller must call global_flush_tlb() after this. |
157 | */ | |
158 | int change_page_attr(struct page *page, int numpages, pgprot_t prot) | |
159 | { | |
9f4c815c | 160 | int err = 0, i; |
1da177e4 | 161 | |
9f4c815c | 162 | for (i = 0; i < numpages; i++, page++) { |
1da177e4 | 163 | err = __change_page_attr(page, prot); |
9f4c815c IM |
164 | if (err) |
165 | break; | |
166 | } | |
9f4c815c | 167 | |
1da177e4 LT |
168 | return err; |
169 | } | |
9f4c815c | 170 | EXPORT_SYMBOL(change_page_attr); |
1da177e4 | 171 | |
78c94aba | 172 | int change_page_attr_addr(unsigned long addr, int numpages, pgprot_t prot) |
626ab0e6 | 173 | { |
78c94aba IM |
174 | int i; |
175 | unsigned long pfn = (addr >> PAGE_SHIFT); | |
1da177e4 | 176 | |
78c94aba IM |
177 | for (i = 0; i < numpages; i++) { |
178 | if (!pfn_valid(pfn + i)) { | |
179 | break; | |
180 | } else { | |
181 | int level; | |
182 | pte_t *pte = lookup_address(addr + i*PAGE_SIZE, &level); | |
183 | BUG_ON(pte && !pte_none(*pte)); | |
184 | } | |
185 | } | |
186 | return change_page_attr(virt_to_page(addr), i, prot); | |
187 | } | |
188 | ||
189 | static void flush_kernel_map(void *arg) | |
190 | { | |
191 | /* | |
192 | * Flush all to work around Errata in early athlons regarding | |
193 | * large page flushing. | |
194 | */ | |
195 | __flush_tlb_all(); | |
196 | ||
197 | if (boot_cpu_data.x86_model >= 4) | |
198 | wbinvd(); | |
199 | } | |
200 | ||
201 | void global_flush_tlb(void) | |
202 | { | |
1da177e4 LT |
203 | BUG_ON(irqs_disabled()); |
204 | ||
78c94aba | 205 | on_each_cpu(flush_kernel_map, NULL, 1, 1); |
626ab0e6 | 206 | } |
9f4c815c | 207 | EXPORT_SYMBOL(global_flush_tlb); |
1da177e4 LT |
208 | |
209 | #ifdef CONFIG_DEBUG_PAGEALLOC | |
210 | void kernel_map_pages(struct page *page, int numpages, int enable) | |
211 | { | |
212 | if (PageHighMem(page)) | |
213 | return; | |
9f4c815c | 214 | if (!enable) { |
f9b8404c IM |
215 | debug_check_no_locks_freed(page_address(page), |
216 | numpages * PAGE_SIZE); | |
9f4c815c | 217 | } |
de5097c2 | 218 | |
9f4c815c IM |
219 | /* |
220 | * the return value is ignored - the calls cannot fail, | |
1da177e4 LT |
221 | * large pages are disabled at boot time. |
222 | */ | |
223 | change_page_attr(page, numpages, enable ? PAGE_KERNEL : __pgprot(0)); | |
9f4c815c IM |
224 | |
225 | /* | |
226 | * we should perform an IPI and flush all tlbs, | |
1da177e4 LT |
227 | * but that can deadlock->flush only current cpu. |
228 | */ | |
229 | __flush_tlb_all(); | |
230 | } | |
231 | #endif |