]>
Commit | Line | Data |
---|---|---|
ac41aaee | 1 | // SPDX-License-Identifier: GPL-2.0 |
3610cce8 | 2 | /* |
a53c8fab | 3 | * Copyright IBM Corp. 2007, 2011 |
3610cce8 MS |
4 | * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> |
5 | */ | |
6 | ||
7 | #include <linux/sched.h> | |
8 | #include <linux/kernel.h> | |
9 | #include <linux/errno.h> | |
5a0e3ad6 | 10 | #include <linux/gfp.h> |
3610cce8 MS |
11 | #include <linux/mm.h> |
12 | #include <linux/swap.h> | |
13 | #include <linux/smp.h> | |
3610cce8 | 14 | #include <linux/spinlock.h> |
80217147 | 15 | #include <linux/rcupdate.h> |
e5992f2e | 16 | #include <linux/slab.h> |
b31288fa | 17 | #include <linux/swapops.h> |
0b46e0a3 | 18 | #include <linux/sysctl.h> |
3ac8e380 DD |
19 | #include <linux/ksm.h> |
20 | #include <linux/mman.h> | |
3610cce8 | 21 | |
3610cce8 MS |
22 | #include <asm/pgtable.h> |
23 | #include <asm/pgalloc.h> | |
24 | #include <asm/tlb.h> | |
25 | #include <asm/tlbflush.h> | |
6252d702 | 26 | #include <asm/mmu_context.h> |
2d42f947 | 27 | #include <asm/page-states.h> |
3610cce8 | 28 | |
118bd31b | 29 | static inline void ptep_ipte_local(struct mm_struct *mm, unsigned long addr, |
cd774b90 | 30 | pte_t *ptep, int nodat) |
118bd31b MS |
31 | { |
32 | unsigned long opt, asce; | |
33 | ||
34 | if (MACHINE_HAS_TLB_GUEST) { | |
35 | opt = 0; | |
36 | asce = READ_ONCE(mm->context.gmap_asce); | |
cd774b90 | 37 | if (asce == 0UL || nodat) |
118bd31b | 38 | opt |= IPTE_NODAT; |
28c807e5 MS |
39 | if (asce != -1UL) { |
40 | asce = asce ? : mm->context.asce; | |
41 | opt |= IPTE_GUEST_ASCE; | |
42 | } | |
43 | __ptep_ipte(addr, ptep, opt, asce, IPTE_LOCAL); | |
118bd31b | 44 | } else { |
28c807e5 | 45 | __ptep_ipte(addr, ptep, 0, 0, IPTE_LOCAL); |
118bd31b MS |
46 | } |
47 | } | |
48 | ||
49 | static inline void ptep_ipte_global(struct mm_struct *mm, unsigned long addr, | |
cd774b90 | 50 | pte_t *ptep, int nodat) |
118bd31b MS |
51 | { |
52 | unsigned long opt, asce; | |
53 | ||
54 | if (MACHINE_HAS_TLB_GUEST) { | |
55 | opt = 0; | |
56 | asce = READ_ONCE(mm->context.gmap_asce); | |
cd774b90 | 57 | if (asce == 0UL || nodat) |
118bd31b | 58 | opt |= IPTE_NODAT; |
28c807e5 MS |
59 | if (asce != -1UL) { |
60 | asce = asce ? : mm->context.asce; | |
61 | opt |= IPTE_GUEST_ASCE; | |
62 | } | |
63 | __ptep_ipte(addr, ptep, opt, asce, IPTE_GLOBAL); | |
118bd31b | 64 | } else { |
28c807e5 | 65 | __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL); |
118bd31b MS |
66 | } |
67 | } | |
68 | ||
ebde765c | 69 | static inline pte_t ptep_flush_direct(struct mm_struct *mm, |
cd774b90 MS |
70 | unsigned long addr, pte_t *ptep, |
71 | int nodat) | |
ebde765c | 72 | { |
ebde765c MS |
73 | pte_t old; |
74 | ||
75 | old = *ptep; | |
76 | if (unlikely(pte_val(old) & _PAGE_INVALID)) | |
77 | return old; | |
64f31d58 MS |
78 | atomic_inc(&mm->context.flush_count); |
79 | if (MACHINE_HAS_TLB_LC && | |
ebde765c | 80 | cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) |
cd774b90 | 81 | ptep_ipte_local(mm, addr, ptep, nodat); |
ebde765c | 82 | else |
cd774b90 | 83 | ptep_ipte_global(mm, addr, ptep, nodat); |
64f31d58 | 84 | atomic_dec(&mm->context.flush_count); |
ebde765c MS |
85 | return old; |
86 | } | |
87 | ||
88 | static inline pte_t ptep_flush_lazy(struct mm_struct *mm, | |
cd774b90 MS |
89 | unsigned long addr, pte_t *ptep, |
90 | int nodat) | |
ebde765c | 91 | { |
ebde765c MS |
92 | pte_t old; |
93 | ||
94 | old = *ptep; | |
95 | if (unlikely(pte_val(old) & _PAGE_INVALID)) | |
96 | return old; | |
64f31d58 MS |
97 | atomic_inc(&mm->context.flush_count); |
98 | if (cpumask_equal(&mm->context.cpu_attach_mask, | |
99 | cpumask_of(smp_processor_id()))) { | |
ebde765c MS |
100 | pte_val(*ptep) |= _PAGE_INVALID; |
101 | mm->context.flush_mm = 1; | |
102 | } else | |
cd774b90 | 103 | ptep_ipte_global(mm, addr, ptep, nodat); |
64f31d58 | 104 | atomic_dec(&mm->context.flush_count); |
ebde765c MS |
105 | return old; |
106 | } | |
107 | ||
1e133ab2 MS |
108 | static inline pgste_t pgste_get_lock(pte_t *ptep) |
109 | { | |
110 | unsigned long new = 0; | |
111 | #ifdef CONFIG_PGSTE | |
112 | unsigned long old; | |
113 | ||
1e133ab2 MS |
114 | asm( |
115 | " lg %0,%2\n" | |
116 | "0: lgr %1,%0\n" | |
117 | " nihh %0,0xff7f\n" /* clear PCL bit in old */ | |
118 | " oihh %1,0x0080\n" /* set PCL bit in new */ | |
119 | " csg %0,%1,%2\n" | |
120 | " jl 0b\n" | |
121 | : "=&d" (old), "=&d" (new), "=Q" (ptep[PTRS_PER_PTE]) | |
122 | : "Q" (ptep[PTRS_PER_PTE]) : "cc", "memory"); | |
123 | #endif | |
124 | return __pgste(new); | |
125 | } | |
126 | ||
127 | static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste) | |
128 | { | |
129 | #ifdef CONFIG_PGSTE | |
130 | asm( | |
131 | " nihh %1,0xff7f\n" /* clear PCL bit */ | |
132 | " stg %1,%0\n" | |
133 | : "=Q" (ptep[PTRS_PER_PTE]) | |
134 | : "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE]) | |
135 | : "cc", "memory"); | |
1e133ab2 MS |
136 | #endif |
137 | } | |
138 | ||
139 | static inline pgste_t pgste_get(pte_t *ptep) | |
140 | { | |
141 | unsigned long pgste = 0; | |
142 | #ifdef CONFIG_PGSTE | |
143 | pgste = *(unsigned long *)(ptep + PTRS_PER_PTE); | |
144 | #endif | |
145 | return __pgste(pgste); | |
146 | } | |
147 | ||
148 | static inline void pgste_set(pte_t *ptep, pgste_t pgste) | |
149 | { | |
150 | #ifdef CONFIG_PGSTE | |
151 | *(pgste_t *)(ptep + PTRS_PER_PTE) = pgste; | |
152 | #endif | |
153 | } | |
154 | ||
ebde765c MS |
155 | static inline pgste_t pgste_update_all(pte_t pte, pgste_t pgste, |
156 | struct mm_struct *mm) | |
157 | { | |
158 | #ifdef CONFIG_PGSTE | |
159 | unsigned long address, bits, skey; | |
160 | ||
161 | if (!mm_use_skey(mm) || pte_val(pte) & _PAGE_INVALID) | |
162 | return pgste; | |
163 | address = pte_val(pte) & PAGE_MASK; | |
164 | skey = (unsigned long) page_get_storage_key(address); | |
165 | bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED); | |
166 | /* Transfer page changed & referenced bit to guest bits in pgste */ | |
167 | pgste_val(pgste) |= bits << 48; /* GR bit & GC bit */ | |
168 | /* Copy page access key and fetch protection bit to pgste */ | |
169 | pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT); | |
170 | pgste_val(pgste) |= (skey & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56; | |
171 | #endif | |
172 | return pgste; | |
173 | ||
174 | } | |
175 | ||
176 | static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry, | |
177 | struct mm_struct *mm) | |
178 | { | |
179 | #ifdef CONFIG_PGSTE | |
180 | unsigned long address; | |
181 | unsigned long nkey; | |
182 | ||
183 | if (!mm_use_skey(mm) || pte_val(entry) & _PAGE_INVALID) | |
184 | return; | |
185 | VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID)); | |
186 | address = pte_val(entry) & PAGE_MASK; | |
187 | /* | |
188 | * Set page access key and fetch protection bit from pgste. | |
189 | * The guest C/R information is still in the PGSTE, set real | |
190 | * key C/R to 0. | |
191 | */ | |
192 | nkey = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56; | |
193 | nkey |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48; | |
194 | page_set_storage_key(address, nkey, 0); | |
195 | #endif | |
196 | } | |
197 | ||
198 | static inline pgste_t pgste_set_pte(pte_t *ptep, pgste_t pgste, pte_t entry) | |
199 | { | |
200 | #ifdef CONFIG_PGSTE | |
201 | if ((pte_val(entry) & _PAGE_PRESENT) && | |
202 | (pte_val(entry) & _PAGE_WRITE) && | |
203 | !(pte_val(entry) & _PAGE_INVALID)) { | |
204 | if (!MACHINE_HAS_ESOP) { | |
205 | /* | |
206 | * Without enhanced suppression-on-protection force | |
207 | * the dirty bit on for all writable ptes. | |
208 | */ | |
209 | pte_val(entry) |= _PAGE_DIRTY; | |
210 | pte_val(entry) &= ~_PAGE_PROTECT; | |
211 | } | |
212 | if (!(pte_val(entry) & _PAGE_PROTECT)) | |
213 | /* This pte allows write access, set user-dirty */ | |
214 | pgste_val(pgste) |= PGSTE_UC_BIT; | |
215 | } | |
216 | #endif | |
217 | *ptep = entry; | |
218 | return pgste; | |
219 | } | |
220 | ||
b2d73b2a MS |
221 | static inline pgste_t pgste_pte_notify(struct mm_struct *mm, |
222 | unsigned long addr, | |
223 | pte_t *ptep, pgste_t pgste) | |
ebde765c MS |
224 | { |
225 | #ifdef CONFIG_PGSTE | |
4be130a0 MS |
226 | unsigned long bits; |
227 | ||
228 | bits = pgste_val(pgste) & (PGSTE_IN_BIT | PGSTE_VSIE_BIT); | |
229 | if (bits) { | |
230 | pgste_val(pgste) ^= bits; | |
231 | ptep_notify(mm, addr, ptep, bits); | |
ebde765c MS |
232 | } |
233 | #endif | |
234 | return pgste; | |
235 | } | |
236 | ||
ebde765c MS |
237 | static inline pgste_t ptep_xchg_start(struct mm_struct *mm, |
238 | unsigned long addr, pte_t *ptep) | |
239 | { | |
240 | pgste_t pgste = __pgste(0); | |
241 | ||
242 | if (mm_has_pgste(mm)) { | |
243 | pgste = pgste_get_lock(ptep); | |
b2d73b2a | 244 | pgste = pgste_pte_notify(mm, addr, ptep, pgste); |
ebde765c MS |
245 | } |
246 | return pgste; | |
247 | } | |
248 | ||
0d6da872 | 249 | static inline pte_t ptep_xchg_commit(struct mm_struct *mm, |
ebde765c MS |
250 | unsigned long addr, pte_t *ptep, |
251 | pgste_t pgste, pte_t old, pte_t new) | |
252 | { | |
253 | if (mm_has_pgste(mm)) { | |
254 | if (pte_val(old) & _PAGE_INVALID) | |
255 | pgste_set_key(ptep, pgste, new, mm); | |
256 | if (pte_val(new) & _PAGE_INVALID) { | |
257 | pgste = pgste_update_all(old, pgste, mm); | |
258 | if ((pgste_val(pgste) & _PGSTE_GPS_USAGE_MASK) == | |
259 | _PGSTE_GPS_USAGE_UNUSED) | |
260 | pte_val(old) |= _PAGE_UNUSED; | |
261 | } | |
262 | pgste = pgste_set_pte(ptep, pgste, new); | |
263 | pgste_set_unlock(ptep, pgste); | |
264 | } else { | |
265 | *ptep = new; | |
266 | } | |
0d6da872 | 267 | return old; |
ebde765c MS |
268 | } |
269 | ||
270 | pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr, | |
271 | pte_t *ptep, pte_t new) | |
272 | { | |
273 | pgste_t pgste; | |
274 | pte_t old; | |
cd774b90 | 275 | int nodat; |
ebde765c | 276 | |
a9809407 | 277 | preempt_disable(); |
ebde765c | 278 | pgste = ptep_xchg_start(mm, addr, ptep); |
cd774b90 MS |
279 | nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT); |
280 | old = ptep_flush_direct(mm, addr, ptep, nodat); | |
0d6da872 | 281 | old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new); |
a9809407 | 282 | preempt_enable(); |
ebde765c MS |
283 | return old; |
284 | } | |
285 | EXPORT_SYMBOL(ptep_xchg_direct); | |
286 | ||
287 | pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr, | |
288 | pte_t *ptep, pte_t new) | |
289 | { | |
290 | pgste_t pgste; | |
291 | pte_t old; | |
cd774b90 | 292 | int nodat; |
ebde765c | 293 | |
a9809407 | 294 | preempt_disable(); |
ebde765c | 295 | pgste = ptep_xchg_start(mm, addr, ptep); |
cd774b90 MS |
296 | nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT); |
297 | old = ptep_flush_lazy(mm, addr, ptep, nodat); | |
0d6da872 | 298 | old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new); |
a9809407 | 299 | preempt_enable(); |
ebde765c MS |
300 | return old; |
301 | } | |
302 | EXPORT_SYMBOL(ptep_xchg_lazy); | |
303 | ||
304 | pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr, | |
305 | pte_t *ptep) | |
306 | { | |
307 | pgste_t pgste; | |
308 | pte_t old; | |
cd774b90 | 309 | int nodat; |
ebde765c | 310 | |
a9809407 | 311 | preempt_disable(); |
ebde765c | 312 | pgste = ptep_xchg_start(mm, addr, ptep); |
cd774b90 MS |
313 | nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT); |
314 | old = ptep_flush_lazy(mm, addr, ptep, nodat); | |
ebde765c MS |
315 | if (mm_has_pgste(mm)) { |
316 | pgste = pgste_update_all(old, pgste, mm); | |
317 | pgste_set(ptep, pgste); | |
318 | } | |
319 | return old; | |
320 | } | |
321 | EXPORT_SYMBOL(ptep_modify_prot_start); | |
322 | ||
323 | void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr, | |
324 | pte_t *ptep, pte_t pte) | |
325 | { | |
326 | pgste_t pgste; | |
327 | ||
57d7f939 MS |
328 | if (!MACHINE_HAS_NX) |
329 | pte_val(pte) &= ~_PAGE_NOEXEC; | |
ebde765c MS |
330 | if (mm_has_pgste(mm)) { |
331 | pgste = pgste_get(ptep); | |
332 | pgste_set_key(ptep, pgste, pte, mm); | |
333 | pgste = pgste_set_pte(ptep, pgste, pte); | |
334 | pgste_set_unlock(ptep, pgste); | |
335 | } else { | |
336 | *ptep = pte; | |
337 | } | |
a9809407 | 338 | preempt_enable(); |
ebde765c MS |
339 | } |
340 | EXPORT_SYMBOL(ptep_modify_prot_commit); | |
341 | ||
118bd31b MS |
342 | static inline void pmdp_idte_local(struct mm_struct *mm, |
343 | unsigned long addr, pmd_t *pmdp) | |
344 | { | |
345 | if (MACHINE_HAS_TLB_GUEST) | |
28c807e5 MS |
346 | __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE, |
347 | mm->context.asce, IDTE_LOCAL); | |
118bd31b | 348 | else |
28c807e5 | 349 | __pmdp_idte(addr, pmdp, 0, 0, IDTE_LOCAL); |
118bd31b MS |
350 | } |
351 | ||
352 | static inline void pmdp_idte_global(struct mm_struct *mm, | |
353 | unsigned long addr, pmd_t *pmdp) | |
354 | { | |
355 | if (MACHINE_HAS_TLB_GUEST) | |
28c807e5 MS |
356 | __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE, |
357 | mm->context.asce, IDTE_GLOBAL); | |
118bd31b | 358 | else if (MACHINE_HAS_IDTE) |
28c807e5 | 359 | __pmdp_idte(addr, pmdp, 0, 0, IDTE_GLOBAL); |
118bd31b MS |
360 | else |
361 | __pmdp_csp(pmdp); | |
362 | } | |
363 | ||
227be799 MS |
364 | static inline pmd_t pmdp_flush_direct(struct mm_struct *mm, |
365 | unsigned long addr, pmd_t *pmdp) | |
366 | { | |
227be799 MS |
367 | pmd_t old; |
368 | ||
369 | old = *pmdp; | |
370 | if (pmd_val(old) & _SEGMENT_ENTRY_INVALID) | |
371 | return old; | |
64f31d58 MS |
372 | atomic_inc(&mm->context.flush_count); |
373 | if (MACHINE_HAS_TLB_LC && | |
227be799 | 374 | cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) |
118bd31b | 375 | pmdp_idte_local(mm, addr, pmdp); |
227be799 | 376 | else |
118bd31b | 377 | pmdp_idte_global(mm, addr, pmdp); |
64f31d58 | 378 | atomic_dec(&mm->context.flush_count); |
227be799 MS |
379 | return old; |
380 | } | |
381 | ||
382 | static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm, | |
383 | unsigned long addr, pmd_t *pmdp) | |
384 | { | |
227be799 MS |
385 | pmd_t old; |
386 | ||
387 | old = *pmdp; | |
388 | if (pmd_val(old) & _SEGMENT_ENTRY_INVALID) | |
389 | return old; | |
64f31d58 MS |
390 | atomic_inc(&mm->context.flush_count); |
391 | if (cpumask_equal(&mm->context.cpu_attach_mask, | |
392 | cpumask_of(smp_processor_id()))) { | |
227be799 MS |
393 | pmd_val(*pmdp) |= _SEGMENT_ENTRY_INVALID; |
394 | mm->context.flush_mm = 1; | |
118bd31b MS |
395 | } else { |
396 | pmdp_idte_global(mm, addr, pmdp); | |
397 | } | |
64f31d58 | 398 | atomic_dec(&mm->context.flush_count); |
227be799 MS |
399 | return old; |
400 | } | |
401 | ||
402 | pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr, | |
403 | pmd_t *pmdp, pmd_t new) | |
404 | { | |
405 | pmd_t old; | |
406 | ||
a9809407 | 407 | preempt_disable(); |
227be799 MS |
408 | old = pmdp_flush_direct(mm, addr, pmdp); |
409 | *pmdp = new; | |
a9809407 | 410 | preempt_enable(); |
227be799 MS |
411 | return old; |
412 | } | |
413 | EXPORT_SYMBOL(pmdp_xchg_direct); | |
414 | ||
415 | pmd_t pmdp_xchg_lazy(struct mm_struct *mm, unsigned long addr, | |
416 | pmd_t *pmdp, pmd_t new) | |
417 | { | |
418 | pmd_t old; | |
419 | ||
a9809407 | 420 | preempt_disable(); |
227be799 MS |
421 | old = pmdp_flush_lazy(mm, addr, pmdp); |
422 | *pmdp = new; | |
a9809407 | 423 | preempt_enable(); |
227be799 MS |
424 | return old; |
425 | } | |
426 | EXPORT_SYMBOL(pmdp_xchg_lazy); | |
427 | ||
118bd31b MS |
428 | static inline void pudp_idte_local(struct mm_struct *mm, |
429 | unsigned long addr, pud_t *pudp) | |
d08de8e2 | 430 | { |
118bd31b | 431 | if (MACHINE_HAS_TLB_GUEST) |
28c807e5 MS |
432 | __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE, |
433 | mm->context.asce, IDTE_LOCAL); | |
118bd31b | 434 | else |
28c807e5 | 435 | __pudp_idte(addr, pudp, 0, 0, IDTE_LOCAL); |
118bd31b | 436 | } |
d08de8e2 | 437 | |
118bd31b MS |
438 | static inline void pudp_idte_global(struct mm_struct *mm, |
439 | unsigned long addr, pud_t *pudp) | |
440 | { | |
441 | if (MACHINE_HAS_TLB_GUEST) | |
28c807e5 MS |
442 | __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE, |
443 | mm->context.asce, IDTE_GLOBAL); | |
118bd31b | 444 | else if (MACHINE_HAS_IDTE) |
28c807e5 | 445 | __pudp_idte(addr, pudp, 0, 0, IDTE_GLOBAL); |
118bd31b | 446 | else |
d08de8e2 GS |
447 | /* |
448 | * Invalid bit position is the same for pmd and pud, so we can | |
449 | * re-use _pmd_csp() here | |
450 | */ | |
451 | __pmdp_csp((pmd_t *) pudp); | |
118bd31b MS |
452 | } |
453 | ||
454 | static inline pud_t pudp_flush_direct(struct mm_struct *mm, | |
455 | unsigned long addr, pud_t *pudp) | |
456 | { | |
457 | pud_t old; | |
458 | ||
459 | old = *pudp; | |
460 | if (pud_val(old) & _REGION_ENTRY_INVALID) | |
d08de8e2 | 461 | return old; |
d08de8e2 GS |
462 | atomic_inc(&mm->context.flush_count); |
463 | if (MACHINE_HAS_TLB_LC && | |
464 | cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) | |
118bd31b | 465 | pudp_idte_local(mm, addr, pudp); |
d08de8e2 | 466 | else |
118bd31b | 467 | pudp_idte_global(mm, addr, pudp); |
d08de8e2 GS |
468 | atomic_dec(&mm->context.flush_count); |
469 | return old; | |
470 | } | |
471 | ||
472 | pud_t pudp_xchg_direct(struct mm_struct *mm, unsigned long addr, | |
473 | pud_t *pudp, pud_t new) | |
474 | { | |
475 | pud_t old; | |
476 | ||
477 | preempt_disable(); | |
478 | old = pudp_flush_direct(mm, addr, pudp); | |
479 | *pudp = new; | |
480 | preempt_enable(); | |
481 | return old; | |
482 | } | |
483 | EXPORT_SYMBOL(pudp_xchg_direct); | |
484 | ||
75077afb | 485 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
6b0b50b0 AK |
486 | void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, |
487 | pgtable_t pgtable) | |
9501d09f GS |
488 | { |
489 | struct list_head *lh = (struct list_head *) pgtable; | |
490 | ||
ec66ad66 | 491 | assert_spin_locked(pmd_lockptr(mm, pmdp)); |
9501d09f GS |
492 | |
493 | /* FIFO */ | |
c389a250 | 494 | if (!pmd_huge_pte(mm, pmdp)) |
9501d09f GS |
495 | INIT_LIST_HEAD(lh); |
496 | else | |
c389a250 KS |
497 | list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp)); |
498 | pmd_huge_pte(mm, pmdp) = pgtable; | |
9501d09f GS |
499 | } |
500 | ||
6b0b50b0 | 501 | pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) |
9501d09f GS |
502 | { |
503 | struct list_head *lh; | |
504 | pgtable_t pgtable; | |
505 | pte_t *ptep; | |
506 | ||
ec66ad66 | 507 | assert_spin_locked(pmd_lockptr(mm, pmdp)); |
9501d09f GS |
508 | |
509 | /* FIFO */ | |
c389a250 | 510 | pgtable = pmd_huge_pte(mm, pmdp); |
9501d09f GS |
511 | lh = (struct list_head *) pgtable; |
512 | if (list_empty(lh)) | |
c389a250 | 513 | pmd_huge_pte(mm, pmdp) = NULL; |
9501d09f | 514 | else { |
c389a250 | 515 | pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next; |
9501d09f GS |
516 | list_del(lh); |
517 | } | |
518 | ptep = (pte_t *) pgtable; | |
e5098611 | 519 | pte_val(*ptep) = _PAGE_INVALID; |
9501d09f | 520 | ptep++; |
e5098611 | 521 | pte_val(*ptep) = _PAGE_INVALID; |
9501d09f GS |
522 | return pgtable; |
523 | } | |
75077afb | 524 | #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ |
1e133ab2 MS |
525 | |
526 | #ifdef CONFIG_PGSTE | |
527 | void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr, | |
528 | pte_t *ptep, pte_t entry) | |
529 | { | |
530 | pgste_t pgste; | |
531 | ||
532 | /* the mm_has_pgste() check is done in set_pte_at() */ | |
a9809407 | 533 | preempt_disable(); |
1e133ab2 MS |
534 | pgste = pgste_get_lock(ptep); |
535 | pgste_val(pgste) &= ~_PGSTE_GPS_ZERO; | |
536 | pgste_set_key(ptep, pgste, entry, mm); | |
537 | pgste = pgste_set_pte(ptep, pgste, entry); | |
538 | pgste_set_unlock(ptep, pgste); | |
a9809407 | 539 | preempt_enable(); |
1e133ab2 MS |
540 | } |
541 | ||
542 | void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep) | |
543 | { | |
544 | pgste_t pgste; | |
545 | ||
a9809407 | 546 | preempt_disable(); |
1e133ab2 MS |
547 | pgste = pgste_get_lock(ptep); |
548 | pgste_val(pgste) |= PGSTE_IN_BIT; | |
549 | pgste_set_unlock(ptep, pgste); | |
a9809407 | 550 | preempt_enable(); |
1e133ab2 MS |
551 | } |
552 | ||
b2d73b2a MS |
553 | /** |
554 | * ptep_force_prot - change access rights of a locked pte | |
555 | * @mm: pointer to the process mm_struct | |
556 | * @addr: virtual address in the guest address space | |
557 | * @ptep: pointer to the page table entry | |
558 | * @prot: indicates guest access rights: PROT_NONE, PROT_READ or PROT_WRITE | |
4be130a0 | 559 | * @bit: pgste bit to set (e.g. for notification) |
b2d73b2a MS |
560 | * |
561 | * Returns 0 if the access rights were changed and -EAGAIN if the current | |
562 | * and requested access rights are incompatible. | |
563 | */ | |
564 | int ptep_force_prot(struct mm_struct *mm, unsigned long addr, | |
4be130a0 | 565 | pte_t *ptep, int prot, unsigned long bit) |
b2d73b2a MS |
566 | { |
567 | pte_t entry; | |
568 | pgste_t pgste; | |
cd774b90 | 569 | int pte_i, pte_p, nodat; |
b2d73b2a MS |
570 | |
571 | pgste = pgste_get_lock(ptep); | |
572 | entry = *ptep; | |
573 | /* Check pte entry after all locks have been acquired */ | |
574 | pte_i = pte_val(entry) & _PAGE_INVALID; | |
575 | pte_p = pte_val(entry) & _PAGE_PROTECT; | |
576 | if ((pte_i && (prot != PROT_NONE)) || | |
577 | (pte_p && (prot & PROT_WRITE))) { | |
578 | pgste_set_unlock(ptep, pgste); | |
579 | return -EAGAIN; | |
580 | } | |
4be130a0 | 581 | /* Change access rights and set pgste bit */ |
cd774b90 | 582 | nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT); |
b2d73b2a | 583 | if (prot == PROT_NONE && !pte_i) { |
cd774b90 | 584 | ptep_flush_direct(mm, addr, ptep, nodat); |
b2d73b2a MS |
585 | pgste = pgste_update_all(entry, pgste, mm); |
586 | pte_val(entry) |= _PAGE_INVALID; | |
587 | } | |
588 | if (prot == PROT_READ && !pte_p) { | |
cd774b90 | 589 | ptep_flush_direct(mm, addr, ptep, nodat); |
b2d73b2a MS |
590 | pte_val(entry) &= ~_PAGE_INVALID; |
591 | pte_val(entry) |= _PAGE_PROTECT; | |
592 | } | |
4be130a0 | 593 | pgste_val(pgste) |= bit; |
b2d73b2a MS |
594 | pgste = pgste_set_pte(ptep, pgste, entry); |
595 | pgste_set_unlock(ptep, pgste); | |
596 | return 0; | |
597 | } | |
598 | ||
4be130a0 | 599 | int ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr, |
a9d23e71 | 600 | pte_t *sptep, pte_t *tptep, pte_t pte) |
4be130a0 MS |
601 | { |
602 | pgste_t spgste, tpgste; | |
603 | pte_t spte, tpte; | |
604 | int rc = -EAGAIN; | |
605 | ||
a9d23e71 DH |
606 | if (!(pte_val(*tptep) & _PAGE_INVALID)) |
607 | return 0; /* already shadowed */ | |
4be130a0 MS |
608 | spgste = pgste_get_lock(sptep); |
609 | spte = *sptep; | |
610 | if (!(pte_val(spte) & _PAGE_INVALID) && | |
a9d23e71 DH |
611 | !((pte_val(spte) & _PAGE_PROTECT) && |
612 | !(pte_val(pte) & _PAGE_PROTECT))) { | |
4be130a0 MS |
613 | pgste_val(spgste) |= PGSTE_VSIE_BIT; |
614 | tpgste = pgste_get_lock(tptep); | |
615 | pte_val(tpte) = (pte_val(spte) & PAGE_MASK) | | |
a9d23e71 | 616 | (pte_val(pte) & _PAGE_PROTECT); |
4be130a0 MS |
617 | /* don't touch the storage key - it belongs to parent pgste */ |
618 | tpgste = pgste_set_pte(tptep, tpgste, tpte); | |
619 | pgste_set_unlock(tptep, tpgste); | |
a9d23e71 | 620 | rc = 1; |
4be130a0 MS |
621 | } |
622 | pgste_set_unlock(sptep, spgste); | |
623 | return rc; | |
624 | } | |
625 | ||
626 | void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep) | |
627 | { | |
628 | pgste_t pgste; | |
cd774b90 | 629 | int nodat; |
4be130a0 MS |
630 | |
631 | pgste = pgste_get_lock(ptep); | |
632 | /* notifier is called by the caller */ | |
cd774b90 MS |
633 | nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT); |
634 | ptep_flush_direct(mm, saddr, ptep, nodat); | |
4be130a0 MS |
635 | /* don't touch the storage key - it belongs to parent pgste */ |
636 | pgste = pgste_set_pte(ptep, pgste, __pte(_PAGE_INVALID)); | |
637 | pgste_set_unlock(ptep, pgste); | |
638 | } | |
639 | ||
1e133ab2 MS |
640 | static void ptep_zap_swap_entry(struct mm_struct *mm, swp_entry_t entry) |
641 | { | |
642 | if (!non_swap_entry(entry)) | |
643 | dec_mm_counter(mm, MM_SWAPENTS); | |
644 | else if (is_migration_entry(entry)) { | |
645 | struct page *page = migration_entry_to_page(entry); | |
646 | ||
647 | dec_mm_counter(mm, mm_counter(page)); | |
648 | } | |
649 | free_swap_and_cache(entry); | |
650 | } | |
651 | ||
652 | void ptep_zap_unused(struct mm_struct *mm, unsigned long addr, | |
653 | pte_t *ptep, int reset) | |
654 | { | |
655 | unsigned long pgstev; | |
656 | pgste_t pgste; | |
657 | pte_t pte; | |
658 | ||
659 | /* Zap unused and logically-zero pages */ | |
a9809407 | 660 | preempt_disable(); |
1e133ab2 MS |
661 | pgste = pgste_get_lock(ptep); |
662 | pgstev = pgste_val(pgste); | |
663 | pte = *ptep; | |
1c343f7b | 664 | if (!reset && pte_swap(pte) && |
1e133ab2 MS |
665 | ((pgstev & _PGSTE_GPS_USAGE_MASK) == _PGSTE_GPS_USAGE_UNUSED || |
666 | (pgstev & _PGSTE_GPS_ZERO))) { | |
667 | ptep_zap_swap_entry(mm, pte_to_swp_entry(pte)); | |
668 | pte_clear(mm, addr, ptep); | |
669 | } | |
670 | if (reset) | |
671 | pgste_val(pgste) &= ~_PGSTE_GPS_USAGE_MASK; | |
672 | pgste_set_unlock(ptep, pgste); | |
a9809407 | 673 | preempt_enable(); |
1e133ab2 MS |
674 | } |
675 | ||
676 | void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep) | |
677 | { | |
678 | unsigned long ptev; | |
679 | pgste_t pgste; | |
680 | ||
97ca7bfc | 681 | /* Clear storage key ACC and F, but set R/C */ |
a9809407 | 682 | preempt_disable(); |
1e133ab2 | 683 | pgste = pgste_get_lock(ptep); |
97ca7bfc CB |
684 | pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT); |
685 | pgste_val(pgste) |= PGSTE_GR_BIT | PGSTE_GC_BIT; | |
1e133ab2 MS |
686 | ptev = pte_val(*ptep); |
687 | if (!(ptev & _PAGE_INVALID) && (ptev & _PAGE_WRITE)) | |
688 | page_set_storage_key(ptev & PAGE_MASK, PAGE_DEFAULT_KEY, 1); | |
689 | pgste_set_unlock(ptep, pgste); | |
a9809407 | 690 | preempt_enable(); |
1e133ab2 MS |
691 | } |
692 | ||
693 | /* | |
694 | * Test and reset if a guest page is dirty | |
695 | */ | |
696 | bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long addr) | |
697 | { | |
698 | spinlock_t *ptl; | |
2e4d8800 | 699 | pgd_t *pgd; |
1aea9b3f | 700 | p4d_t *p4d; |
2e4d8800 JF |
701 | pud_t *pud; |
702 | pmd_t *pmd; | |
1e133ab2 MS |
703 | pgste_t pgste; |
704 | pte_t *ptep; | |
705 | pte_t pte; | |
706 | bool dirty; | |
cd774b90 | 707 | int nodat; |
1e133ab2 | 708 | |
2e4d8800 | 709 | pgd = pgd_offset(mm, addr); |
1aea9b3f MS |
710 | p4d = p4d_alloc(mm, pgd, addr); |
711 | if (!p4d) | |
712 | return false; | |
713 | pud = pud_alloc(mm, p4d, addr); | |
2e4d8800 JF |
714 | if (!pud) |
715 | return false; | |
716 | pmd = pmd_alloc(mm, pud, addr); | |
717 | if (!pmd) | |
718 | return false; | |
719 | /* We can't run guests backed by huge pages, but userspace can | |
720 | * still set them up and then try to migrate them without any | |
721 | * migration support. | |
722 | */ | |
723 | if (pmd_large(*pmd)) | |
724 | return true; | |
725 | ||
726 | ptep = pte_alloc_map_lock(mm, pmd, addr, &ptl); | |
1e133ab2 MS |
727 | if (unlikely(!ptep)) |
728 | return false; | |
729 | ||
730 | pgste = pgste_get_lock(ptep); | |
731 | dirty = !!(pgste_val(pgste) & PGSTE_UC_BIT); | |
732 | pgste_val(pgste) &= ~PGSTE_UC_BIT; | |
733 | pte = *ptep; | |
734 | if (dirty && (pte_val(pte) & _PAGE_PRESENT)) { | |
b2d73b2a | 735 | pgste = pgste_pte_notify(mm, addr, ptep, pgste); |
cd774b90 MS |
736 | nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT); |
737 | ptep_ipte_global(mm, addr, ptep, nodat); | |
1e133ab2 MS |
738 | if (MACHINE_HAS_ESOP || !(pte_val(pte) & _PAGE_WRITE)) |
739 | pte_val(pte) |= _PAGE_PROTECT; | |
740 | else | |
741 | pte_val(pte) |= _PAGE_INVALID; | |
742 | *ptep = pte; | |
743 | } | |
744 | pgste_set_unlock(ptep, pgste); | |
745 | ||
746 | spin_unlock(ptl); | |
747 | return dirty; | |
748 | } | |
749 | EXPORT_SYMBOL_GPL(test_and_clear_guest_dirty); | |
750 | ||
751 | int set_guest_storage_key(struct mm_struct *mm, unsigned long addr, | |
752 | unsigned char key, bool nq) | |
753 | { | |
754 | unsigned long keyul; | |
755 | spinlock_t *ptl; | |
756 | pgste_t old, new; | |
757 | pte_t *ptep; | |
758 | ||
1e133ab2 | 759 | ptep = get_locked_pte(mm, addr, &ptl); |
d3ed1cee | 760 | if (unlikely(!ptep)) |
1e133ab2 | 761 | return -EFAULT; |
1e133ab2 MS |
762 | |
763 | new = old = pgste_get_lock(ptep); | |
764 | pgste_val(new) &= ~(PGSTE_GR_BIT | PGSTE_GC_BIT | | |
765 | PGSTE_ACC_BITS | PGSTE_FP_BIT); | |
766 | keyul = (unsigned long) key; | |
767 | pgste_val(new) |= (keyul & (_PAGE_CHANGED | _PAGE_REFERENCED)) << 48; | |
768 | pgste_val(new) |= (keyul & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56; | |
769 | if (!(pte_val(*ptep) & _PAGE_INVALID)) { | |
770 | unsigned long address, bits, skey; | |
771 | ||
772 | address = pte_val(*ptep) & PAGE_MASK; | |
773 | skey = (unsigned long) page_get_storage_key(address); | |
774 | bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED); | |
775 | skey = key & (_PAGE_ACC_BITS | _PAGE_FP_BIT); | |
776 | /* Set storage key ACC and FP */ | |
777 | page_set_storage_key(address, skey, !nq); | |
778 | /* Merge host changed & referenced into pgste */ | |
779 | pgste_val(new) |= bits << 52; | |
780 | } | |
781 | /* changing the guest storage key is considered a change of the page */ | |
782 | if ((pgste_val(new) ^ pgste_val(old)) & | |
783 | (PGSTE_ACC_BITS | PGSTE_FP_BIT | PGSTE_GR_BIT | PGSTE_GC_BIT)) | |
784 | pgste_val(new) |= PGSTE_UC_BIT; | |
785 | ||
786 | pgste_set_unlock(ptep, new); | |
787 | pte_unmap_unlock(ptep, ptl); | |
1e133ab2 MS |
788 | return 0; |
789 | } | |
790 | EXPORT_SYMBOL(set_guest_storage_key); | |
791 | ||
1824c723 DH |
792 | /** |
793 | * Conditionally set a guest storage key (handling csske). | |
794 | * oldkey will be updated when either mr or mc is set and a pointer is given. | |
795 | * | |
796 | * Returns 0 if a guests storage key update wasn't necessary, 1 if the guest | |
797 | * storage key was updated and -EFAULT on access errors. | |
798 | */ | |
799 | int cond_set_guest_storage_key(struct mm_struct *mm, unsigned long addr, | |
800 | unsigned char key, unsigned char *oldkey, | |
801 | bool nq, bool mr, bool mc) | |
802 | { | |
803 | unsigned char tmp, mask = _PAGE_ACC_BITS | _PAGE_FP_BIT; | |
804 | int rc; | |
805 | ||
806 | /* we can drop the pgste lock between getting and setting the key */ | |
807 | if (mr | mc) { | |
808 | rc = get_guest_storage_key(current->mm, addr, &tmp); | |
809 | if (rc) | |
810 | return rc; | |
811 | if (oldkey) | |
812 | *oldkey = tmp; | |
813 | if (!mr) | |
814 | mask |= _PAGE_REFERENCED; | |
815 | if (!mc) | |
816 | mask |= _PAGE_CHANGED; | |
817 | if (!((tmp ^ key) & mask)) | |
818 | return 0; | |
819 | } | |
820 | rc = set_guest_storage_key(current->mm, addr, key, nq); | |
821 | return rc < 0 ? rc : 1; | |
822 | } | |
823 | EXPORT_SYMBOL(cond_set_guest_storage_key); | |
824 | ||
a7e19ab5 DH |
825 | /** |
826 | * Reset a guest reference bit (rrbe), returning the reference and changed bit. | |
827 | * | |
828 | * Returns < 0 in case of error, otherwise the cc to be reported to the guest. | |
829 | */ | |
830 | int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr) | |
1e133ab2 | 831 | { |
1e133ab2 | 832 | spinlock_t *ptl; |
a7e19ab5 | 833 | pgste_t old, new; |
1e133ab2 | 834 | pte_t *ptep; |
a7e19ab5 | 835 | int cc = 0; |
1e133ab2 | 836 | |
1e133ab2 | 837 | ptep = get_locked_pte(mm, addr, &ptl); |
a7e19ab5 | 838 | if (unlikely(!ptep)) |
1e133ab2 | 839 | return -EFAULT; |
1e133ab2 | 840 | |
a7e19ab5 DH |
841 | new = old = pgste_get_lock(ptep); |
842 | /* Reset guest reference bit only */ | |
843 | pgste_val(new) &= ~PGSTE_GR_BIT; | |
1e133ab2 | 844 | |
a7e19ab5 DH |
845 | if (!(pte_val(*ptep) & _PAGE_INVALID)) { |
846 | cc = page_reset_referenced(pte_val(*ptep) & PAGE_MASK); | |
847 | /* Merge real referenced bit into host-set */ | |
848 | pgste_val(new) |= ((unsigned long) cc << 53) & PGSTE_HR_BIT; | |
1e133ab2 | 849 | } |
a7e19ab5 DH |
850 | /* Reflect guest's logical view, not physical */ |
851 | cc |= (pgste_val(old) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 49; | |
852 | /* Changing the guest storage key is considered a change of the page */ | |
853 | if ((pgste_val(new) ^ pgste_val(old)) & PGSTE_GR_BIT) | |
854 | pgste_val(new) |= PGSTE_UC_BIT; | |
855 | ||
856 | pgste_set_unlock(ptep, new); | |
857 | pte_unmap_unlock(ptep, ptl); | |
4bead2a4 | 858 | return cc; |
a7e19ab5 DH |
859 | } |
860 | EXPORT_SYMBOL(reset_guest_reference_bit); | |
861 | ||
154c8c19 DH |
862 | int get_guest_storage_key(struct mm_struct *mm, unsigned long addr, |
863 | unsigned char *key) | |
1e133ab2 | 864 | { |
1e133ab2 MS |
865 | spinlock_t *ptl; |
866 | pgste_t pgste; | |
867 | pte_t *ptep; | |
1e133ab2 | 868 | |
1e133ab2 | 869 | ptep = get_locked_pte(mm, addr, &ptl); |
d3ed1cee | 870 | if (unlikely(!ptep)) |
1e133ab2 | 871 | return -EFAULT; |
1e133ab2 | 872 | |
d3ed1cee | 873 | pgste = pgste_get_lock(ptep); |
154c8c19 | 874 | *key = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56; |
8d6037a7 | 875 | if (!(pte_val(*ptep) & _PAGE_INVALID)) |
154c8c19 | 876 | *key = page_get_storage_key(pte_val(*ptep) & PAGE_MASK); |
8d6037a7 | 877 | /* Reflect guest's logical view, not physical */ |
154c8c19 | 878 | *key |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48; |
1e133ab2 MS |
879 | pgste_set_unlock(ptep, pgste); |
880 | pte_unmap_unlock(ptep, ptl); | |
154c8c19 | 881 | return 0; |
1e133ab2 MS |
882 | } |
883 | EXPORT_SYMBOL(get_guest_storage_key); | |
2d42f947 CI |
884 | |
885 | /** | |
886 | * pgste_perform_essa - perform ESSA actions on the PGSTE. | |
887 | * @mm: the memory context. It must have PGSTEs, no check is performed here! | |
888 | * @hva: the host virtual address of the page whose PGSTE is to be processed | |
889 | * @orc: the specific action to perform, see the ESSA_SET_* macros. | |
890 | * @oldpte: the PTE will be saved there if the pointer is not NULL. | |
891 | * @oldpgste: the old PGSTE will be saved there if the pointer is not NULL. | |
892 | * | |
893 | * Return: 1 if the page is to be added to the CBRL, otherwise 0, | |
894 | * or < 0 in case of error. -EINVAL is returned for invalid values | |
895 | * of orc, -EFAULT for invalid addresses. | |
896 | */ | |
897 | int pgste_perform_essa(struct mm_struct *mm, unsigned long hva, int orc, | |
898 | unsigned long *oldpte, unsigned long *oldpgste) | |
899 | { | |
900 | unsigned long pgstev; | |
901 | spinlock_t *ptl; | |
902 | pgste_t pgste; | |
903 | pte_t *ptep; | |
904 | int res = 0; | |
905 | ||
906 | WARN_ON_ONCE(orc > ESSA_MAX); | |
907 | if (unlikely(orc > ESSA_MAX)) | |
908 | return -EINVAL; | |
909 | ptep = get_locked_pte(mm, hva, &ptl); | |
910 | if (unlikely(!ptep)) | |
911 | return -EFAULT; | |
912 | pgste = pgste_get_lock(ptep); | |
913 | pgstev = pgste_val(pgste); | |
914 | if (oldpte) | |
915 | *oldpte = pte_val(*ptep); | |
916 | if (oldpgste) | |
917 | *oldpgste = pgstev; | |
918 | ||
919 | switch (orc) { | |
920 | case ESSA_GET_STATE: | |
921 | break; | |
922 | case ESSA_SET_STABLE: | |
1bab1c02 | 923 | pgstev &= ~(_PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT); |
2d42f947 CI |
924 | pgstev |= _PGSTE_GPS_USAGE_STABLE; |
925 | break; | |
926 | case ESSA_SET_UNUSED: | |
927 | pgstev &= ~_PGSTE_GPS_USAGE_MASK; | |
928 | pgstev |= _PGSTE_GPS_USAGE_UNUSED; | |
929 | if (pte_val(*ptep) & _PAGE_INVALID) | |
930 | res = 1; | |
931 | break; | |
932 | case ESSA_SET_VOLATILE: | |
933 | pgstev &= ~_PGSTE_GPS_USAGE_MASK; | |
934 | pgstev |= _PGSTE_GPS_USAGE_VOLATILE; | |
935 | if (pte_val(*ptep) & _PAGE_INVALID) | |
936 | res = 1; | |
937 | break; | |
938 | case ESSA_SET_POT_VOLATILE: | |
939 | pgstev &= ~_PGSTE_GPS_USAGE_MASK; | |
940 | if (!(pte_val(*ptep) & _PAGE_INVALID)) { | |
941 | pgstev |= _PGSTE_GPS_USAGE_POT_VOLATILE; | |
942 | break; | |
943 | } | |
944 | if (pgstev & _PGSTE_GPS_ZERO) { | |
945 | pgstev |= _PGSTE_GPS_USAGE_VOLATILE; | |
946 | break; | |
947 | } | |
948 | if (!(pgstev & PGSTE_GC_BIT)) { | |
949 | pgstev |= _PGSTE_GPS_USAGE_VOLATILE; | |
950 | res = 1; | |
951 | break; | |
952 | } | |
953 | break; | |
954 | case ESSA_SET_STABLE_RESIDENT: | |
955 | pgstev &= ~_PGSTE_GPS_USAGE_MASK; | |
956 | pgstev |= _PGSTE_GPS_USAGE_STABLE; | |
957 | /* | |
958 | * Since the resident state can go away any time after this | |
959 | * call, we will not make this page resident. We can revisit | |
960 | * this decision if a guest will ever start using this. | |
961 | */ | |
962 | break; | |
963 | case ESSA_SET_STABLE_IF_RESIDENT: | |
964 | if (!(pte_val(*ptep) & _PAGE_INVALID)) { | |
965 | pgstev &= ~_PGSTE_GPS_USAGE_MASK; | |
966 | pgstev |= _PGSTE_GPS_USAGE_STABLE; | |
967 | } | |
968 | break; | |
1bab1c02 CI |
969 | case ESSA_SET_STABLE_NODAT: |
970 | pgstev &= ~_PGSTE_GPS_USAGE_MASK; | |
971 | pgstev |= _PGSTE_GPS_USAGE_STABLE | _PGSTE_GPS_NODAT; | |
972 | break; | |
2d42f947 CI |
973 | default: |
974 | /* we should never get here! */ | |
975 | break; | |
976 | } | |
977 | /* If we are discarding a page, set it to logical zero */ | |
978 | if (res) | |
979 | pgstev |= _PGSTE_GPS_ZERO; | |
980 | ||
981 | pgste_val(pgste) = pgstev; | |
982 | pgste_set_unlock(ptep, pgste); | |
983 | pte_unmap_unlock(ptep, ptl); | |
984 | return res; | |
985 | } | |
986 | EXPORT_SYMBOL(pgste_perform_essa); | |
987 | ||
988 | /** | |
989 | * set_pgste_bits - set specific PGSTE bits. | |
990 | * @mm: the memory context. It must have PGSTEs, no check is performed here! | |
991 | * @hva: the host virtual address of the page whose PGSTE is to be processed | |
992 | * @bits: a bitmask representing the bits that will be touched | |
993 | * @value: the values of the bits to be written. Only the bits in the mask | |
994 | * will be written. | |
995 | * | |
996 | * Return: 0 on success, < 0 in case of error. | |
997 | */ | |
998 | int set_pgste_bits(struct mm_struct *mm, unsigned long hva, | |
999 | unsigned long bits, unsigned long value) | |
1000 | { | |
1001 | spinlock_t *ptl; | |
1002 | pgste_t new; | |
1003 | pte_t *ptep; | |
1004 | ||
1005 | ptep = get_locked_pte(mm, hva, &ptl); | |
1006 | if (unlikely(!ptep)) | |
1007 | return -EFAULT; | |
1008 | new = pgste_get_lock(ptep); | |
1009 | ||
1010 | pgste_val(new) &= ~bits; | |
1011 | pgste_val(new) |= value & bits; | |
1012 | ||
1013 | pgste_set_unlock(ptep, new); | |
1014 | pte_unmap_unlock(ptep, ptl); | |
1015 | return 0; | |
1016 | } | |
1017 | EXPORT_SYMBOL(set_pgste_bits); | |
1018 | ||
1019 | /** | |
1020 | * get_pgste - get the current PGSTE for the given address. | |
1021 | * @mm: the memory context. It must have PGSTEs, no check is performed here! | |
1022 | * @hva: the host virtual address of the page whose PGSTE is to be processed | |
1023 | * @pgstep: will be written with the current PGSTE for the given address. | |
1024 | * | |
1025 | * Return: 0 on success, < 0 in case of error. | |
1026 | */ | |
1027 | int get_pgste(struct mm_struct *mm, unsigned long hva, unsigned long *pgstep) | |
1028 | { | |
1029 | spinlock_t *ptl; | |
1030 | pte_t *ptep; | |
1031 | ||
1032 | ptep = get_locked_pte(mm, hva, &ptl); | |
1033 | if (unlikely(!ptep)) | |
1034 | return -EFAULT; | |
1035 | *pgstep = pgste_val(pgste_get(ptep)); | |
1036 | pte_unmap_unlock(ptep, ptl); | |
1037 | return 0; | |
1038 | } | |
1039 | EXPORT_SYMBOL(get_pgste); | |
1e133ab2 | 1040 | #endif |