]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
1da177e4 LT |
2 | #ifndef _S390_TLBFLUSH_H |
3 | #define _S390_TLBFLUSH_H | |
4 | ||
1da177e4 | 5 | #include <linux/mm.h> |
53492b1d | 6 | #include <linux/sched.h> |
1da177e4 | 7 | #include <asm/processor.h> |
c1821c2e | 8 | #include <asm/pgalloc.h> |
4ccccc52 | 9 | #include <asm/pgtable.h> |
1da177e4 LT |
10 | |
11 | /* | |
1b948d6c | 12 | * Flush all TLB entries on the local CPU. |
1da177e4 | 13 | */ |
ba8a9229 | 14 | static inline void __tlb_flush_local(void) |
1da177e4 | 15 | { |
ba8a9229 | 16 | asm volatile("ptlb" : : : "memory"); |
1da177e4 | 17 | } |
1da177e4 | 18 | |
ba8a9229 | 19 | /* |
1b948d6c | 20 | * Flush TLB entries for a specific ASCE on all CPUs |
ba8a9229 | 21 | */ |
1b948d6c MS |
22 | static inline void __tlb_flush_idte(unsigned long asce) |
23 | { | |
118bd31b MS |
24 | unsigned long opt; |
25 | ||
26 | opt = IDTE_PTOA; | |
28c807e5 MS |
27 | if (MACHINE_HAS_TLB_GUEST) |
28 | opt |= IDTE_GUEST_ASCE; | |
1b948d6c MS |
29 | /* Global TLB flush for the mm */ |
30 | asm volatile( | |
31 | " .insn rrf,0xb98e0000,0,%0,%1,0" | |
118bd31b | 32 | : : "a" (opt), "a" (asce) : "cc"); |
1b948d6c MS |
33 | } |
34 | ||
1b948d6c | 35 | #ifdef CONFIG_SMP |
a806170e HC |
36 | void smp_ptlb_all(void); |
37 | ||
1b948d6c MS |
38 | /* |
39 | * Flush all TLB entries on all CPUs. | |
40 | */ | |
ba8a9229 | 41 | static inline void __tlb_flush_global(void) |
1da177e4 | 42 | { |
4ccccc52 | 43 | unsigned int dummy = 0; |
94c12cc7 | 44 | |
4ccccc52 | 45 | csp(&dummy, 0, 0); |
1da177e4 LT |
46 | } |
47 | ||
1b948d6c MS |
48 | /* |
49 | * Flush TLB entries for a specific mm on all CPUs (in case gmap is used | |
50 | * this implicates multiple ASCEs!). | |
51 | */ | |
44b6cc81 | 52 | static inline void __tlb_flush_mm(struct mm_struct *mm) |
1b948d6c | 53 | { |
44b6cc81 MS |
54 | unsigned long gmap_asce; |
55 | ||
56 | /* | |
57 | * If the machine has IDTE we prefer to do a per mm flush | |
58 | * on all cpus instead of doing a local flush if the mm | |
59 | * only ran on the local cpu. | |
60 | */ | |
1b948d6c | 61 | preempt_disable(); |
64f31d58 | 62 | atomic_inc(&mm->context.flush_count); |
b3e5dc45 MS |
63 | /* Reset TLB flush mask */ |
64 | cpumask_copy(mm_cpumask(mm), &mm->context.cpu_attach_mask); | |
65 | barrier(); | |
44b6cc81 MS |
66 | gmap_asce = READ_ONCE(mm->context.gmap_asce); |
67 | if (MACHINE_HAS_IDTE && gmap_asce != -1UL) { | |
68 | if (gmap_asce) | |
69 | __tlb_flush_idte(gmap_asce); | |
70 | __tlb_flush_idte(mm->context.asce); | |
71 | } else { | |
b3e5dc45 MS |
72 | /* Global TLB flush */ |
73 | __tlb_flush_global(); | |
44b6cc81 | 74 | } |
64f31d58 | 75 | atomic_dec(&mm->context.flush_count); |
1b948d6c MS |
76 | preempt_enable(); |
77 | } | |
78 | ||
79 | static inline void __tlb_flush_kernel(void) | |
80 | { | |
81 | if (MACHINE_HAS_IDTE) | |
723cacbd | 82 | __tlb_flush_idte(init_mm.context.asce); |
1b948d6c MS |
83 | else |
84 | __tlb_flush_global(); | |
85 | } | |
374b8f45 | 86 | #else |
e1c4d014 | 87 | #define __tlb_flush_global() __tlb_flush_local() |
374b8f45 | 88 | |
1da177e4 | 89 | /* |
1b948d6c | 90 | * Flush TLB entries for a specific ASCE on all CPUs. |
1da177e4 | 91 | */ |
44b6cc81 | 92 | static inline void __tlb_flush_mm(struct mm_struct *mm) |
ba8a9229 | 93 | { |
d5dcafee | 94 | __tlb_flush_local(); |
ba8a9229 | 95 | } |
1da177e4 | 96 | |
1b948d6c MS |
97 | static inline void __tlb_flush_kernel(void) |
98 | { | |
d5dcafee | 99 | __tlb_flush_local(); |
1b948d6c MS |
100 | } |
101 | #endif | |
102 | ||
5c474a1e | 103 | static inline void __tlb_flush_mm_lazy(struct mm_struct * mm) |
1da177e4 | 104 | { |
60f07c8e | 105 | spin_lock(&mm->context.lock); |
050eef36 | 106 | if (mm->context.flush_mm) { |
050eef36 | 107 | mm->context.flush_mm = 0; |
60f07c8e | 108 | __tlb_flush_mm(mm); |
050eef36 | 109 | } |
60f07c8e | 110 | spin_unlock(&mm->context.lock); |
1da177e4 LT |
111 | } |
112 | ||
ba8a9229 MS |
113 | /* |
114 | * TLB flushing: | |
115 | * flush_tlb() - flushes the current mm struct TLBs | |
116 | * flush_tlb_all() - flushes all processes TLBs | |
117 | * flush_tlb_mm(mm) - flushes the specified mm context TLB's | |
118 | * flush_tlb_page(vma, vmaddr) - flushes one page | |
119 | * flush_tlb_range(vma, start, end) - flushes a range of pages | |
120 | * flush_tlb_kernel_range(start, end) - flushes a range of kernel pages | |
121 | */ | |
1da177e4 | 122 | |
ba8a9229 MS |
123 | /* |
124 | * flush_tlb_mm goes together with ptep_set_wrprotect for the | |
125 | * copy_page_range operation and flush_tlb_range is related to | |
126 | * ptep_get_and_clear for change_protection. ptep_set_wrprotect and | |
127 | * ptep_get_and_clear do not flush the TLBs directly if the mm has | |
128 | * only one user. At the end of the update the flush_tlb_mm and | |
129 | * flush_tlb_range functions need to do the flush. | |
130 | */ | |
131 | #define flush_tlb() do { } while (0) | |
132 | #define flush_tlb_all() do { } while (0) | |
ba8a9229 | 133 | #define flush_tlb_page(vma, addr) do { } while (0) |
8ffd74a0 MS |
134 | |
135 | static inline void flush_tlb_mm(struct mm_struct *mm) | |
136 | { | |
5c474a1e | 137 | __tlb_flush_mm_lazy(mm); |
8ffd74a0 MS |
138 | } |
139 | ||
140 | static inline void flush_tlb_range(struct vm_area_struct *vma, | |
141 | unsigned long start, unsigned long end) | |
142 | { | |
5c474a1e | 143 | __tlb_flush_mm_lazy(vma->vm_mm); |
8ffd74a0 MS |
144 | } |
145 | ||
146 | static inline void flush_tlb_kernel_range(unsigned long start, | |
147 | unsigned long end) | |
148 | { | |
1b948d6c | 149 | __tlb_flush_kernel(); |
8ffd74a0 | 150 | } |
1da177e4 | 151 | |
1da177e4 | 152 | #endif /* _S390_TLBFLUSH_H */ |