]>
Commit | Line | Data |
---|---|---|
321d628a FG |
1 | From c8bd55aa2deb4e7491561567e8e01b71ab61ffe4 Mon Sep 17 00:00:00 2001 |
2 | From: Peter Zijlstra <peterz@infradead.org> | |
3 | Date: Tue, 5 Dec 2017 13:34:53 +0100 | |
e4cdf2a5 | 4 | Subject: [PATCH 212/241] x86/mm: Clarify the whole ASID/kernel PCID/user PCID |
321d628a FG |
5 | naming |
6 | MIME-Version: 1.0 | |
7 | Content-Type: text/plain; charset=UTF-8 | |
8 | Content-Transfer-Encoding: 8bit | |
9 | ||
10 | CVE-2017-5754 | |
11 | ||
12 | Ideally we'd also use sparse to enforce this separation so it becomes much | |
13 | more difficult to mess up. | |
14 | ||
15 | Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> | |
16 | Signed-off-by: Thomas Gleixner <tglx@linutronix.de> | |
17 | Cc: Andy Lutomirski <luto@kernel.org> | |
18 | Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com> | |
19 | Cc: Borislav Petkov <bp@alien8.de> | |
20 | Cc: Brian Gerst <brgerst@gmail.com> | |
21 | Cc: Dave Hansen <dave.hansen@linux.intel.com> | |
22 | Cc: David Laight <David.Laight@aculab.com> | |
23 | Cc: Denys Vlasenko <dvlasenk@redhat.com> | |
24 | Cc: Eduardo Valentin <eduval@amazon.com> | |
25 | Cc: Greg KH <gregkh@linuxfoundation.org> | |
26 | Cc: H. Peter Anvin <hpa@zytor.com> | |
27 | Cc: Josh Poimboeuf <jpoimboe@redhat.com> | |
28 | Cc: Juergen Gross <jgross@suse.com> | |
29 | Cc: Linus Torvalds <torvalds@linux-foundation.org> | |
30 | Cc: Peter Zijlstra <peterz@infradead.org> | |
31 | Cc: Will Deacon <will.deacon@arm.com> | |
32 | Cc: aliguori@amazon.com | |
33 | Cc: daniel.gruss@iaik.tugraz.at | |
34 | Cc: hughd@google.com | |
35 | Cc: keescook@google.com | |
36 | Cc: linux-mm@kvack.org | |
37 | Signed-off-by: Ingo Molnar <mingo@kernel.org> | |
38 | (cherry picked from commit 0a126abd576ebc6403f063dbe20cf7416c9d9393) | |
39 | Signed-off-by: Andy Whitcroft <apw@canonical.com> | |
40 | Signed-off-by: Kleber Sacilotto de Souza <kleber.souza@canonical.com> | |
41 | (cherry picked from commit 2ee6efc0f708e21cfd08471132ac2255fac54553) | |
42 | Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com> | |
43 | --- | |
44 | arch/x86/include/asm/tlbflush.h | 55 ++++++++++++++++++++++++++++++++--------- | |
45 | 1 file changed, 43 insertions(+), 12 deletions(-) | |
46 | ||
47 | diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h | |
48 | index 979e590648a5..7a04a1f1ca11 100644 | |
49 | --- a/arch/x86/include/asm/tlbflush.h | |
50 | +++ b/arch/x86/include/asm/tlbflush.h | |
51 | @@ -12,16 +12,33 @@ | |
52 | #include <asm/pti.h> | |
53 | #include <asm/processor-flags.h> | |
54 | ||
55 | -static inline u64 inc_mm_tlb_gen(struct mm_struct *mm) | |
56 | -{ | |
57 | - /* | |
58 | - * Bump the generation count. This also serves as a full barrier | |
59 | - * that synchronizes with switch_mm(): callers are required to order | |
60 | - * their read of mm_cpumask after their writes to the paging | |
61 | - * structures. | |
62 | - */ | |
63 | - return atomic64_inc_return(&mm->context.tlb_gen); | |
64 | -} | |
65 | +/* | |
66 | + * The x86 feature is called PCID (Process Context IDentifier). It is similar | |
67 | + * to what is traditionally called ASID on the RISC processors. | |
68 | + * | |
69 | + * We don't use the traditional ASID implementation, where each process/mm gets | |
70 | + * its own ASID and flush/restart when we run out of ASID space. | |
71 | + * | |
72 | + * Instead we have a small per-cpu array of ASIDs and cache the last few mm's | |
73 | + * that came by on this CPU, allowing cheaper switch_mm between processes on | |
74 | + * this CPU. | |
75 | + * | |
76 | + * We end up with different spaces for different things. To avoid confusion we | |
77 | + * use different names for each of them: | |
78 | + * | |
79 | + * ASID - [0, TLB_NR_DYN_ASIDS-1] | |
80 | + * the canonical identifier for an mm | |
81 | + * | |
82 | + * kPCID - [1, TLB_NR_DYN_ASIDS] | |
83 | + * the value we write into the PCID part of CR3; corresponds to the | |
84 | + * ASID+1, because PCID 0 is special. | |
85 | + * | |
86 | + * uPCID - [2048 + 1, 2048 + TLB_NR_DYN_ASIDS] | |
87 | + * for KPTI each mm has two address spaces and thus needs two | |
88 | + * PCID values, but we can still do with a single ASID denomination | |
89 | + * for each mm. Corresponds to kPCID + 2048. | |
90 | + * | |
91 | + */ | |
92 | ||
93 | /* There are 12 bits of space for ASIDS in CR3 */ | |
94 | #define CR3_HW_ASID_BITS 12 | |
95 | @@ -40,7 +57,7 @@ static inline u64 inc_mm_tlb_gen(struct mm_struct *mm) | |
96 | ||
97 | /* | |
98 | * ASIDs are zero-based: 0->MAX_AVAIL_ASID are valid. -1 below to account | |
99 | - * for them being zero-based. Another -1 is because ASID 0 is reserved for | |
100 | + * for them being zero-based. Another -1 is because PCID 0 is reserved for | |
101 | * use by non-PCID-aware users. | |
102 | */ | |
103 | #define MAX_ASID_AVAILABLE ((1 << CR3_AVAIL_PCID_BITS) - 2) | |
104 | @@ -51,6 +68,9 @@ static inline u64 inc_mm_tlb_gen(struct mm_struct *mm) | |
105 | */ | |
106 | #define TLB_NR_DYN_ASIDS 6 | |
107 | ||
108 | +/* | |
109 | + * Given @asid, compute kPCID | |
110 | + */ | |
111 | static inline u16 kern_pcid(u16 asid) | |
112 | { | |
113 | VM_WARN_ON_ONCE(asid > MAX_ASID_AVAILABLE); | |
114 | @@ -85,7 +105,7 @@ static inline u16 kern_pcid(u16 asid) | |
115 | } | |
116 | ||
117 | /* | |
118 | - * The user PCID is just the kernel one, plus the "switch bit". | |
119 | + * Given @asid, compute uPCID | |
120 | */ | |
121 | static inline u16 user_pcid(u16 asid) | |
122 | { | |
123 | @@ -473,6 +493,17 @@ static inline void flush_tlb_page(struct vm_area_struct *vma, unsigned long a) | |
124 | void native_flush_tlb_others(const struct cpumask *cpumask, | |
125 | const struct flush_tlb_info *info); | |
126 | ||
127 | +static inline u64 inc_mm_tlb_gen(struct mm_struct *mm) | |
128 | +{ | |
129 | + /* | |
130 | + * Bump the generation count. This also serves as a full barrier | |
131 | + * that synchronizes with switch_mm(): callers are required to order | |
132 | + * their read of mm_cpumask after their writes to the paging | |
133 | + * structures. | |
134 | + */ | |
135 | + return atomic64_inc_return(&mm->context.tlb_gen); | |
136 | +} | |
137 | + | |
138 | static inline void arch_tlbbatch_add_mm(struct arch_tlbflush_unmap_batch *batch, | |
139 | struct mm_struct *mm) | |
140 | { | |
141 | -- | |
142 | 2.14.2 | |
143 |