]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blame - arch/x86/include/asm/alternative.h
Merge tag 'pinctrl-v4.1-1' of git://git.kernel.org/pub/scm/linux/kernel/git/linusw...
[mirror_ubuntu-zesty-kernel.git] / arch / x86 / include / asm / alternative.h
CommitLineData
1965aae3
PA
1#ifndef _ASM_X86_ALTERNATIVE_H
2#define _ASM_X86_ALTERNATIVE_H
6b592570
PA
3
4#include <linux/types.h>
5#include <linux/stddef.h>
edc953fa 6#include <linux/stringify.h>
6b592570 7#include <asm/asm.h>
17f41571 8#include <asm/ptrace.h>
6b592570
PA
9
10/*
11 * Alternative inline assembly for SMP.
12 *
13 * The LOCK_PREFIX macro defined here replaces the LOCK and
14 * LOCK_PREFIX macros used everywhere in the source tree.
15 *
16 * SMP alternatives use the same data structures as the other
17 * alternatives and the X86_FEATURE_UP flag to indicate the case of a
18 * UP system running a SMP kernel. The existing apply_alternatives()
19 * works fine for patching a SMP kernel for UP.
20 *
21 * The SMP alternative tables can be kept after boot and contain both
22 * UP and SMP versions of the instructions to allow switching back to
23 * SMP at runtime, when hotplugging in a new CPU, which is especially
24 * useful in virtualized environments.
25 *
26 * The very common lock prefix is handled as special case in a
27 * separate table which is a pure address list without replacement ptr
28 * and size information. That keeps the table sizes small.
29 */
30
31#ifdef CONFIG_SMP
b3ac891b 32#define LOCK_PREFIX_HERE \
9cebed42
PA
33 ".pushsection .smp_locks,\"a\"\n" \
34 ".balign 4\n" \
35 ".long 671f - .\n" /* offset */ \
36 ".popsection\n" \
b3ac891b
LB
37 "671:"
38
39#define LOCK_PREFIX LOCK_PREFIX_HERE "\n\tlock; "
6b592570
PA
40
41#else /* ! CONFIG_SMP */
b701a47b 42#define LOCK_PREFIX_HERE ""
6b592570
PA
43#define LOCK_PREFIX ""
44#endif
45
6b592570 46struct alt_instr {
59e97e4d
AL
47 s32 instr_offset; /* original instruction */
48 s32 repl_offset; /* offset to replacement instruction */
83a7a2ad 49 u16 cpuid; /* cpuid bit set for replacement */
6b592570 50 u8 instrlen; /* length of original instruction */
4332195c
BP
51 u8 replacementlen; /* length of new instruction */
52 u8 padlen; /* length of build-time padding */
53} __packed;
6b592570
PA
54
55extern void alternative_instructions(void);
56extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end);
57
58struct module;
59
60#ifdef CONFIG_SMP
61extern void alternatives_smp_module_add(struct module *mod, char *name,
62 void *locks, void *locks_end,
63 void *text, void *text_end);
64extern void alternatives_smp_module_del(struct module *mod);
816afe4f 65extern void alternatives_enable_smp(void);
2cfa1978 66extern int alternatives_text_reserved(void *start, void *end);
3fb82d56 67extern bool skip_smp_alternatives;
6b592570
PA
68#else
69static inline void alternatives_smp_module_add(struct module *mod, char *name,
2ac1ea7c
JP
70 void *locks, void *locks_end,
71 void *text, void *text_end) {}
6b592570 72static inline void alternatives_smp_module_del(struct module *mod) {}
816afe4f 73static inline void alternatives_enable_smp(void) {}
2cfa1978
MH
74static inline int alternatives_text_reserved(void *start, void *end)
75{
76 return 0;
77}
6b592570
PA
78#endif /* CONFIG_SMP */
79
4332195c
BP
80#define b_replacement(num) "664"#num
81#define e_replacement(num) "665"#num
954e482b 82
4332195c
BP
83#define alt_end_marker "663"
84#define alt_slen "662b-661b"
85#define alt_pad_len alt_end_marker"b-662b"
86#define alt_total_slen alt_end_marker"b-661b"
87#define alt_rlen(num) e_replacement(num)"f-"b_replacement(num)"f"
954e482b 88
4332195c
BP
89#define __OLDINSTR(oldinstr, num) \
90 "661:\n\t" oldinstr "\n662:\n" \
91 ".skip -(((" alt_rlen(num) ")-(" alt_slen ")) > 0) * " \
92 "((" alt_rlen(num) ")-(" alt_slen ")),0x90\n"
954e482b 93
4332195c
BP
94#define OLDINSTR(oldinstr, num) \
95 __OLDINSTR(oldinstr, num) \
96 alt_end_marker ":\n"
97
dbe4058a
BP
98/*
99 * max without conditionals. Idea adapted from:
100 * http://graphics.stanford.edu/~seander/bithacks.html#IntegerMinOrMax
101 *
102 * The additional "-" is needed because gas works with s32s.
103 */
104#define alt_max_short(a, b) "((" a ") ^ (((" a ") ^ (" b ")) & -(-((" a ") - (" b ")))))"
105
4332195c
BP
106/*
107 * Pad the second replacement alternative with additional NOPs if it is
108 * additionally longer than the first replacement alternative.
109 */
dbe4058a
BP
110#define OLDINSTR_2(oldinstr, num1, num2) \
111 "661:\n\t" oldinstr "\n662:\n" \
112 ".skip -((" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - (" alt_slen ")) > 0) * " \
113 "(" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - (" alt_slen ")), 0x90\n" \
4332195c
BP
114 alt_end_marker ":\n"
115
116#define ALTINSTR_ENTRY(feature, num) \
954e482b 117 " .long 661b - .\n" /* label */ \
4332195c 118 " .long " b_replacement(num)"f - .\n" /* new instruction */ \
954e482b 119 " .word " __stringify(feature) "\n" /* feature bit */ \
4332195c
BP
120 " .byte " alt_total_slen "\n" /* source len */ \
121 " .byte " alt_rlen(num) "\n" /* replacement len */ \
122 " .byte " alt_pad_len "\n" /* pad len */
954e482b 123
4332195c
BP
124#define ALTINSTR_REPLACEMENT(newinstr, feature, num) /* replacement */ \
125 b_replacement(num)":\n\t" newinstr "\n" e_replacement(num) ":\n\t"
954e482b 126
edc953fa
MD
127/* alternative assembly primitive: */
128#define ALTERNATIVE(oldinstr, newinstr, feature) \
4332195c 129 OLDINSTR(oldinstr, 1) \
9cebed42 130 ".pushsection .altinstructions,\"a\"\n" \
954e482b 131 ALTINSTR_ENTRY(feature, 1) \
9cebed42 132 ".popsection\n" \
9cebed42 133 ".pushsection .altinstr_replacement, \"ax\"\n" \
954e482b 134 ALTINSTR_REPLACEMENT(newinstr, feature, 1) \
9cebed42 135 ".popsection"
954e482b
FY
136
137#define ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2)\
4332195c 138 OLDINSTR_2(oldinstr, 1, 2) \
9cebed42 139 ".pushsection .altinstructions,\"a\"\n" \
954e482b
FY
140 ALTINSTR_ENTRY(feature1, 1) \
141 ALTINSTR_ENTRY(feature2, 2) \
9cebed42 142 ".popsection\n" \
9cebed42 143 ".pushsection .altinstr_replacement, \"ax\"\n" \
954e482b
FY
144 ALTINSTR_REPLACEMENT(newinstr1, feature1, 1) \
145 ALTINSTR_REPLACEMENT(newinstr2, feature2, 2) \
9cebed42 146 ".popsection"
edc953fa 147
d61931d8
BP
148/*
149 * This must be included *after* the definition of ALTERNATIVE due to
150 * <asm/arch_hweight.h>
151 */
152#include <asm/cpufeature.h>
153
6b592570
PA
154/*
155 * Alternative instructions for different CPU types or capabilities.
156 *
157 * This allows to use optimized instructions even on generic binary
158 * kernels.
159 *
160 * length of oldinstr must be longer or equal the length of newinstr
161 * It can be padded with nops as needed.
162 *
163 * For non barrier like inlines please define new variants
164 * without volatile and memory clobber.
165 */
166#define alternative(oldinstr, newinstr, feature) \
edc953fa 167 asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) : : : "memory")
6b592570 168
4332195c
BP
169#define alternative_2(oldinstr, newinstr1, feature1, newinstr2, feature2) \
170 asm volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2) ::: "memory")
171
6b592570
PA
172/*
173 * Alternative inline assembly with input.
174 *
175 * Pecularities:
176 * No memory clobber here.
177 * Argument numbers start with 1.
178 * Best is to use constraints that are fixed size (like (%1) ... "r")
179 * If you use variable sized constraints like "m" or "g" in the
180 * replacement make sure to pad to the worst case length.
edc953fa 181 * Leaving an unused argument 0 to keep API compatibility.
6b592570
PA
182 */
183#define alternative_input(oldinstr, newinstr, feature, input...) \
edc953fa
MD
184 asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) \
185 : : "i" (0), ## input)
6b592570 186
5b3e83f4
FY
187/*
188 * This is similar to alternative_input. But it has two features and
189 * respective instructions.
190 *
191 * If CPU has feature2, newinstr2 is used.
192 * Otherwise, if CPU has feature1, newinstr1 is used.
193 * Otherwise, oldinstr is used.
194 */
195#define alternative_input_2(oldinstr, newinstr1, feature1, newinstr2, \
196 feature2, input...) \
197 asm volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1, \
198 newinstr2, feature2) \
199 : : "i" (0), ## input)
200
6b592570
PA
201/* Like alternative_input, but with a single output argument */
202#define alternative_io(oldinstr, newinstr, feature, output, input...) \
edc953fa
MD
203 asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) \
204 : output : "i" (0), ## input)
6b592570 205
1b1d9258
JB
206/* Like alternative_io, but for replacing a direct call with another one. */
207#define alternative_call(oldfunc, newfunc, feature, output, input...) \
208 asm volatile (ALTERNATIVE("call %P[old]", "call %P[new]", feature) \
209 : output : [old] "i" (oldfunc), [new] "i" (newfunc), ## input)
210
954e482b
FY
211/*
212 * Like alternative_call, but there are two features and respective functions.
213 * If CPU has feature2, function2 is used.
214 * Otherwise, if CPU has feature1, function1 is used.
215 * Otherwise, old function is used.
216 */
217#define alternative_call_2(oldfunc, newfunc1, feature1, newfunc2, feature2, \
218 output, input...) \
219 asm volatile (ALTERNATIVE_2("call %P[old]", "call %P[new1]", feature1,\
220 "call %P[new2]", feature2) \
221 : output : [old] "i" (oldfunc), [new1] "i" (newfunc1), \
222 [new2] "i" (newfunc2), ## input)
223
6b592570
PA
224/*
225 * use this macro(s) if you need more than one output parameter
226 * in alternative_io
227 */
1b1d9258 228#define ASM_OUTPUT2(a...) a
6b592570 229
819165fb
JB
230/*
231 * use this macro if you need clobbers but no inputs in
232 * alternative_{input,io,call}()
233 */
234#define ASM_NO_INPUT_CLOBBER(clbr...) "i" (0) : clbr
235
6b592570
PA
236struct paravirt_patch_site;
237#ifdef CONFIG_PARAVIRT
238void apply_paravirt(struct paravirt_patch_site *start,
239 struct paravirt_patch_site *end);
96a388de 240#else
2ac1ea7c
JP
241static inline void apply_paravirt(struct paravirt_patch_site *start,
242 struct paravirt_patch_site *end)
6b592570
PA
243{}
244#define __parainstructions NULL
245#define __parainstructions_end NULL
96a388de 246#endif
6b592570 247
fa6f2cc7
JB
248extern void *text_poke_early(void *addr, const void *opcode, size_t len);
249
e587cadd
MD
250/*
251 * Clear and restore the kernel write-protection flag on the local CPU.
252 * Allows the kernel to edit read-only pages.
253 * Side-effect: any interrupt handler running between save and restore will have
254 * the ability to write to read-only pages.
255 *
256 * Warning:
257 * Code patching in the UP case is safe if NMIs and MCE handlers are stopped and
258 * no thread can be preempted in the instructions being modified (no iret to an
259 * invalid instruction possible) or if the instructions are changed from a
260 * consistent state to another consistent state atomically.
e587cadd
MD
261 * On the local CPU you need to be protected again NMI or MCE handlers seeing an
262 * inconsistent instruction while you patch.
e587cadd 263 */
e587cadd 264extern void *text_poke(void *addr, const void *opcode, size_t len);
17f41571 265extern int poke_int3_handler(struct pt_regs *regs);
fd4363ff 266extern void *text_poke_bp(void *addr, const void *opcode, size_t len, void *handler);
6b592570 267
1965aae3 268#endif /* _ASM_X86_ALTERNATIVE_H */