arch/arm64/include/asm/alternative.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef __ASM_ALTERNATIVE_H
   3 #define __ASM_ALTERNATIVE_H
   4
   5 #include <asm/cpucaps.h>
   6 #include <asm/insn.h>
   7
   8 #ifndef __ASSEMBLY__
   9
  10 #include <linux/init.h>
  11 #include <linux/types.h>
  12 #include <linux/stddef.h>
  13 #include <linux/stringify.h>
  14
  15 extern int alternatives_applied;
  16
  17 struct alt_instr {
  18         s32 orig_offset;        /* offset to original instruction */
  19         s32 alt_offset;         /* offset to replacement instruction */
  20         u16 cpufeature;         /* cpufeature bit set for replacement */
  21         u8  orig_len;           /* size of original instruction(s) */
  22         u8  alt_len;            /* size of new instruction(s), <= orig_len */
  23 };
  24
  25 void __init apply_alternatives_all(void);
  26 void apply_alternatives(void *start, size_t length);
  27
  28 #define ALTINSTR_ENTRY(feature)                                               \
  29         " .word 661b - .\n"                             /* label           */ \
  30         " .word 663f - .\n"                             /* new instruction */ \
  31         " .hword " __stringify(feature) "\n"            /* feature bit     */ \
  32         " .byte 662b-661b\n"                            /* source len      */ \
  33         " .byte 664f-663f\n"                            /* replacement len */
  34
  35 /*
  36  * alternative assembly primitive:
  37  *
  38  * If any of these .org directive fail, it means that insn1 and insn2
  39  * don't have the same length. This used to be written as
  40  *
  41  * .if ((664b-663b) != (662b-661b))
  42  *      .error "Alternatives instruction length mismatch"
  43  * .endif
  44  *
  45  * but most assemblers die if insn1 or insn2 have a .inst. This should
  46  * be fixed in a binutils release posterior to 2.25.51.0.2 (anything
  47  * containing commit 4e4d08cf7399b606 or c1baaddf8861).
  48  */
  49 #define __ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg_enabled)     \
  50         ".if "__stringify(cfg_enabled)" == 1\n"                         \
  51         "661:\n\t"                                                      \
  52         oldinstr "\n"                                                   \
  53         "662:\n"                                                        \
  54         ".pushsection .altinstructions,\"a\"\n"                         \
  55         ALTINSTR_ENTRY(feature)                                         \
  56         ".popsection\n"                                                 \
  57         ".pushsection .altinstr_replacement, \"a\"\n"                   \
  58         "663:\n\t"                                                      \
  59         newinstr "\n"                                                   \
  60         "664:\n\t"                                                      \
  61         ".popsection\n\t"                                               \
  62         ".org   . - (664b-663b) + (662b-661b)\n\t"                      \
  63         ".org   . - (662b-661b) + (664b-663b)\n"                        \
  64         ".endif\n"
  65
  66 #define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...) \
  67         __ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg))
  68
  69 #else
  70
  71 #include <asm/assembler.h>
  72
  73 .macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len
  74         .word \orig_offset - .
  75         .word \alt_offset - .
  76         .hword \feature
  77         .byte \orig_len
  78         .byte \alt_len
  79 .endm
  80
  81 .macro alternative_insn insn1, insn2, cap, enable = 1
  82         .if \enable
  83 661:    \insn1
  84 662:    .pushsection .altinstructions, "a"
  85         altinstruction_entry 661b, 663f, \cap, 662b-661b, 664f-663f
  86         .popsection
  87         .pushsection .altinstr_replacement, "ax"
  88 663:    \insn2
  89 664:    .popsection
  90         .org    . - (664b-663b) + (662b-661b)
  91         .org    . - (662b-661b) + (664b-663b)
  92         .endif
  93 .endm
  94
  95 /*
  96  * Alternative sequences
  97  *
  98  * The code for the case where the capability is not present will be
  99  * assembled and linked as normal. There are no restrictions on this
 100  * code.
 101  *
 102  * The code for the case where the capability is present will be
 103  * assembled into a special section to be used for dynamic patching.
 104  * Code for that case must:
 105  *
 106  * 1. Be exactly the same length (in bytes) as the default code
 107  *    sequence.
 108  *
 109  * 2. Not contain a branch target that is used outside of the
 110  *    alternative sequence it is defined in (branches into an
 111  *    alternative sequence are not fixed up).
 112  */
 113
 114 /*
 115  * Begin an alternative code sequence.
 116  */
 117 .macro alternative_if_not cap
 118         .set .Lasm_alt_mode, 0
 119         .pushsection .altinstructions, "a"
 120         altinstruction_entry 661f, 663f, \cap, 662f-661f, 664f-663f
 121         .popsection
 122 661:
 123 .endm
 124
 125 .macro alternative_if cap
 126         .set .Lasm_alt_mode, 1
 127         .pushsection .altinstructions, "a"
 128         altinstruction_entry 663f, 661f, \cap, 664f-663f, 662f-661f
 129         .popsection
 130         .pushsection .altinstr_replacement, "ax"
 131         .align 2        /* So GAS knows label 661 is suitably aligned */
 132 661:
 133 .endm
 134
 135 /*
 136  * Provide the other half of the alternative code sequence.
 137  */
 138 .macro alternative_else
 139 662:
 140         .if .Lasm_alt_mode==0
 141         .pushsection .altinstr_replacement, "ax"
 142         .else
 143         .popsection
 144         .endif
 145 663:
 146 .endm
 147
 148 /*
 149  * Complete an alternative code sequence.
 150  */
 151 .macro alternative_endif
 152 664:
 153         .if .Lasm_alt_mode==0
 154         .popsection
 155         .endif
 156         .org    . - (664b-663b) + (662b-661b)
 157         .org    . - (662b-661b) + (664b-663b)
 158 .endm
 159
 160 /*
 161  * Provides a trivial alternative or default sequence consisting solely
 162  * of NOPs. The number of NOPs is chosen automatically to match the
 163  * previous case.
 164  */
 165 .macro alternative_else_nop_endif
 166 alternative_else
 167         nops    (662b-661b) / AARCH64_INSN_SIZE
 168 alternative_endif
 169 .endm
 170
 171 #define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...)   \
 172         alternative_insn insn1, insn2, cap, IS_ENABLED(cfg)
 173
 174 .macro user_alt, label, oldinstr, newinstr, cond
 175 9999:   alternative_insn "\oldinstr", "\newinstr", \cond
 176         _ASM_EXTABLE 9999b, \label
 177 .endm
 178
 179 /*
 180  * Generate the assembly for UAO alternatives with exception table entries.
 181  * This is complicated as there is no post-increment or pair versions of the
 182  * unprivileged instructions, and USER() only works for single instructions.
 183  */
 184 #ifdef CONFIG_ARM64_UAO
 185         .macro uao_ldp l, reg1, reg2, addr, post_inc
 186                 alternative_if_not ARM64_HAS_UAO
 187 8888:                   ldp     \reg1, \reg2, [\addr], \post_inc;
 188 8889:                   nop;
 189                         nop;
 190                 alternative_else
 191                         ldtr    \reg1, [\addr];
 192                         ldtr    \reg2, [\addr, #8];
 193                         add     \addr, \addr, \post_inc;
 194                 alternative_endif
 195
 196                 _asm_extable    8888b,\l;
 197                 _asm_extable    8889b,\l;
 198         .endm
 199
 200         .macro uao_stp l, reg1, reg2, addr, post_inc
 201                 alternative_if_not ARM64_HAS_UAO
 202 8888:                   stp     \reg1, \reg2, [\addr], \post_inc;
 203 8889:                   nop;
 204                         nop;
 205                 alternative_else
 206                         sttr    \reg1, [\addr];
 207                         sttr    \reg2, [\addr, #8];
 208                         add     \addr, \addr, \post_inc;
 209                 alternative_endif
 210
 211                 _asm_extable    8888b,\l;
 212                 _asm_extable    8889b,\l;
 213         .endm
 214
 215         .macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc
 216                 alternative_if_not ARM64_HAS_UAO
 217 8888:                   \inst   \reg, [\addr], \post_inc;
 218                         nop;
 219                 alternative_else
 220                         \alt_inst       \reg, [\addr];
 221                         add             \addr, \addr, \post_inc;
 222                 alternative_endif
 223
 224                 _asm_extable    8888b,\l;
 225         .endm
 226 #else
 227         .macro uao_ldp l, reg1, reg2, addr, post_inc
 228                 USER(\l, ldp \reg1, \reg2, [\addr], \post_inc)
 229         .endm
 230         .macro uao_stp l, reg1, reg2, addr, post_inc
 231                 USER(\l, stp \reg1, \reg2, [\addr], \post_inc)
 232         .endm
 233         .macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc
 234                 USER(\l, \inst \reg, [\addr], \post_inc)
 235         .endm
 236 #endif
 237
 238 #endif  /*  __ASSEMBLY__  */
 239
 240 /*
 241  * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature));
 242  *
 243  * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature, CONFIG_FOO));
 244  * N.B. If CONFIG_FOO is specified, but not selected, the whole block
 245  *      will be omitted, including oldinstr.
 246  */
 247 #define ALTERNATIVE(oldinstr, newinstr, ...)   \
 248         _ALTERNATIVE_CFG(oldinstr, newinstr, __VA_ARGS__, 1)
 249
 250 #endif /* __ASM_ALTERNATIVE_H */