projects / mirror_ubuntu-bionic-kernel.git / blame
| Commit | Line | Data |
|---|---|---|
| b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
| d9f5ab7b JB |
2 | #ifndef _ASM_X86_JUMP_LABEL_H |
| 3 | #define _ASM_X86_JUMP_LABEL_H | |
| 4 | ||
| c2845433 AL |
5 | #ifndef HAVE_JUMP_LABEL |
| 6 | /* | |
| 7 | * For better or for worse, if jump labels (the gcc extension) are missing, | |
| 8 | * then the entire static branch patching infrastructure is compiled out. | |
| 9 | * If that happens, the code in here will malfunction. Raise a compiler | |
| 10 | * error instead. | |
| 11 | * | |
| 12 | * In theory, jump labels and the static branch patching infrastructure | |
| 13 | * could be decoupled to fix this. | |
| 14 | */ | |
| 15 | #error asm/jump_label.h included on a non-jump-label kernel | |
| 16 | #endif | |
| 17 | ||
| d9f5ab7b JB |
18 | #define JUMP_LABEL_NOP_SIZE 5 |
| 19 | ||
| c3c7f14a SR |
20 | #ifdef CONFIG_X86_64 |
| 21 | # define STATIC_KEY_INIT_NOP P6_NOP5_ATOMIC | |
| 22 | #else | |
| 23 | # define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC | |
| 24 | #endif | |
| d430d3d7 | 25 | |
| 2671c3e4 AL |
26 | #include <asm/asm.h> |
| 27 | #include <asm/nops.h> | |
| 28 | ||
| 29 | #ifndef __ASSEMBLY__ | |
| 30 | ||
| 31 | #include <linux/stringify.h> | |
| 32 | #include <linux/types.h> | |
| 33 | ||
| 11276d53 | 34 | static __always_inline bool arch_static_branch(struct static_key *key, bool branch) |
| d430d3d7 | 35 | { |
| 3f0116c3 | 36 | asm_volatile_goto("1:" |
| c3c7f14a | 37 | ".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t" |
| d430d3d7 | 38 | ".pushsection __jump_table, \"aw\" \n\t" |
| ef647894 | 39 | _ASM_ALIGN "\n\t" |
| d420acd8 | 40 | _ASM_PTR "1b, %l[l_yes], %c0 + %c1 \n\t" |
| d430d3d7 | 41 | ".popsection \n\t" |
| d420acd8 | 42 | : : "i" (key), "i" (branch) : : l_yes); |
| 11276d53 PZ |
43 | |
| 44 | return false; | |
| 45 | l_yes: | |
| 46 | return true; | |
| 47 | } | |
| 48 | ||
| 49 | static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) | |
| 50 | { | |
| 51 | asm_volatile_goto("1:" | |
| 52 | ".byte 0xe9\n\t .long %l[l_yes] - 2f\n\t" | |
| 53 | "2:\n\t" | |
| 54 | ".pushsection __jump_table, \"aw\" \n\t" | |
| 55 | _ASM_ALIGN "\n\t" | |
| d420acd8 | 56 | _ASM_PTR "1b, %l[l_yes], %c0 + %c1 \n\t" |
| 11276d53 | 57 | ".popsection \n\t" |
| d420acd8 | 58 | : : "i" (key), "i" (branch) : : l_yes); |
| 11276d53 | 59 | |
| d430d3d7 JB |
60 | return false; |
| 61 | l_yes: | |
| 62 | return true; | |
| 63 | } | |
| d9f5ab7b | 64 | |
| d9f5ab7b | 65 | #ifdef CONFIG_X86_64 |
| d9f5ab7b | 66 | typedef u64 jump_label_t; |
| d9f5ab7b | 67 | #else |
| d9f5ab7b | 68 | typedef u32 jump_label_t; |
| 95fccd46 | 69 | #endif |
| d9f5ab7b JB |
70 | |
| 71 | struct jump_entry { | |
| 72 | jump_label_t code; | |
| 73 | jump_label_t target; | |
| 74 | jump_label_t key; | |
| 75 | }; | |
| 76 | ||
| 2671c3e4 AL |
77 | #else /* __ASSEMBLY__ */ |
| 78 | ||
| 79 | .macro STATIC_JUMP_IF_TRUE target, key, def | |
| 80 | .Lstatic_jump_\@: | |
| 81 | .if \def | |
| 82 | /* Equivalent to "jmp.d32 \target" */ | |
| 83 | .byte 0xe9 | |
| 84 | .long \target - .Lstatic_jump_after_\@ | |
| 85 | .Lstatic_jump_after_\@: | |
| 86 | .else | |
| 87 | .byte STATIC_KEY_INIT_NOP | |
| 88 | .endif | |
| 89 | .pushsection __jump_table, "aw" | |
| 90 | _ASM_ALIGN | |
| 91 | _ASM_PTR .Lstatic_jump_\@, \target, \key | |
| 92 | .popsection | |
| 93 | .endm | |
| 94 | ||
| 95 | .macro STATIC_JUMP_IF_FALSE target, key, def | |
| 96 | .Lstatic_jump_\@: | |
| 97 | .if \def | |
| 98 | .byte STATIC_KEY_INIT_NOP | |
| 99 | .else | |
| 100 | /* Equivalent to "jmp.d32 \target" */ | |
| 101 | .byte 0xe9 | |
| 102 | .long \target - .Lstatic_jump_after_\@ | |
| 103 | .Lstatic_jump_after_\@: | |
| 104 | .endif | |
| 105 | .pushsection __jump_table, "aw" | |
| 106 | _ASM_ALIGN | |
| 107 | _ASM_PTR .Lstatic_jump_\@, \target, \key + 1 | |
| 108 | .popsection | |
| 109 | .endm | |
| 110 | ||
| 111 | #endif /* __ASSEMBLY__ */ | |
| 112 | ||
| d9f5ab7b | 113 | #endif |