include/linux/static_call.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _LINUX_STATIC_CALL_H
   3 #define _LINUX_STATIC_CALL_H
   4
   5 /*
   6  * Static call support
   7  *
   8  * Static calls use code patching to hard-code function pointers into direct
   9  * branch instructions. They give the flexibility of function pointers, but
  10  * with improved performance. This is especially important for cases where
  11  * retpolines would otherwise be used, as retpolines can significantly impact
  12  * performance.
  13  *
  14  *
  15  * API overview:
  16  *
  17  *   DECLARE_STATIC_CALL(name, func);
  18  *   DEFINE_STATIC_CALL(name, func);
  19  *   static_call(name)(args...);
  20  *   static_call_update(name, func);
  21  *
  22  * Usage example:
  23  *
  24  *   # Start with the following functions (with identical prototypes):
  25  *   int func_a(int arg1, int arg2);
  26  *   int func_b(int arg1, int arg2);
  27  *
  28  *   # Define a 'my_name' reference, associated with func_a() by default
  29  *   DEFINE_STATIC_CALL(my_name, func_a);
  30  *
  31  *   # Call func_a()
  32  *   static_call(my_name)(arg1, arg2);
  33  *
  34  *   # Update 'my_name' to point to func_b()
  35  *   static_call_update(my_name, &func_b);
  36  *
  37  *   # Call func_b()
  38  *   static_call(my_name)(arg1, arg2);
  39  *
  40  *
  41  * Implementation details:
  42  *
  43  *   This requires some arch-specific code (CONFIG_HAVE_STATIC_CALL).
  44  *   Otherwise basic indirect calls are used (with function pointers).
  45  *
  46  *   Each static_call() site calls into a trampoline associated with the name.
  47  *   The trampoline has a direct branch to the default function.  Updates to a
  48  *   name will modify the trampoline's branch destination.
  49  *
  50  *   If the arch has CONFIG_HAVE_STATIC_CALL_INLINE, then the call sites
  51  *   themselves will be patched at runtime to call the functions directly,
  52  *   rather than calling through the trampoline.  This requires objtool or a
  53  *   compiler plugin to detect all the static_call() sites and annotate them
  54  *   in the .static_call_sites section.
  55  */
  56
  57 #include <linux/types.h>
  58 #include <linux/cpu.h>
  59 #include <linux/static_call_types.h>
  60
  61 #ifdef CONFIG_HAVE_STATIC_CALL
  62 #include <asm/static_call.h>
  63
  64 /*
  65  * Either @site or @tramp can be NULL.
  66  */
  67 extern void arch_static_call_transform(void *site, void *tramp, void *func);
  68
  69 #define STATIC_CALL_TRAMP_ADDR(name) &STATIC_CALL_TRAMP(name)
  70
  71 /*
  72  * __ADDRESSABLE() is used to ensure the key symbol doesn't get stripped from
  73  * the symbol table so that objtool can reference it when it generates the
  74  * .static_call_sites section.
  75  */
  76 #define __static_call(name)                                             \
  77 ({                                                                      \
  78         __ADDRESSABLE(STATIC_CALL_KEY(name));                           \
  79         &STATIC_CALL_TRAMP(name);                                       \
  80 })
  81
  82 #else
  83 #define STATIC_CALL_TRAMP_ADDR(name) NULL
  84 #endif
  85
  86
  87 #define DECLARE_STATIC_CALL(name, func)                                 \
  88         extern struct static_call_key STATIC_CALL_KEY(name);            \
  89         extern typeof(func) STATIC_CALL_TRAMP(name);
  90
  91 #define static_call_update(name, func)                                  \
  92 ({                                                                      \
  93         BUILD_BUG_ON(!__same_type(*(func), STATIC_CALL_TRAMP(name)));   \
  94         __static_call_update(&STATIC_CALL_KEY(name),                    \
  95                              STATIC_CALL_TRAMP_ADDR(name), func);       \
  96 })
  97
  98 #ifdef CONFIG_HAVE_STATIC_CALL_INLINE
  99
 100 struct static_call_mod {
 101         struct static_call_mod *next;
 102         struct module *mod; /* for vmlinux, mod == NULL */
 103         struct static_call_site *sites;
 104 };
 105
 106 struct static_call_key {
 107         void *func;
 108         struct static_call_mod *mods;
 109 };
 110
 111 extern void __static_call_update(struct static_call_key *key, void *tramp, void *func);
 112 extern int static_call_mod_init(struct module *mod);
 113
 114 #define DEFINE_STATIC_CALL(name, _func)                                 \
 115         DECLARE_STATIC_CALL(name, _func);                               \
 116         struct static_call_key STATIC_CALL_KEY(name) = {                \
 117                 .func = _func,                                          \
 118                 .mods = NULL,                                           \
 119         };                                                              \
 120         ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
 121
 122 #define static_call(name)       __static_call(name)
 123
 124 #define EXPORT_STATIC_CALL(name)                                        \
 125         EXPORT_SYMBOL(STATIC_CALL_KEY(name));                           \
 126         EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
 127
 128 #define EXPORT_STATIC_CALL_GPL(name)                                    \
 129         EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name));                       \
 130         EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
 131
 132 #elif defined(CONFIG_HAVE_STATIC_CALL)
 133
 134 struct static_call_key {
 135         void *func;
 136 };
 137
 138 #define DEFINE_STATIC_CALL(name, _func)                                 \
 139         DECLARE_STATIC_CALL(name, _func);                               \
 140         struct static_call_key STATIC_CALL_KEY(name) = {                \
 141                 .func = _func,                                          \
 142         };                                                              \
 143         ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
 144
 145 #define static_call(name)       __static_call(name)
 146
 147 static inline
 148 void __static_call_update(struct static_call_key *key, void *tramp, void *func)
 149 {
 150         cpus_read_lock();
 151         WRITE_ONCE(key->func, func);
 152         arch_static_call_transform(NULL, tramp, func);
 153         cpus_read_unlock();
 154 }
 155
 156 #define EXPORT_STATIC_CALL(name)                                        \
 157         EXPORT_SYMBOL(STATIC_CALL_KEY(name));                           \
 158         EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
 159
 160 #define EXPORT_STATIC_CALL_GPL(name)                                    \
 161         EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name));                       \
 162         EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
 163
 164 #else /* Generic implementation */
 165
 166 struct static_call_key {
 167         void *func;
 168 };
 169
 170 #define DEFINE_STATIC_CALL(name, _func)                                 \
 171         DECLARE_STATIC_CALL(name, _func);                               \
 172         struct static_call_key STATIC_CALL_KEY(name) = {                \
 173                 .func = _func,                                          \
 174         }
 175
 176 #define static_call(name)                                               \
 177         ((typeof(STATIC_CALL_TRAMP(name))*)(STATIC_CALL_KEY(name).func))
 178
 179 static inline
 180 void __static_call_update(struct static_call_key *key, void *tramp, void *func)
 181 {
 182         WRITE_ONCE(key->func, func);
 183 }
 184
 185 #define EXPORT_STATIC_CALL(name)        EXPORT_SYMBOL(STATIC_CALL_KEY(name))
 186 #define EXPORT_STATIC_CALL_GPL(name)    EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name))
 187
 188 #endif /* CONFIG_HAVE_STATIC_CALL */
 189
 190 #endif /* _LINUX_STATIC_CALL_H */