]>
Commit | Line | Data |
---|---|---|
1ccea77e | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
439e7271 JL |
2 | /* |
3 | * shadow.c - Shadow Variables | |
4 | * | |
5 | * Copyright (C) 2014 Josh Poimboeuf <jpoimboe@redhat.com> | |
6 | * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com> | |
7 | * Copyright (C) 2017 Joe Lawrence <joe.lawrence@redhat.com> | |
439e7271 JL |
8 | */ |
9 | ||
10 | /** | |
11 | * DOC: Shadow variable API concurrency notes: | |
12 | * | |
13 | * The shadow variable API provides a simple relationship between an | |
14 | * <obj, id> pair and a pointer value. It is the responsibility of the | |
15 | * caller to provide any mutual exclusion required of the shadow data. | |
16 | * | |
17 | * Once a shadow variable is attached to its parent object via the | |
18 | * klp_shadow_*alloc() API calls, it is considered live: any subsequent | |
19 | * call to klp_shadow_get() may then return the shadow variable's data | |
20 | * pointer. Callers of klp_shadow_*alloc() should prepare shadow data | |
21 | * accordingly. | |
22 | * | |
23 | * The klp_shadow_*alloc() API calls may allocate memory for new shadow | |
24 | * variable structures. Their implementation does not call kmalloc | |
25 | * inside any spinlocks, but API callers should pass GFP flags according | |
26 | * to their specific needs. | |
27 | * | |
28 | * The klp_shadow_hash is an RCU-enabled hashtable and is safe against | |
29 | * concurrent klp_shadow_free() and klp_shadow_get() operations. | |
30 | */ | |
31 | ||
32 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
33 | ||
34 | #include <linux/hashtable.h> | |
35 | #include <linux/slab.h> | |
36 | #include <linux/livepatch.h> | |
37 | ||
38 | static DEFINE_HASHTABLE(klp_shadow_hash, 12); | |
39 | ||
40 | /* | |
41 | * klp_shadow_lock provides exclusive access to the klp_shadow_hash and | |
42 | * the shadow variables it references. | |
43 | */ | |
44 | static DEFINE_SPINLOCK(klp_shadow_lock); | |
45 | ||
46 | /** | |
47 | * struct klp_shadow - shadow variable structure | |
48 | * @node: klp_shadow_hash hash table node | |
49 | * @rcu_head: RCU is used to safely free this structure | |
50 | * @obj: pointer to parent object | |
51 | * @id: data identifier | |
52 | * @data: data area | |
53 | */ | |
54 | struct klp_shadow { | |
55 | struct hlist_node node; | |
56 | struct rcu_head rcu_head; | |
57 | void *obj; | |
58 | unsigned long id; | |
59 | char data[]; | |
60 | }; | |
61 | ||
62 | /** | |
63 | * klp_shadow_match() - verify a shadow variable matches given <obj, id> | |
64 | * @shadow: shadow variable to match | |
65 | * @obj: pointer to parent object | |
66 | * @id: data identifier | |
67 | * | |
68 | * Return: true if the shadow variable matches. | |
69 | */ | |
70 | static inline bool klp_shadow_match(struct klp_shadow *shadow, void *obj, | |
71 | unsigned long id) | |
72 | { | |
73 | return shadow->obj == obj && shadow->id == id; | |
74 | } | |
75 | ||
76 | /** | |
77 | * klp_shadow_get() - retrieve a shadow variable data pointer | |
78 | * @obj: pointer to parent object | |
79 | * @id: data identifier | |
80 | * | |
81 | * Return: the shadow variable data element, NULL on failure. | |
82 | */ | |
83 | void *klp_shadow_get(void *obj, unsigned long id) | |
84 | { | |
85 | struct klp_shadow *shadow; | |
86 | ||
87 | rcu_read_lock(); | |
88 | ||
89 | hash_for_each_possible_rcu(klp_shadow_hash, shadow, node, | |
90 | (unsigned long)obj) { | |
91 | ||
92 | if (klp_shadow_match(shadow, obj, id)) { | |
93 | rcu_read_unlock(); | |
94 | return shadow->data; | |
95 | } | |
96 | } | |
97 | ||
98 | rcu_read_unlock(); | |
99 | ||
100 | return NULL; | |
101 | } | |
102 | EXPORT_SYMBOL_GPL(klp_shadow_get); | |
103 | ||
e91c2518 PM |
104 | static void *__klp_shadow_get_or_alloc(void *obj, unsigned long id, |
105 | size_t size, gfp_t gfp_flags, | |
106 | klp_shadow_ctor_t ctor, void *ctor_data, | |
107 | bool warn_on_exist) | |
439e7271 JL |
108 | { |
109 | struct klp_shadow *new_shadow; | |
110 | void *shadow_data; | |
111 | unsigned long flags; | |
112 | ||
113 | /* Check if the shadow variable already exists */ | |
114 | shadow_data = klp_shadow_get(obj, id); | |
115 | if (shadow_data) | |
116 | goto exists; | |
117 | ||
e91c2518 PM |
118 | /* |
119 | * Allocate a new shadow variable. Fill it with zeroes by default. | |
120 | * More complex setting can be done by @ctor function. But it is | |
121 | * called only when the buffer is really used (under klp_shadow_lock). | |
122 | */ | |
439e7271 JL |
123 | new_shadow = kzalloc(size + sizeof(*new_shadow), gfp_flags); |
124 | if (!new_shadow) | |
125 | return NULL; | |
126 | ||
439e7271 JL |
127 | /* Look for <obj, id> again under the lock */ |
128 | spin_lock_irqsave(&klp_shadow_lock, flags); | |
129 | shadow_data = klp_shadow_get(obj, id); | |
130 | if (unlikely(shadow_data)) { | |
131 | /* | |
132 | * Shadow variable was found, throw away speculative | |
133 | * allocation. | |
134 | */ | |
135 | spin_unlock_irqrestore(&klp_shadow_lock, flags); | |
136 | kfree(new_shadow); | |
137 | goto exists; | |
138 | } | |
139 | ||
e91c2518 PM |
140 | new_shadow->obj = obj; |
141 | new_shadow->id = id; | |
142 | ||
143 | if (ctor) { | |
144 | int err; | |
145 | ||
146 | err = ctor(obj, new_shadow->data, ctor_data); | |
147 | if (err) { | |
148 | spin_unlock_irqrestore(&klp_shadow_lock, flags); | |
149 | kfree(new_shadow); | |
150 | pr_err("Failed to construct shadow variable <%p, %lx> (%d)\n", | |
151 | obj, id, err); | |
152 | return NULL; | |
153 | } | |
154 | } | |
155 | ||
439e7271 JL |
156 | /* No <obj, id> found, so attach the newly allocated one */ |
157 | hash_add_rcu(klp_shadow_hash, &new_shadow->node, | |
158 | (unsigned long)new_shadow->obj); | |
159 | spin_unlock_irqrestore(&klp_shadow_lock, flags); | |
160 | ||
161 | return new_shadow->data; | |
162 | ||
163 | exists: | |
164 | if (warn_on_exist) { | |
165 | WARN(1, "Duplicate shadow variable <%p, %lx>\n", obj, id); | |
166 | return NULL; | |
167 | } | |
168 | ||
169 | return shadow_data; | |
170 | } | |
171 | ||
172 | /** | |
173 | * klp_shadow_alloc() - allocate and add a new shadow variable | |
174 | * @obj: pointer to parent object | |
175 | * @id: data identifier | |
439e7271 JL |
176 | * @size: size of attached data |
177 | * @gfp_flags: GFP mask for allocation | |
e91c2518 PM |
178 | * @ctor: custom constructor to initialize the shadow data (optional) |
179 | * @ctor_data: pointer to any data needed by @ctor (optional) | |
180 | * | |
181 | * Allocates @size bytes for new shadow variable data using @gfp_flags. | |
182 | * The data are zeroed by default. They are further initialized by @ctor | |
183 | * function if it is not NULL. The new shadow variable is then added | |
184 | * to the global hashtable. | |
439e7271 | 185 | * |
e91c2518 PM |
186 | * If an existing <obj, id> shadow variable can be found, this routine will |
187 | * issue a WARN, exit early and return NULL. | |
439e7271 | 188 | * |
e91c2518 PM |
189 | * This function guarantees that the constructor function is called only when |
190 | * the variable did not exist before. The cost is that @ctor is called | |
191 | * in atomic context under a spin lock. | |
439e7271 JL |
192 | * |
193 | * Return: the shadow variable data element, NULL on duplicate or | |
194 | * failure. | |
195 | */ | |
e91c2518 PM |
196 | void *klp_shadow_alloc(void *obj, unsigned long id, |
197 | size_t size, gfp_t gfp_flags, | |
198 | klp_shadow_ctor_t ctor, void *ctor_data) | |
439e7271 | 199 | { |
e91c2518 PM |
200 | return __klp_shadow_get_or_alloc(obj, id, size, gfp_flags, |
201 | ctor, ctor_data, true); | |
439e7271 JL |
202 | } |
203 | EXPORT_SYMBOL_GPL(klp_shadow_alloc); | |
204 | ||
205 | /** | |
206 | * klp_shadow_get_or_alloc() - get existing or allocate a new shadow variable | |
207 | * @obj: pointer to parent object | |
208 | * @id: data identifier | |
439e7271 JL |
209 | * @size: size of attached data |
210 | * @gfp_flags: GFP mask for allocation | |
e91c2518 PM |
211 | * @ctor: custom constructor to initialize the shadow data (optional) |
212 | * @ctor_data: pointer to any data needed by @ctor (optional) | |
439e7271 JL |
213 | * |
214 | * Returns a pointer to existing shadow data if an <obj, id> shadow | |
215 | * variable is already present. Otherwise, it creates a new shadow | |
216 | * variable like klp_shadow_alloc(). | |
217 | * | |
e91c2518 PM |
218 | * This function guarantees that only one shadow variable exists with the given |
219 | * @id for the given @obj. It also guarantees that the constructor function | |
220 | * will be called only when the variable did not exist before. The cost is | |
221 | * that @ctor is called in atomic context under a spin lock. | |
439e7271 JL |
222 | * |
223 | * Return: the shadow variable data element, NULL on failure. | |
224 | */ | |
e91c2518 PM |
225 | void *klp_shadow_get_or_alloc(void *obj, unsigned long id, |
226 | size_t size, gfp_t gfp_flags, | |
227 | klp_shadow_ctor_t ctor, void *ctor_data) | |
439e7271 | 228 | { |
e91c2518 PM |
229 | return __klp_shadow_get_or_alloc(obj, id, size, gfp_flags, |
230 | ctor, ctor_data, false); | |
439e7271 JL |
231 | } |
232 | EXPORT_SYMBOL_GPL(klp_shadow_get_or_alloc); | |
233 | ||
3b2c77d0 PM |
234 | static void klp_shadow_free_struct(struct klp_shadow *shadow, |
235 | klp_shadow_dtor_t dtor) | |
236 | { | |
237 | hash_del_rcu(&shadow->node); | |
238 | if (dtor) | |
239 | dtor(shadow->obj, shadow->data); | |
240 | kfree_rcu(shadow, rcu_head); | |
241 | } | |
242 | ||
439e7271 JL |
243 | /** |
244 | * klp_shadow_free() - detach and free a <obj, id> shadow variable | |
245 | * @obj: pointer to parent object | |
246 | * @id: data identifier | |
3b2c77d0 PM |
247 | * @dtor: custom callback that can be used to unregister the variable |
248 | * and/or free data that the shadow variable points to (optional) | |
439e7271 JL |
249 | * |
250 | * This function releases the memory for this <obj, id> shadow variable | |
251 | * instance, callers should stop referencing it accordingly. | |
252 | */ | |
3b2c77d0 | 253 | void klp_shadow_free(void *obj, unsigned long id, klp_shadow_dtor_t dtor) |
439e7271 JL |
254 | { |
255 | struct klp_shadow *shadow; | |
256 | unsigned long flags; | |
257 | ||
258 | spin_lock_irqsave(&klp_shadow_lock, flags); | |
259 | ||
260 | /* Delete <obj, id> from hash */ | |
261 | hash_for_each_possible(klp_shadow_hash, shadow, node, | |
262 | (unsigned long)obj) { | |
263 | ||
264 | if (klp_shadow_match(shadow, obj, id)) { | |
3b2c77d0 | 265 | klp_shadow_free_struct(shadow, dtor); |
439e7271 JL |
266 | break; |
267 | } | |
268 | } | |
269 | ||
270 | spin_unlock_irqrestore(&klp_shadow_lock, flags); | |
271 | } | |
272 | EXPORT_SYMBOL_GPL(klp_shadow_free); | |
273 | ||
274 | /** | |
275 | * klp_shadow_free_all() - detach and free all <*, id> shadow variables | |
276 | * @id: data identifier | |
3b2c77d0 PM |
277 | * @dtor: custom callback that can be used to unregister the variable |
278 | * and/or free data that the shadow variable points to (optional) | |
439e7271 JL |
279 | * |
280 | * This function releases the memory for all <*, id> shadow variable | |
281 | * instances, callers should stop referencing them accordingly. | |
282 | */ | |
3b2c77d0 | 283 | void klp_shadow_free_all(unsigned long id, klp_shadow_dtor_t dtor) |
439e7271 JL |
284 | { |
285 | struct klp_shadow *shadow; | |
286 | unsigned long flags; | |
287 | int i; | |
288 | ||
289 | spin_lock_irqsave(&klp_shadow_lock, flags); | |
290 | ||
291 | /* Delete all <*, id> from hash */ | |
292 | hash_for_each(klp_shadow_hash, i, shadow, node) { | |
3b2c77d0 PM |
293 | if (klp_shadow_match(shadow, shadow->obj, id)) |
294 | klp_shadow_free_struct(shadow, dtor); | |
439e7271 JL |
295 | } |
296 | ||
297 | spin_unlock_irqrestore(&klp_shadow_lock, flags); | |
298 | } | |
299 | EXPORT_SYMBOL_GPL(klp_shadow_free_all); |