]>
Commit | Line | Data |
---|---|---|
1 | // SPDX-License-Identifier: GPL-2.0 | |
2 | /* | |
3 | * Test cases for SL[AOU]B/page initialization at alloc/free time. | |
4 | */ | |
5 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
6 | ||
7 | #include <linux/init.h> | |
8 | #include <linux/kernel.h> | |
9 | #include <linux/mm.h> | |
10 | #include <linux/module.h> | |
11 | #include <linux/slab.h> | |
12 | #include <linux/string.h> | |
13 | #include <linux/vmalloc.h> | |
14 | ||
15 | #define GARBAGE_INT (0x09A7BA9E) | |
16 | #define GARBAGE_BYTE (0x9E) | |
17 | ||
18 | #define REPORT_FAILURES_IN_FN() \ | |
19 | do { \ | |
20 | if (failures) \ | |
21 | pr_info("%s failed %d out of %d times\n", \ | |
22 | __func__, failures, num_tests); \ | |
23 | else \ | |
24 | pr_info("all %d tests in %s passed\n", \ | |
25 | num_tests, __func__); \ | |
26 | } while (0) | |
27 | ||
28 | /* Calculate the number of uninitialized bytes in the buffer. */ | |
29 | static int __init count_nonzero_bytes(void *ptr, size_t size) | |
30 | { | |
31 | int i, ret = 0; | |
32 | unsigned char *p = (unsigned char *)ptr; | |
33 | ||
34 | for (i = 0; i < size; i++) | |
35 | if (p[i]) | |
36 | ret++; | |
37 | return ret; | |
38 | } | |
39 | ||
40 | /* Fill a buffer with garbage, skipping |skip| first bytes. */ | |
41 | static void __init fill_with_garbage_skip(void *ptr, int size, size_t skip) | |
42 | { | |
43 | unsigned int *p = (unsigned int *)((char *)ptr + skip); | |
44 | int i = 0; | |
45 | ||
46 | WARN_ON(skip > size); | |
47 | size -= skip; | |
48 | ||
49 | while (size >= sizeof(*p)) { | |
50 | p[i] = GARBAGE_INT; | |
51 | i++; | |
52 | size -= sizeof(*p); | |
53 | } | |
54 | if (size) | |
55 | memset(&p[i], GARBAGE_BYTE, size); | |
56 | } | |
57 | ||
58 | static void __init fill_with_garbage(void *ptr, size_t size) | |
59 | { | |
60 | fill_with_garbage_skip(ptr, size, 0); | |
61 | } | |
62 | ||
63 | static int __init do_alloc_pages_order(int order, int *total_failures) | |
64 | { | |
65 | struct page *page; | |
66 | void *buf; | |
67 | size_t size = PAGE_SIZE << order; | |
68 | ||
69 | page = alloc_pages(GFP_KERNEL, order); | |
70 | buf = page_address(page); | |
71 | fill_with_garbage(buf, size); | |
72 | __free_pages(page, order); | |
73 | ||
74 | page = alloc_pages(GFP_KERNEL, order); | |
75 | buf = page_address(page); | |
76 | if (count_nonzero_bytes(buf, size)) | |
77 | (*total_failures)++; | |
78 | fill_with_garbage(buf, size); | |
79 | __free_pages(page, order); | |
80 | return 1; | |
81 | } | |
82 | ||
83 | /* Test the page allocator by calling alloc_pages with different orders. */ | |
84 | static int __init test_pages(int *total_failures) | |
85 | { | |
86 | int failures = 0, num_tests = 0; | |
87 | int i; | |
88 | ||
89 | for (i = 0; i < 10; i++) | |
90 | num_tests += do_alloc_pages_order(i, &failures); | |
91 | ||
92 | REPORT_FAILURES_IN_FN(); | |
93 | *total_failures += failures; | |
94 | return num_tests; | |
95 | } | |
96 | ||
97 | /* Test kmalloc() with given parameters. */ | |
98 | static int __init do_kmalloc_size(size_t size, int *total_failures) | |
99 | { | |
100 | void *buf; | |
101 | ||
102 | buf = kmalloc(size, GFP_KERNEL); | |
103 | fill_with_garbage(buf, size); | |
104 | kfree(buf); | |
105 | ||
106 | buf = kmalloc(size, GFP_KERNEL); | |
107 | if (count_nonzero_bytes(buf, size)) | |
108 | (*total_failures)++; | |
109 | fill_with_garbage(buf, size); | |
110 | kfree(buf); | |
111 | return 1; | |
112 | } | |
113 | ||
114 | /* Test vmalloc() with given parameters. */ | |
115 | static int __init do_vmalloc_size(size_t size, int *total_failures) | |
116 | { | |
117 | void *buf; | |
118 | ||
119 | buf = vmalloc(size); | |
120 | fill_with_garbage(buf, size); | |
121 | vfree(buf); | |
122 | ||
123 | buf = vmalloc(size); | |
124 | if (count_nonzero_bytes(buf, size)) | |
125 | (*total_failures)++; | |
126 | fill_with_garbage(buf, size); | |
127 | vfree(buf); | |
128 | return 1; | |
129 | } | |
130 | ||
131 | /* Test kmalloc()/vmalloc() by allocating objects of different sizes. */ | |
132 | static int __init test_kvmalloc(int *total_failures) | |
133 | { | |
134 | int failures = 0, num_tests = 0; | |
135 | int i, size; | |
136 | ||
137 | for (i = 0; i < 20; i++) { | |
138 | size = 1 << i; | |
139 | num_tests += do_kmalloc_size(size, &failures); | |
140 | num_tests += do_vmalloc_size(size, &failures); | |
141 | } | |
142 | ||
143 | REPORT_FAILURES_IN_FN(); | |
144 | *total_failures += failures; | |
145 | return num_tests; | |
146 | } | |
147 | ||
148 | #define CTOR_BYTES (sizeof(unsigned int)) | |
149 | #define CTOR_PATTERN (0x41414141) | |
150 | /* Initialize the first 4 bytes of the object. */ | |
151 | static void test_ctor(void *obj) | |
152 | { | |
153 | *(unsigned int *)obj = CTOR_PATTERN; | |
154 | } | |
155 | ||
156 | /* | |
157 | * Check the invariants for the buffer allocated from a slab cache. | |
158 | * If the cache has a test constructor, the first 4 bytes of the object must | |
159 | * always remain equal to CTOR_PATTERN. | |
160 | * If the cache isn't an RCU-typesafe one, or if the allocation is done with | |
161 | * __GFP_ZERO, then the object contents must be zeroed after allocation. | |
162 | * If the cache is an RCU-typesafe one, the object contents must never be | |
163 | * zeroed after the first use. This is checked by memcmp() in | |
164 | * do_kmem_cache_size(). | |
165 | */ | |
166 | static bool __init check_buf(void *buf, int size, bool want_ctor, | |
167 | bool want_rcu, bool want_zero) | |
168 | { | |
169 | int bytes; | |
170 | bool fail = false; | |
171 | ||
172 | bytes = count_nonzero_bytes(buf, size); | |
173 | WARN_ON(want_ctor && want_zero); | |
174 | if (want_zero) | |
175 | return bytes; | |
176 | if (want_ctor) { | |
177 | if (*(unsigned int *)buf != CTOR_PATTERN) | |
178 | fail = 1; | |
179 | } else { | |
180 | if (bytes) | |
181 | fail = !want_rcu; | |
182 | } | |
183 | return fail; | |
184 | } | |
185 | ||
186 | /* | |
187 | * Test kmem_cache with given parameters: | |
188 | * want_ctor - use a constructor; | |
189 | * want_rcu - use SLAB_TYPESAFE_BY_RCU; | |
190 | * want_zero - use __GFP_ZERO. | |
191 | */ | |
192 | static int __init do_kmem_cache_size(size_t size, bool want_ctor, | |
193 | bool want_rcu, bool want_zero, | |
194 | int *total_failures) | |
195 | { | |
196 | struct kmem_cache *c; | |
197 | int iter; | |
198 | bool fail = false; | |
199 | gfp_t alloc_mask = GFP_KERNEL | (want_zero ? __GFP_ZERO : 0); | |
200 | void *buf, *buf_copy; | |
201 | ||
202 | c = kmem_cache_create("test_cache", size, 1, | |
203 | want_rcu ? SLAB_TYPESAFE_BY_RCU : 0, | |
204 | want_ctor ? test_ctor : NULL); | |
205 | for (iter = 0; iter < 10; iter++) { | |
206 | buf = kmem_cache_alloc(c, alloc_mask); | |
207 | /* Check that buf is zeroed, if it must be. */ | |
208 | fail = check_buf(buf, size, want_ctor, want_rcu, want_zero); | |
209 | fill_with_garbage_skip(buf, size, want_ctor ? CTOR_BYTES : 0); | |
210 | ||
211 | if (!want_rcu) { | |
212 | kmem_cache_free(c, buf); | |
213 | continue; | |
214 | } | |
215 | ||
216 | /* | |
217 | * If this is an RCU cache, use a critical section to ensure we | |
218 | * can touch objects after they're freed. | |
219 | */ | |
220 | rcu_read_lock(); | |
221 | /* | |
222 | * Copy the buffer to check that it's not wiped on | |
223 | * free(). | |
224 | */ | |
225 | buf_copy = kmalloc(size, GFP_ATOMIC); | |
226 | if (buf_copy) | |
227 | memcpy(buf_copy, buf, size); | |
228 | ||
229 | kmem_cache_free(c, buf); | |
230 | /* | |
231 | * Check that |buf| is intact after kmem_cache_free(). | |
232 | * |want_zero| is false, because we wrote garbage to | |
233 | * the buffer already. | |
234 | */ | |
235 | fail |= check_buf(buf, size, want_ctor, want_rcu, | |
236 | false); | |
237 | if (buf_copy) { | |
238 | fail |= (bool)memcmp(buf, buf_copy, size); | |
239 | kfree(buf_copy); | |
240 | } | |
241 | rcu_read_unlock(); | |
242 | } | |
243 | kmem_cache_destroy(c); | |
244 | ||
245 | *total_failures += fail; | |
246 | return 1; | |
247 | } | |
248 | ||
249 | /* | |
250 | * Check that the data written to an RCU-allocated object survives | |
251 | * reallocation. | |
252 | */ | |
253 | static int __init do_kmem_cache_rcu_persistent(int size, int *total_failures) | |
254 | { | |
255 | struct kmem_cache *c; | |
256 | void *buf, *buf_contents, *saved_ptr; | |
257 | void **used_objects; | |
258 | int i, iter, maxiter = 1024; | |
259 | bool fail = false; | |
260 | ||
261 | c = kmem_cache_create("test_cache", size, size, SLAB_TYPESAFE_BY_RCU, | |
262 | NULL); | |
263 | buf = kmem_cache_alloc(c, GFP_KERNEL); | |
264 | saved_ptr = buf; | |
265 | fill_with_garbage(buf, size); | |
266 | buf_contents = kmalloc(size, GFP_KERNEL); | |
267 | if (!buf_contents) | |
268 | goto out; | |
269 | used_objects = kmalloc_array(maxiter, sizeof(void *), GFP_KERNEL); | |
270 | if (!used_objects) { | |
271 | kfree(buf_contents); | |
272 | goto out; | |
273 | } | |
274 | memcpy(buf_contents, buf, size); | |
275 | kmem_cache_free(c, buf); | |
276 | /* | |
277 | * Run for a fixed number of iterations. If we never hit saved_ptr, | |
278 | * assume the test passes. | |
279 | */ | |
280 | for (iter = 0; iter < maxiter; iter++) { | |
281 | buf = kmem_cache_alloc(c, GFP_KERNEL); | |
282 | used_objects[iter] = buf; | |
283 | if (buf == saved_ptr) { | |
284 | fail = memcmp(buf_contents, buf, size); | |
285 | for (i = 0; i <= iter; i++) | |
286 | kmem_cache_free(c, used_objects[i]); | |
287 | goto free_out; | |
288 | } | |
289 | } | |
290 | ||
291 | free_out: | |
292 | kmem_cache_destroy(c); | |
293 | kfree(buf_contents); | |
294 | kfree(used_objects); | |
295 | out: | |
296 | *total_failures += fail; | |
297 | return 1; | |
298 | } | |
299 | ||
300 | static int __init do_kmem_cache_size_bulk(int size, int *total_failures) | |
301 | { | |
302 | struct kmem_cache *c; | |
303 | int i, iter, maxiter = 1024; | |
304 | int num, bytes; | |
305 | bool fail = false; | |
306 | void *objects[10]; | |
307 | ||
308 | c = kmem_cache_create("test_cache", size, size, 0, NULL); | |
309 | for (iter = 0; (iter < maxiter) && !fail; iter++) { | |
310 | num = kmem_cache_alloc_bulk(c, GFP_KERNEL, ARRAY_SIZE(objects), | |
311 | objects); | |
312 | for (i = 0; i < num; i++) { | |
313 | bytes = count_nonzero_bytes(objects[i], size); | |
314 | if (bytes) | |
315 | fail = true; | |
316 | fill_with_garbage(objects[i], size); | |
317 | } | |
318 | ||
319 | if (num) | |
320 | kmem_cache_free_bulk(c, num, objects); | |
321 | } | |
322 | *total_failures += fail; | |
323 | return 1; | |
324 | } | |
325 | ||
326 | /* | |
327 | * Test kmem_cache allocation by creating caches of different sizes, with and | |
328 | * without constructors, with and without SLAB_TYPESAFE_BY_RCU. | |
329 | */ | |
330 | static int __init test_kmemcache(int *total_failures) | |
331 | { | |
332 | int failures = 0, num_tests = 0; | |
333 | int i, flags, size; | |
334 | bool ctor, rcu, zero; | |
335 | ||
336 | for (i = 0; i < 10; i++) { | |
337 | size = 8 << i; | |
338 | for (flags = 0; flags < 8; flags++) { | |
339 | ctor = flags & 1; | |
340 | rcu = flags & 2; | |
341 | zero = flags & 4; | |
342 | if (ctor & zero) | |
343 | continue; | |
344 | num_tests += do_kmem_cache_size(size, ctor, rcu, zero, | |
345 | &failures); | |
346 | } | |
347 | num_tests += do_kmem_cache_size_bulk(size, &failures); | |
348 | } | |
349 | REPORT_FAILURES_IN_FN(); | |
350 | *total_failures += failures; | |
351 | return num_tests; | |
352 | } | |
353 | ||
354 | /* Test the behavior of SLAB_TYPESAFE_BY_RCU caches of different sizes. */ | |
355 | static int __init test_rcu_persistent(int *total_failures) | |
356 | { | |
357 | int failures = 0, num_tests = 0; | |
358 | int i, size; | |
359 | ||
360 | for (i = 0; i < 10; i++) { | |
361 | size = 8 << i; | |
362 | num_tests += do_kmem_cache_rcu_persistent(size, &failures); | |
363 | } | |
364 | REPORT_FAILURES_IN_FN(); | |
365 | *total_failures += failures; | |
366 | return num_tests; | |
367 | } | |
368 | ||
369 | /* | |
370 | * Run the tests. Each test function returns the number of executed tests and | |
371 | * updates |failures| with the number of failed tests. | |
372 | */ | |
373 | static int __init test_meminit_init(void) | |
374 | { | |
375 | int failures = 0, num_tests = 0; | |
376 | ||
377 | num_tests += test_pages(&failures); | |
378 | num_tests += test_kvmalloc(&failures); | |
379 | num_tests += test_kmemcache(&failures); | |
380 | num_tests += test_rcu_persistent(&failures); | |
381 | ||
382 | if (failures == 0) | |
383 | pr_info("all %d tests passed!\n", num_tests); | |
384 | else | |
385 | pr_info("failures: %d out of %d\n", failures, num_tests); | |
386 | ||
387 | return failures ? -EINVAL : 0; | |
388 | } | |
389 | module_init(test_meminit_init); | |
390 | ||
391 | MODULE_LICENSE("GPL"); |