]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
011d8261 BP |
2 | #include <linux/mm.h> |
3 | #include <linux/gfp.h> | |
4 | #include <linux/kernel.h> | |
5 | ||
6 | #include <asm/mce.h> | |
7 | ||
8 | #include "debugfs.h" | |
9 | ||
10 | /* | |
11 | * RAS Correctable Errors Collector | |
12 | * | |
13 | * This is a simple gadget which collects correctable errors and counts their | |
14 | * occurrence per physical page address. | |
15 | * | |
16 | * We've opted for possibly the simplest data structure to collect those - an | |
17 | * array of the size of a memory page. It stores 512 u64's with the following | |
18 | * structure: | |
19 | * | |
20 | * [63 ... PFN ... 12 | 11 ... generation ... 10 | 9 ... count ... 0] | |
21 | * | |
22 | * The generation in the two highest order bits is two bits which are set to 11b | |
23 | * on every insertion. During the course of each entry's existence, the | |
24 | * generation field gets decremented during spring cleaning to 10b, then 01b and | |
25 | * then 00b. | |
26 | * | |
27 | * This way we're employing the natural numeric ordering to make sure that newly | |
28 | * inserted/touched elements have higher 12-bit counts (which we've manufactured) | |
29 | * and thus iterating over the array initially won't kick out those elements | |
30 | * which were inserted last. | |
31 | * | |
32 | * Spring cleaning is what we do when we reach a certain number CLEAN_ELEMS of | |
33 | * elements entered into the array, during which, we're decaying all elements. | |
34 | * If, after decay, an element gets inserted again, its generation is set to 11b | |
35 | * to make sure it has higher numerical count than other, older elements and | |
36 | * thus emulate an an LRU-like behavior when deleting elements to free up space | |
37 | * in the page. | |
38 | * | |
39 | * When an element reaches it's max count of count_threshold, we try to poison | |
40 | * it by assuming that errors triggered count_threshold times in a single page | |
41 | * are excessive and that page shouldn't be used anymore. count_threshold is | |
42 | * initialized to COUNT_MASK which is the maximum. | |
43 | * | |
44 | * That error event entry causes cec_add_elem() to return !0 value and thus | |
45 | * signal to its callers to log the error. | |
46 | * | |
47 | * To the question why we've chosen a page and moving elements around with | |
48 | * memmove(), it is because it is a very simple structure to handle and max data | |
49 | * movement is 4K which on highly optimized modern CPUs is almost unnoticeable. | |
50 | * We wanted to avoid the pointer traversal of more complex structures like a | |
51 | * linked list or some sort of a balancing search tree. | |
52 | * | |
53 | * Deleting an element takes O(n) but since it is only a single page, it should | |
54 | * be fast enough and it shouldn't happen all too often depending on error | |
55 | * patterns. | |
56 | */ | |
57 | ||
58 | #undef pr_fmt | |
59 | #define pr_fmt(fmt) "RAS: " fmt | |
60 | ||
61 | /* | |
62 | * We use DECAY_BITS bits of PAGE_SHIFT bits for counting decay, i.e., how long | |
63 | * elements have stayed in the array without having been accessed again. | |
64 | */ | |
65 | #define DECAY_BITS 2 | |
66 | #define DECAY_MASK ((1ULL << DECAY_BITS) - 1) | |
67 | #define MAX_ELEMS (PAGE_SIZE / sizeof(u64)) | |
68 | ||
69 | /* | |
70 | * Threshold amount of inserted elements after which we start spring | |
71 | * cleaning. | |
72 | */ | |
73 | #define CLEAN_ELEMS (MAX_ELEMS >> DECAY_BITS) | |
74 | ||
75 | /* Bits which count the number of errors happened in this 4K page. */ | |
76 | #define COUNT_BITS (PAGE_SHIFT - DECAY_BITS) | |
77 | #define COUNT_MASK ((1ULL << COUNT_BITS) - 1) | |
78 | #define FULL_COUNT_MASK (PAGE_SIZE - 1) | |
79 | ||
80 | /* | |
81 | * u64: [ 63 ... 12 | DECAY_BITS | COUNT_BITS ] | |
82 | */ | |
83 | ||
84 | #define PFN(e) ((e) >> PAGE_SHIFT) | |
85 | #define DECAY(e) (((e) >> COUNT_BITS) & DECAY_MASK) | |
86 | #define COUNT(e) ((unsigned int)(e) & COUNT_MASK) | |
87 | #define FULL_COUNT(e) ((e) & (PAGE_SIZE - 1)) | |
88 | ||
89 | static struct ce_array { | |
90 | u64 *array; /* container page */ | |
91 | unsigned int n; /* number of elements in the array */ | |
92 | ||
93 | unsigned int decay_count; /* | |
94 | * number of element insertions/increments | |
95 | * since the last spring cleaning. | |
96 | */ | |
97 | ||
98 | u64 pfns_poisoned; /* | |
99 | * number of PFNs which got poisoned. | |
100 | */ | |
101 | ||
102 | u64 ces_entered; /* | |
103 | * The number of correctable errors | |
104 | * entered into the collector. | |
105 | */ | |
106 | ||
107 | u64 decays_done; /* | |
108 | * Times we did spring cleaning. | |
109 | */ | |
110 | ||
111 | union { | |
112 | struct { | |
113 | __u32 disabled : 1, /* cmdline disabled */ | |
114 | __resv : 31; | |
115 | }; | |
116 | __u32 flags; | |
117 | }; | |
118 | } ce_arr; | |
119 | ||
120 | static DEFINE_MUTEX(ce_mutex); | |
121 | static u64 dfs_pfn; | |
122 | ||
123 | /* Amount of errors after which we offline */ | |
124 | static unsigned int count_threshold = COUNT_MASK; | |
125 | ||
126 | /* | |
127 | * The timer "decays" element count each timer_interval which is 24hrs by | |
128 | * default. | |
129 | */ | |
130 | ||
131 | #define CEC_TIMER_DEFAULT_INTERVAL 24 * 60 * 60 /* 24 hrs */ | |
132 | #define CEC_TIMER_MIN_INTERVAL 1 * 60 * 60 /* 1h */ | |
133 | #define CEC_TIMER_MAX_INTERVAL 30 * 24 * 60 * 60 /* one month */ | |
134 | static struct timer_list cec_timer; | |
135 | static u64 timer_interval = CEC_TIMER_DEFAULT_INTERVAL; | |
136 | ||
137 | /* | |
138 | * Decrement decay value. We're using DECAY_BITS bits to denote decay of an | |
139 | * element in the array. On insertion and any access, it gets reset to max. | |
140 | */ | |
141 | static void do_spring_cleaning(struct ce_array *ca) | |
142 | { | |
143 | int i; | |
144 | ||
145 | for (i = 0; i < ca->n; i++) { | |
146 | u8 decay = DECAY(ca->array[i]); | |
147 | ||
148 | if (!decay) | |
149 | continue; | |
150 | ||
151 | decay--; | |
152 | ||
153 | ca->array[i] &= ~(DECAY_MASK << COUNT_BITS); | |
154 | ca->array[i] |= (decay << COUNT_BITS); | |
155 | } | |
156 | ca->decay_count = 0; | |
157 | ca->decays_done++; | |
158 | } | |
159 | ||
160 | /* | |
161 | * @interval in seconds | |
162 | */ | |
163 | static void cec_mod_timer(struct timer_list *t, unsigned long interval) | |
164 | { | |
165 | unsigned long iv; | |
166 | ||
167 | iv = interval * HZ + jiffies; | |
168 | ||
169 | mod_timer(t, round_jiffies(iv)); | |
170 | } | |
171 | ||
254db5bd | 172 | static void cec_timer_fn(struct timer_list *unused) |
011d8261 | 173 | { |
254db5bd | 174 | do_spring_cleaning(&ce_arr); |
011d8261 BP |
175 | |
176 | cec_mod_timer(&cec_timer, timer_interval); | |
177 | } | |
178 | ||
179 | /* | |
180 | * @to: index of the smallest element which is >= then @pfn. | |
181 | * | |
182 | * Return the index of the pfn if found, otherwise negative value. | |
183 | */ | |
184 | static int __find_elem(struct ce_array *ca, u64 pfn, unsigned int *to) | |
185 | { | |
186 | u64 this_pfn; | |
187 | int min = 0, max = ca->n; | |
188 | ||
189 | while (min < max) { | |
190 | int tmp = (max + min) >> 1; | |
191 | ||
192 | this_pfn = PFN(ca->array[tmp]); | |
193 | ||
194 | if (this_pfn < pfn) | |
195 | min = tmp + 1; | |
196 | else if (this_pfn > pfn) | |
197 | max = tmp; | |
198 | else { | |
199 | min = tmp; | |
200 | break; | |
201 | } | |
202 | } | |
203 | ||
204 | if (to) | |
205 | *to = min; | |
206 | ||
207 | this_pfn = PFN(ca->array[min]); | |
208 | ||
209 | if (this_pfn == pfn) | |
210 | return min; | |
211 | ||
212 | return -ENOKEY; | |
213 | } | |
214 | ||
215 | static int find_elem(struct ce_array *ca, u64 pfn, unsigned int *to) | |
216 | { | |
217 | WARN_ON(!to); | |
218 | ||
219 | if (!ca->n) { | |
220 | *to = 0; | |
221 | return -ENOKEY; | |
222 | } | |
223 | return __find_elem(ca, pfn, to); | |
224 | } | |
225 | ||
226 | static void del_elem(struct ce_array *ca, int idx) | |
227 | { | |
228 | /* Save us a function call when deleting the last element. */ | |
229 | if (ca->n - (idx + 1)) | |
230 | memmove((void *)&ca->array[idx], | |
231 | (void *)&ca->array[idx + 1], | |
232 | (ca->n - (idx + 1)) * sizeof(u64)); | |
233 | ||
234 | ca->n--; | |
235 | } | |
236 | ||
237 | static u64 del_lru_elem_unlocked(struct ce_array *ca) | |
238 | { | |
239 | unsigned int min = FULL_COUNT_MASK; | |
240 | int i, min_idx = 0; | |
241 | ||
242 | for (i = 0; i < ca->n; i++) { | |
243 | unsigned int this = FULL_COUNT(ca->array[i]); | |
244 | ||
245 | if (min > this) { | |
246 | min = this; | |
247 | min_idx = i; | |
248 | } | |
249 | } | |
250 | ||
251 | del_elem(ca, min_idx); | |
252 | ||
253 | return PFN(ca->array[min_idx]); | |
254 | } | |
255 | ||
256 | /* | |
257 | * We return the 0th pfn in the error case under the assumption that it cannot | |
258 | * be poisoned and excessive CEs in there are a serious deal anyway. | |
259 | */ | |
260 | static u64 __maybe_unused del_lru_elem(void) | |
261 | { | |
262 | struct ce_array *ca = &ce_arr; | |
263 | u64 pfn; | |
264 | ||
265 | if (!ca->n) | |
266 | return 0; | |
267 | ||
268 | mutex_lock(&ce_mutex); | |
269 | pfn = del_lru_elem_unlocked(ca); | |
270 | mutex_unlock(&ce_mutex); | |
271 | ||
272 | return pfn; | |
273 | } | |
274 | ||
275 | ||
276 | int cec_add_elem(u64 pfn) | |
277 | { | |
278 | struct ce_array *ca = &ce_arr; | |
279 | unsigned int to; | |
280 | int count, ret = 0; | |
281 | ||
282 | /* | |
283 | * We can be called very early on the identify_cpu() path where we are | |
284 | * not initialized yet. We ignore the error for simplicity. | |
285 | */ | |
286 | if (!ce_arr.array || ce_arr.disabled) | |
287 | return -ENODEV; | |
288 | ||
289 | ca->ces_entered++; | |
290 | ||
291 | mutex_lock(&ce_mutex); | |
292 | ||
293 | if (ca->n == MAX_ELEMS) | |
294 | WARN_ON(!del_lru_elem_unlocked(ca)); | |
295 | ||
296 | ret = find_elem(ca, pfn, &to); | |
297 | if (ret < 0) { | |
298 | /* | |
299 | * Shift range [to-end] to make room for one more element. | |
300 | */ | |
301 | memmove((void *)&ca->array[to + 1], | |
302 | (void *)&ca->array[to], | |
303 | (ca->n - to) * sizeof(u64)); | |
304 | ||
305 | ca->array[to] = (pfn << PAGE_SHIFT) | | |
306 | (DECAY_MASK << COUNT_BITS) | 1; | |
307 | ||
308 | ca->n++; | |
309 | ||
310 | ret = 0; | |
311 | ||
312 | goto decay; | |
313 | } | |
314 | ||
315 | count = COUNT(ca->array[to]); | |
316 | ||
317 | if (count < count_threshold) { | |
318 | ca->array[to] |= (DECAY_MASK << COUNT_BITS); | |
319 | ca->array[to]++; | |
320 | ||
321 | ret = 0; | |
322 | } else { | |
323 | u64 pfn = ca->array[to] >> PAGE_SHIFT; | |
324 | ||
325 | if (!pfn_valid(pfn)) { | |
326 | pr_warn("CEC: Invalid pfn: 0x%llx\n", pfn); | |
327 | } else { | |
328 | /* We have reached max count for this page, soft-offline it. */ | |
329 | pr_err("Soft-offlining pfn: 0x%llx\n", pfn); | |
330 | memory_failure_queue(pfn, 0, MF_SOFT_OFFLINE); | |
331 | ca->pfns_poisoned++; | |
332 | } | |
333 | ||
334 | del_elem(ca, to); | |
335 | ||
336 | /* | |
337 | * Return a >0 value to denote that we've reached the offlining | |
338 | * threshold. | |
339 | */ | |
340 | ret = 1; | |
341 | ||
342 | goto unlock; | |
343 | } | |
344 | ||
345 | decay: | |
346 | ca->decay_count++; | |
347 | ||
348 | if (ca->decay_count >= CLEAN_ELEMS) | |
349 | do_spring_cleaning(ca); | |
350 | ||
351 | unlock: | |
352 | mutex_unlock(&ce_mutex); | |
353 | ||
354 | return ret; | |
355 | } | |
356 | ||
357 | static int u64_get(void *data, u64 *val) | |
358 | { | |
359 | *val = *(u64 *)data; | |
360 | ||
361 | return 0; | |
362 | } | |
363 | ||
364 | static int pfn_set(void *data, u64 val) | |
365 | { | |
366 | *(u64 *)data = val; | |
367 | ||
368 | return cec_add_elem(val); | |
369 | } | |
370 | ||
371 | DEFINE_DEBUGFS_ATTRIBUTE(pfn_ops, u64_get, pfn_set, "0x%llx\n"); | |
372 | ||
373 | static int decay_interval_set(void *data, u64 val) | |
374 | { | |
375 | *(u64 *)data = val; | |
376 | ||
377 | if (val < CEC_TIMER_MIN_INTERVAL) | |
378 | return -EINVAL; | |
379 | ||
380 | if (val > CEC_TIMER_MAX_INTERVAL) | |
381 | return -EINVAL; | |
382 | ||
383 | timer_interval = val; | |
384 | ||
385 | cec_mod_timer(&cec_timer, timer_interval); | |
386 | return 0; | |
387 | } | |
388 | DEFINE_DEBUGFS_ATTRIBUTE(decay_interval_ops, u64_get, decay_interval_set, "%lld\n"); | |
389 | ||
390 | static int count_threshold_set(void *data, u64 val) | |
391 | { | |
392 | *(u64 *)data = val; | |
393 | ||
394 | if (val > COUNT_MASK) | |
395 | val = COUNT_MASK; | |
396 | ||
397 | count_threshold = val; | |
398 | ||
399 | return 0; | |
400 | } | |
401 | DEFINE_DEBUGFS_ATTRIBUTE(count_threshold_ops, u64_get, count_threshold_set, "%lld\n"); | |
402 | ||
403 | static int array_dump(struct seq_file *m, void *v) | |
404 | { | |
405 | struct ce_array *ca = &ce_arr; | |
406 | u64 prev = 0; | |
407 | int i; | |
408 | ||
409 | mutex_lock(&ce_mutex); | |
410 | ||
411 | seq_printf(m, "{ n: %d\n", ca->n); | |
412 | for (i = 0; i < ca->n; i++) { | |
413 | u64 this = PFN(ca->array[i]); | |
414 | ||
415 | seq_printf(m, " %03d: [%016llx|%03llx]\n", i, this, FULL_COUNT(ca->array[i])); | |
416 | ||
417 | WARN_ON(prev > this); | |
418 | ||
419 | prev = this; | |
420 | } | |
421 | ||
422 | seq_printf(m, "}\n"); | |
423 | ||
424 | seq_printf(m, "Stats:\nCEs: %llu\nofflined pages: %llu\n", | |
425 | ca->ces_entered, ca->pfns_poisoned); | |
426 | ||
427 | seq_printf(m, "Flags: 0x%x\n", ca->flags); | |
428 | ||
429 | seq_printf(m, "Timer interval: %lld seconds\n", timer_interval); | |
430 | seq_printf(m, "Decays: %lld\n", ca->decays_done); | |
431 | ||
432 | seq_printf(m, "Action threshold: %d\n", count_threshold); | |
433 | ||
434 | mutex_unlock(&ce_mutex); | |
435 | ||
436 | return 0; | |
437 | } | |
438 | ||
439 | static int array_open(struct inode *inode, struct file *filp) | |
440 | { | |
441 | return single_open(filp, array_dump, NULL); | |
442 | } | |
443 | ||
444 | static const struct file_operations array_ops = { | |
445 | .owner = THIS_MODULE, | |
446 | .open = array_open, | |
447 | .read = seq_read, | |
448 | .llseek = seq_lseek, | |
449 | .release = single_release, | |
450 | }; | |
451 | ||
452 | static int __init create_debugfs_nodes(void) | |
453 | { | |
454 | struct dentry *d, *pfn, *decay, *count, *array; | |
455 | ||
456 | d = debugfs_create_dir("cec", ras_debugfs_dir); | |
457 | if (!d) { | |
458 | pr_warn("Error creating cec debugfs node!\n"); | |
459 | return -1; | |
460 | } | |
461 | ||
462 | pfn = debugfs_create_file("pfn", S_IRUSR | S_IWUSR, d, &dfs_pfn, &pfn_ops); | |
463 | if (!pfn) { | |
464 | pr_warn("Error creating pfn debugfs node!\n"); | |
465 | goto err; | |
466 | } | |
467 | ||
468 | array = debugfs_create_file("array", S_IRUSR, d, NULL, &array_ops); | |
469 | if (!array) { | |
470 | pr_warn("Error creating array debugfs node!\n"); | |
471 | goto err; | |
472 | } | |
473 | ||
474 | decay = debugfs_create_file("decay_interval", S_IRUSR | S_IWUSR, d, | |
475 | &timer_interval, &decay_interval_ops); | |
476 | if (!decay) { | |
477 | pr_warn("Error creating decay_interval debugfs node!\n"); | |
478 | goto err; | |
479 | } | |
480 | ||
481 | count = debugfs_create_file("count_threshold", S_IRUSR | S_IWUSR, d, | |
482 | &count_threshold, &count_threshold_ops); | |
32288daf | 483 | if (!count) { |
011d8261 BP |
484 | pr_warn("Error creating count_threshold debugfs node!\n"); |
485 | goto err; | |
486 | } | |
487 | ||
488 | ||
489 | return 0; | |
490 | ||
491 | err: | |
492 | debugfs_remove_recursive(d); | |
493 | ||
494 | return 1; | |
495 | } | |
496 | ||
497 | void __init cec_init(void) | |
498 | { | |
499 | if (ce_arr.disabled) | |
500 | return; | |
501 | ||
502 | ce_arr.array = (void *)get_zeroed_page(GFP_KERNEL); | |
503 | if (!ce_arr.array) { | |
504 | pr_err("Error allocating CE array page!\n"); | |
505 | return; | |
506 | } | |
507 | ||
508 | if (create_debugfs_nodes()) | |
509 | return; | |
510 | ||
254db5bd | 511 | timer_setup(&cec_timer, cec_timer_fn, 0); |
011d8261 BP |
512 | cec_mod_timer(&cec_timer, CEC_TIMER_DEFAULT_INTERVAL); |
513 | ||
514 | pr_info("Correctable Errors collector initialized.\n"); | |
515 | } | |
516 | ||
517 | int __init parse_cec_param(char *str) | |
518 | { | |
519 | if (!str) | |
520 | return 0; | |
521 | ||
522 | if (*str == '=') | |
523 | str++; | |
524 | ||
69a33000 | 525 | if (!strcmp(str, "cec_disable")) |
011d8261 BP |
526 | ce_arr.disabled = 1; |
527 | else | |
528 | return 0; | |
529 | ||
530 | return 1; | |
531 | } |