]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - mm/memory-failure.c
Merge tag 'armsoc-drivers' of git://git.kernel.org/pub/scm/linux/kernel/git/arm/arm-soc
[mirror_ubuntu-bionic-kernel.git] / mm / memory-failure.c
1 /*
2 * Copyright (C) 2008, 2009 Intel Corporation
3 * Authors: Andi Kleen, Fengguang Wu
4 *
5 * This software may be redistributed and/or modified under the terms of
6 * the GNU General Public License ("GPL") version 2 only as published by the
7 * Free Software Foundation.
8 *
9 * High level machine check handler. Handles pages reported by the
10 * hardware as being corrupted usually due to a multi-bit ECC memory or cache
11 * failure.
12 *
13 * In addition there is a "soft offline" entry point that allows stop using
14 * not-yet-corrupted-by-suspicious pages without killing anything.
15 *
16 * Handles page cache pages in various states. The tricky part
17 * here is that we can access any page asynchronously in respect to
18 * other VM users, because memory failures could happen anytime and
19 * anywhere. This could violate some of their assumptions. This is why
20 * this code has to be extremely careful. Generally it tries to use
21 * normal locking rules, as in get the standard locks, even if that means
22 * the error handling takes potentially a long time.
23 *
24 * It can be very tempting to add handling for obscure cases here.
25 * In general any code for handling new cases should only be added iff:
26 * - You know how to test it.
27 * - You have a test that can be added to mce-test
28 * https://git.kernel.org/cgit/utils/cpu/mce/mce-test.git/
29 * - The case actually shows up as a frequent (top 10) page state in
30 * tools/vm/page-types when running a real workload.
31 *
32 * There are several operations here with exponential complexity because
33 * of unsuitable VM data structures. For example the operation to map back
34 * from RMAP chains to processes has to walk the complete process list and
35 * has non linear complexity with the number. But since memory corruptions
36 * are rare we hope to get away with this. This avoids impacting the core
37 * VM.
38 */
39 #include <linux/kernel.h>
40 #include <linux/mm.h>
41 #include <linux/page-flags.h>
42 #include <linux/kernel-page-flags.h>
43 #include <linux/sched/signal.h>
44 #include <linux/sched/task.h>
45 #include <linux/ksm.h>
46 #include <linux/rmap.h>
47 #include <linux/export.h>
48 #include <linux/pagemap.h>
49 #include <linux/swap.h>
50 #include <linux/backing-dev.h>
51 #include <linux/migrate.h>
52 #include <linux/page-isolation.h>
53 #include <linux/suspend.h>
54 #include <linux/slab.h>
55 #include <linux/swapops.h>
56 #include <linux/hugetlb.h>
57 #include <linux/memory_hotplug.h>
58 #include <linux/mm_inline.h>
59 #include <linux/kfifo.h>
60 #include <linux/ratelimit.h>
61 #include "internal.h"
62 #include "ras/ras_event.h"
63
64 int sysctl_memory_failure_early_kill __read_mostly = 0;
65
66 int sysctl_memory_failure_recovery __read_mostly = 1;
67
68 atomic_long_t num_poisoned_pages __read_mostly = ATOMIC_LONG_INIT(0);
69
70 #if defined(CONFIG_HWPOISON_INJECT) || defined(CONFIG_HWPOISON_INJECT_MODULE)
71
72 u32 hwpoison_filter_enable = 0;
73 u32 hwpoison_filter_dev_major = ~0U;
74 u32 hwpoison_filter_dev_minor = ~0U;
75 u64 hwpoison_filter_flags_mask;
76 u64 hwpoison_filter_flags_value;
77 EXPORT_SYMBOL_GPL(hwpoison_filter_enable);
78 EXPORT_SYMBOL_GPL(hwpoison_filter_dev_major);
79 EXPORT_SYMBOL_GPL(hwpoison_filter_dev_minor);
80 EXPORT_SYMBOL_GPL(hwpoison_filter_flags_mask);
81 EXPORT_SYMBOL_GPL(hwpoison_filter_flags_value);
82
83 static int hwpoison_filter_dev(struct page *p)
84 {
85 struct address_space *mapping;
86 dev_t dev;
87
88 if (hwpoison_filter_dev_major == ~0U &&
89 hwpoison_filter_dev_minor == ~0U)
90 return 0;
91
92 /*
93 * page_mapping() does not accept slab pages.
94 */
95 if (PageSlab(p))
96 return -EINVAL;
97
98 mapping = page_mapping(p);
99 if (mapping == NULL || mapping->host == NULL)
100 return -EINVAL;
101
102 dev = mapping->host->i_sb->s_dev;
103 if (hwpoison_filter_dev_major != ~0U &&
104 hwpoison_filter_dev_major != MAJOR(dev))
105 return -EINVAL;
106 if (hwpoison_filter_dev_minor != ~0U &&
107 hwpoison_filter_dev_minor != MINOR(dev))
108 return -EINVAL;
109
110 return 0;
111 }
112
113 static int hwpoison_filter_flags(struct page *p)
114 {
115 if (!hwpoison_filter_flags_mask)
116 return 0;
117
118 if ((stable_page_flags(p) & hwpoison_filter_flags_mask) ==
119 hwpoison_filter_flags_value)
120 return 0;
121 else
122 return -EINVAL;
123 }
124
125 /*
126 * This allows stress tests to limit test scope to a collection of tasks
127 * by putting them under some memcg. This prevents killing unrelated/important
128 * processes such as /sbin/init. Note that the target task may share clean
129 * pages with init (eg. libc text), which is harmless. If the target task
130 * share _dirty_ pages with another task B, the test scheme must make sure B
131 * is also included in the memcg. At last, due to race conditions this filter
132 * can only guarantee that the page either belongs to the memcg tasks, or is
133 * a freed page.
134 */
135 #ifdef CONFIG_MEMCG
136 u64 hwpoison_filter_memcg;
137 EXPORT_SYMBOL_GPL(hwpoison_filter_memcg);
138 static int hwpoison_filter_task(struct page *p)
139 {
140 if (!hwpoison_filter_memcg)
141 return 0;
142
143 if (page_cgroup_ino(p) != hwpoison_filter_memcg)
144 return -EINVAL;
145
146 return 0;
147 }
148 #else
149 static int hwpoison_filter_task(struct page *p) { return 0; }
150 #endif
151
152 int hwpoison_filter(struct page *p)
153 {
154 if (!hwpoison_filter_enable)
155 return 0;
156
157 if (hwpoison_filter_dev(p))
158 return -EINVAL;
159
160 if (hwpoison_filter_flags(p))
161 return -EINVAL;
162
163 if (hwpoison_filter_task(p))
164 return -EINVAL;
165
166 return 0;
167 }
168 #else
169 int hwpoison_filter(struct page *p)
170 {
171 return 0;
172 }
173 #endif
174
175 EXPORT_SYMBOL_GPL(hwpoison_filter);
176
177 /*
178 * Send all the processes who have the page mapped a signal.
179 * ``action optional'' if they are not immediately affected by the error
180 * ``action required'' if error happened in current execution context
181 */
182 static int kill_proc(struct task_struct *t, unsigned long addr, int trapno,
183 unsigned long pfn, struct page *page, int flags)
184 {
185 struct siginfo si;
186 int ret;
187
188 pr_err("Memory failure: %#lx: Killing %s:%d due to hardware memory corruption\n",
189 pfn, t->comm, t->pid);
190 si.si_signo = SIGBUS;
191 si.si_errno = 0;
192 si.si_addr = (void *)addr;
193 #ifdef __ARCH_SI_TRAPNO
194 si.si_trapno = trapno;
195 #endif
196 si.si_addr_lsb = compound_order(compound_head(page)) + PAGE_SHIFT;
197
198 if ((flags & MF_ACTION_REQUIRED) && t->mm == current->mm) {
199 si.si_code = BUS_MCEERR_AR;
200 ret = force_sig_info(SIGBUS, &si, current);
201 } else {
202 /*
203 * Don't use force here, it's convenient if the signal
204 * can be temporarily blocked.
205 * This could cause a loop when the user sets SIGBUS
206 * to SIG_IGN, but hopefully no one will do that?
207 */
208 si.si_code = BUS_MCEERR_AO;
209 ret = send_sig_info(SIGBUS, &si, t); /* synchronous? */
210 }
211 if (ret < 0)
212 pr_info("Memory failure: Error sending signal to %s:%d: %d\n",
213 t->comm, t->pid, ret);
214 return ret;
215 }
216
217 /*
218 * When a unknown page type is encountered drain as many buffers as possible
219 * in the hope to turn the page into a LRU or free page, which we can handle.
220 */
221 void shake_page(struct page *p, int access)
222 {
223 if (PageHuge(p))
224 return;
225
226 if (!PageSlab(p)) {
227 lru_add_drain_all();
228 if (PageLRU(p))
229 return;
230 drain_all_pages(page_zone(p));
231 if (PageLRU(p) || is_free_buddy_page(p))
232 return;
233 }
234
235 /*
236 * Only call shrink_node_slabs here (which would also shrink
237 * other caches) if access is not potentially fatal.
238 */
239 if (access)
240 drop_slab_node(page_to_nid(p));
241 }
242 EXPORT_SYMBOL_GPL(shake_page);
243
244 /*
245 * Kill all processes that have a poisoned page mapped and then isolate
246 * the page.
247 *
248 * General strategy:
249 * Find all processes having the page mapped and kill them.
250 * But we keep a page reference around so that the page is not
251 * actually freed yet.
252 * Then stash the page away
253 *
254 * There's no convenient way to get back to mapped processes
255 * from the VMAs. So do a brute-force search over all
256 * running processes.
257 *
258 * Remember that machine checks are not common (or rather
259 * if they are common you have other problems), so this shouldn't
260 * be a performance issue.
261 *
262 * Also there are some races possible while we get from the
263 * error detection to actually handle it.
264 */
265
266 struct to_kill {
267 struct list_head nd;
268 struct task_struct *tsk;
269 unsigned long addr;
270 char addr_valid;
271 };
272
273 /*
274 * Failure handling: if we can't find or can't kill a process there's
275 * not much we can do. We just print a message and ignore otherwise.
276 */
277
278 /*
279 * Schedule a process for later kill.
280 * Uses GFP_ATOMIC allocations to avoid potential recursions in the VM.
281 * TBD would GFP_NOIO be enough?
282 */
283 static void add_to_kill(struct task_struct *tsk, struct page *p,
284 struct vm_area_struct *vma,
285 struct list_head *to_kill,
286 struct to_kill **tkc)
287 {
288 struct to_kill *tk;
289
290 if (*tkc) {
291 tk = *tkc;
292 *tkc = NULL;
293 } else {
294 tk = kmalloc(sizeof(struct to_kill), GFP_ATOMIC);
295 if (!tk) {
296 pr_err("Memory failure: Out of memory while machine check handling\n");
297 return;
298 }
299 }
300 tk->addr = page_address_in_vma(p, vma);
301 tk->addr_valid = 1;
302
303 /*
304 * In theory we don't have to kill when the page was
305 * munmaped. But it could be also a mremap. Since that's
306 * likely very rare kill anyways just out of paranoia, but use
307 * a SIGKILL because the error is not contained anymore.
308 */
309 if (tk->addr == -EFAULT) {
310 pr_info("Memory failure: Unable to find user space address %lx in %s\n",
311 page_to_pfn(p), tsk->comm);
312 tk->addr_valid = 0;
313 }
314 get_task_struct(tsk);
315 tk->tsk = tsk;
316 list_add_tail(&tk->nd, to_kill);
317 }
318
319 /*
320 * Kill the processes that have been collected earlier.
321 *
322 * Only do anything when DOIT is set, otherwise just free the list
323 * (this is used for clean pages which do not need killing)
324 * Also when FAIL is set do a force kill because something went
325 * wrong earlier.
326 */
327 static void kill_procs(struct list_head *to_kill, int forcekill, int trapno,
328 bool fail, struct page *page, unsigned long pfn,
329 int flags)
330 {
331 struct to_kill *tk, *next;
332
333 list_for_each_entry_safe (tk, next, to_kill, nd) {
334 if (forcekill) {
335 /*
336 * In case something went wrong with munmapping
337 * make sure the process doesn't catch the
338 * signal and then access the memory. Just kill it.
339 */
340 if (fail || tk->addr_valid == 0) {
341 pr_err("Memory failure: %#lx: forcibly killing %s:%d because of failure to unmap corrupted page\n",
342 pfn, tk->tsk->comm, tk->tsk->pid);
343 force_sig(SIGKILL, tk->tsk);
344 }
345
346 /*
347 * In theory the process could have mapped
348 * something else on the address in-between. We could
349 * check for that, but we need to tell the
350 * process anyways.
351 */
352 else if (kill_proc(tk->tsk, tk->addr, trapno,
353 pfn, page, flags) < 0)
354 pr_err("Memory failure: %#lx: Cannot send advisory machine check signal to %s:%d\n",
355 pfn, tk->tsk->comm, tk->tsk->pid);
356 }
357 put_task_struct(tk->tsk);
358 kfree(tk);
359 }
360 }
361
362 /*
363 * Find a dedicated thread which is supposed to handle SIGBUS(BUS_MCEERR_AO)
364 * on behalf of the thread group. Return task_struct of the (first found)
365 * dedicated thread if found, and return NULL otherwise.
366 *
367 * We already hold read_lock(&tasklist_lock) in the caller, so we don't
368 * have to call rcu_read_lock/unlock() in this function.
369 */
370 static struct task_struct *find_early_kill_thread(struct task_struct *tsk)
371 {
372 struct task_struct *t;
373
374 for_each_thread(tsk, t)
375 if ((t->flags & PF_MCE_PROCESS) && (t->flags & PF_MCE_EARLY))
376 return t;
377 return NULL;
378 }
379
380 /*
381 * Determine whether a given process is "early kill" process which expects
382 * to be signaled when some page under the process is hwpoisoned.
383 * Return task_struct of the dedicated thread (main thread unless explicitly
384 * specified) if the process is "early kill," and otherwise returns NULL.
385 */
386 static struct task_struct *task_early_kill(struct task_struct *tsk,
387 int force_early)
388 {
389 struct task_struct *t;
390 if (!tsk->mm)
391 return NULL;
392 if (force_early)
393 return tsk;
394 t = find_early_kill_thread(tsk);
395 if (t)
396 return t;
397 if (sysctl_memory_failure_early_kill)
398 return tsk;
399 return NULL;
400 }
401
402 /*
403 * Collect processes when the error hit an anonymous page.
404 */
405 static void collect_procs_anon(struct page *page, struct list_head *to_kill,
406 struct to_kill **tkc, int force_early)
407 {
408 struct vm_area_struct *vma;
409 struct task_struct *tsk;
410 struct anon_vma *av;
411 pgoff_t pgoff;
412
413 av = page_lock_anon_vma_read(page);
414 if (av == NULL) /* Not actually mapped anymore */
415 return;
416
417 pgoff = page_to_pgoff(page);
418 read_lock(&tasklist_lock);
419 for_each_process (tsk) {
420 struct anon_vma_chain *vmac;
421 struct task_struct *t = task_early_kill(tsk, force_early);
422
423 if (!t)
424 continue;
425 anon_vma_interval_tree_foreach(vmac, &av->rb_root,
426 pgoff, pgoff) {
427 vma = vmac->vma;
428 if (!page_mapped_in_vma(page, vma))
429 continue;
430 if (vma->vm_mm == t->mm)
431 add_to_kill(t, page, vma, to_kill, tkc);
432 }
433 }
434 read_unlock(&tasklist_lock);
435 page_unlock_anon_vma_read(av);
436 }
437
438 /*
439 * Collect processes when the error hit a file mapped page.
440 */
441 static void collect_procs_file(struct page *page, struct list_head *to_kill,
442 struct to_kill **tkc, int force_early)
443 {
444 struct vm_area_struct *vma;
445 struct task_struct *tsk;
446 struct address_space *mapping = page->mapping;
447
448 i_mmap_lock_read(mapping);
449 read_lock(&tasklist_lock);
450 for_each_process(tsk) {
451 pgoff_t pgoff = page_to_pgoff(page);
452 struct task_struct *t = task_early_kill(tsk, force_early);
453
454 if (!t)
455 continue;
456 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff,
457 pgoff) {
458 /*
459 * Send early kill signal to tasks where a vma covers
460 * the page but the corrupted page is not necessarily
461 * mapped it in its pte.
462 * Assume applications who requested early kill want
463 * to be informed of all such data corruptions.
464 */
465 if (vma->vm_mm == t->mm)
466 add_to_kill(t, page, vma, to_kill, tkc);
467 }
468 }
469 read_unlock(&tasklist_lock);
470 i_mmap_unlock_read(mapping);
471 }
472
473 /*
474 * Collect the processes who have the corrupted page mapped to kill.
475 * This is done in two steps for locking reasons.
476 * First preallocate one tokill structure outside the spin locks,
477 * so that we can kill at least one process reasonably reliable.
478 */
479 static void collect_procs(struct page *page, struct list_head *tokill,
480 int force_early)
481 {
482 struct to_kill *tk;
483
484 if (!page->mapping)
485 return;
486
487 tk = kmalloc(sizeof(struct to_kill), GFP_NOIO);
488 if (!tk)
489 return;
490 if (PageAnon(page))
491 collect_procs_anon(page, tokill, &tk, force_early);
492 else
493 collect_procs_file(page, tokill, &tk, force_early);
494 kfree(tk);
495 }
496
497 static const char *action_name[] = {
498 [MF_IGNORED] = "Ignored",
499 [MF_FAILED] = "Failed",
500 [MF_DELAYED] = "Delayed",
501 [MF_RECOVERED] = "Recovered",
502 };
503
504 static const char * const action_page_types[] = {
505 [MF_MSG_KERNEL] = "reserved kernel page",
506 [MF_MSG_KERNEL_HIGH_ORDER] = "high-order kernel page",
507 [MF_MSG_SLAB] = "kernel slab page",
508 [MF_MSG_DIFFERENT_COMPOUND] = "different compound page after locking",
509 [MF_MSG_POISONED_HUGE] = "huge page already hardware poisoned",
510 [MF_MSG_HUGE] = "huge page",
511 [MF_MSG_FREE_HUGE] = "free huge page",
512 [MF_MSG_UNMAP_FAILED] = "unmapping failed page",
513 [MF_MSG_DIRTY_SWAPCACHE] = "dirty swapcache page",
514 [MF_MSG_CLEAN_SWAPCACHE] = "clean swapcache page",
515 [MF_MSG_DIRTY_MLOCKED_LRU] = "dirty mlocked LRU page",
516 [MF_MSG_CLEAN_MLOCKED_LRU] = "clean mlocked LRU page",
517 [MF_MSG_DIRTY_UNEVICTABLE_LRU] = "dirty unevictable LRU page",
518 [MF_MSG_CLEAN_UNEVICTABLE_LRU] = "clean unevictable LRU page",
519 [MF_MSG_DIRTY_LRU] = "dirty LRU page",
520 [MF_MSG_CLEAN_LRU] = "clean LRU page",
521 [MF_MSG_TRUNCATED_LRU] = "already truncated LRU page",
522 [MF_MSG_BUDDY] = "free buddy page",
523 [MF_MSG_BUDDY_2ND] = "free buddy page (2nd try)",
524 [MF_MSG_UNKNOWN] = "unknown page",
525 };
526
527 /*
528 * XXX: It is possible that a page is isolated from LRU cache,
529 * and then kept in swap cache or failed to remove from page cache.
530 * The page count will stop it from being freed by unpoison.
531 * Stress tests should be aware of this memory leak problem.
532 */
533 static int delete_from_lru_cache(struct page *p)
534 {
535 if (!isolate_lru_page(p)) {
536 /*
537 * Clear sensible page flags, so that the buddy system won't
538 * complain when the page is unpoison-and-freed.
539 */
540 ClearPageActive(p);
541 ClearPageUnevictable(p);
542 /*
543 * drop the page count elevated by isolate_lru_page()
544 */
545 put_page(p);
546 return 0;
547 }
548 return -EIO;
549 }
550
551 /*
552 * Error hit kernel page.
553 * Do nothing, try to be lucky and not touch this instead. For a few cases we
554 * could be more sophisticated.
555 */
556 static int me_kernel(struct page *p, unsigned long pfn)
557 {
558 return MF_IGNORED;
559 }
560
561 /*
562 * Page in unknown state. Do nothing.
563 */
564 static int me_unknown(struct page *p, unsigned long pfn)
565 {
566 pr_err("Memory failure: %#lx: Unknown page state\n", pfn);
567 return MF_FAILED;
568 }
569
570 /*
571 * Clean (or cleaned) page cache page.
572 */
573 static int me_pagecache_clean(struct page *p, unsigned long pfn)
574 {
575 int err;
576 int ret = MF_FAILED;
577 struct address_space *mapping;
578
579 delete_from_lru_cache(p);
580
581 /*
582 * For anonymous pages we're done the only reference left
583 * should be the one m_f() holds.
584 */
585 if (PageAnon(p))
586 return MF_RECOVERED;
587
588 /*
589 * Now truncate the page in the page cache. This is really
590 * more like a "temporary hole punch"
591 * Don't do this for block devices when someone else
592 * has a reference, because it could be file system metadata
593 * and that's not safe to truncate.
594 */
595 mapping = page_mapping(p);
596 if (!mapping) {
597 /*
598 * Page has been teared down in the meanwhile
599 */
600 return MF_FAILED;
601 }
602
603 /*
604 * Truncation is a bit tricky. Enable it per file system for now.
605 *
606 * Open: to take i_mutex or not for this? Right now we don't.
607 */
608 if (mapping->a_ops->error_remove_page) {
609 err = mapping->a_ops->error_remove_page(mapping, p);
610 if (err != 0) {
611 pr_info("Memory failure: %#lx: Failed to punch page: %d\n",
612 pfn, err);
613 } else if (page_has_private(p) &&
614 !try_to_release_page(p, GFP_NOIO)) {
615 pr_info("Memory failure: %#lx: failed to release buffers\n",
616 pfn);
617 } else {
618 ret = MF_RECOVERED;
619 }
620 } else {
621 /*
622 * If the file system doesn't support it just invalidate
623 * This fails on dirty or anything with private pages
624 */
625 if (invalidate_inode_page(p))
626 ret = MF_RECOVERED;
627 else
628 pr_info("Memory failure: %#lx: Failed to invalidate\n",
629 pfn);
630 }
631 return ret;
632 }
633
634 /*
635 * Dirty pagecache page
636 * Issues: when the error hit a hole page the error is not properly
637 * propagated.
638 */
639 static int me_pagecache_dirty(struct page *p, unsigned long pfn)
640 {
641 struct address_space *mapping = page_mapping(p);
642
643 SetPageError(p);
644 /* TBD: print more information about the file. */
645 if (mapping) {
646 /*
647 * IO error will be reported by write(), fsync(), etc.
648 * who check the mapping.
649 * This way the application knows that something went
650 * wrong with its dirty file data.
651 *
652 * There's one open issue:
653 *
654 * The EIO will be only reported on the next IO
655 * operation and then cleared through the IO map.
656 * Normally Linux has two mechanisms to pass IO error
657 * first through the AS_EIO flag in the address space
658 * and then through the PageError flag in the page.
659 * Since we drop pages on memory failure handling the
660 * only mechanism open to use is through AS_AIO.
661 *
662 * This has the disadvantage that it gets cleared on
663 * the first operation that returns an error, while
664 * the PageError bit is more sticky and only cleared
665 * when the page is reread or dropped. If an
666 * application assumes it will always get error on
667 * fsync, but does other operations on the fd before
668 * and the page is dropped between then the error
669 * will not be properly reported.
670 *
671 * This can already happen even without hwpoisoned
672 * pages: first on metadata IO errors (which only
673 * report through AS_EIO) or when the page is dropped
674 * at the wrong time.
675 *
676 * So right now we assume that the application DTRT on
677 * the first EIO, but we're not worse than other parts
678 * of the kernel.
679 */
680 mapping_set_error(mapping, EIO);
681 }
682
683 return me_pagecache_clean(p, pfn);
684 }
685
686 /*
687 * Clean and dirty swap cache.
688 *
689 * Dirty swap cache page is tricky to handle. The page could live both in page
690 * cache and swap cache(ie. page is freshly swapped in). So it could be
691 * referenced concurrently by 2 types of PTEs:
692 * normal PTEs and swap PTEs. We try to handle them consistently by calling
693 * try_to_unmap(TTU_IGNORE_HWPOISON) to convert the normal PTEs to swap PTEs,
694 * and then
695 * - clear dirty bit to prevent IO
696 * - remove from LRU
697 * - but keep in the swap cache, so that when we return to it on
698 * a later page fault, we know the application is accessing
699 * corrupted data and shall be killed (we installed simple
700 * interception code in do_swap_page to catch it).
701 *
702 * Clean swap cache pages can be directly isolated. A later page fault will
703 * bring in the known good data from disk.
704 */
705 static int me_swapcache_dirty(struct page *p, unsigned long pfn)
706 {
707 ClearPageDirty(p);
708 /* Trigger EIO in shmem: */
709 ClearPageUptodate(p);
710
711 if (!delete_from_lru_cache(p))
712 return MF_DELAYED;
713 else
714 return MF_FAILED;
715 }
716
717 static int me_swapcache_clean(struct page *p, unsigned long pfn)
718 {
719 delete_from_swap_cache(p);
720
721 if (!delete_from_lru_cache(p))
722 return MF_RECOVERED;
723 else
724 return MF_FAILED;
725 }
726
727 /*
728 * Huge pages. Needs work.
729 * Issues:
730 * - Error on hugepage is contained in hugepage unit (not in raw page unit.)
731 * To narrow down kill region to one page, we need to break up pmd.
732 */
733 static int me_huge_page(struct page *p, unsigned long pfn)
734 {
735 int res = 0;
736 struct page *hpage = compound_head(p);
737
738 if (!PageHuge(hpage))
739 return MF_DELAYED;
740
741 /*
742 * We can safely recover from error on free or reserved (i.e.
743 * not in-use) hugepage by dequeuing it from freelist.
744 * To check whether a hugepage is in-use or not, we can't use
745 * page->lru because it can be used in other hugepage operations,
746 * such as __unmap_hugepage_range() and gather_surplus_pages().
747 * So instead we use page_mapping() and PageAnon().
748 */
749 if (!(page_mapping(hpage) || PageAnon(hpage))) {
750 res = dequeue_hwpoisoned_huge_page(hpage);
751 if (!res)
752 return MF_RECOVERED;
753 }
754 return MF_DELAYED;
755 }
756
757 /*
758 * Various page states we can handle.
759 *
760 * A page state is defined by its current page->flags bits.
761 * The table matches them in order and calls the right handler.
762 *
763 * This is quite tricky because we can access page at any time
764 * in its live cycle, so all accesses have to be extremely careful.
765 *
766 * This is not complete. More states could be added.
767 * For any missing state don't attempt recovery.
768 */
769
770 #define dirty (1UL << PG_dirty)
771 #define sc ((1UL << PG_swapcache) | (1UL << PG_swapbacked))
772 #define unevict (1UL << PG_unevictable)
773 #define mlock (1UL << PG_mlocked)
774 #define writeback (1UL << PG_writeback)
775 #define lru (1UL << PG_lru)
776 #define head (1UL << PG_head)
777 #define slab (1UL << PG_slab)
778 #define reserved (1UL << PG_reserved)
779
780 static struct page_state {
781 unsigned long mask;
782 unsigned long res;
783 enum mf_action_page_type type;
784 int (*action)(struct page *p, unsigned long pfn);
785 } error_states[] = {
786 { reserved, reserved, MF_MSG_KERNEL, me_kernel },
787 /*
788 * free pages are specially detected outside this table:
789 * PG_buddy pages only make a small fraction of all free pages.
790 */
791
792 /*
793 * Could in theory check if slab page is free or if we can drop
794 * currently unused objects without touching them. But just
795 * treat it as standard kernel for now.
796 */
797 { slab, slab, MF_MSG_SLAB, me_kernel },
798
799 { head, head, MF_MSG_HUGE, me_huge_page },
800
801 { sc|dirty, sc|dirty, MF_MSG_DIRTY_SWAPCACHE, me_swapcache_dirty },
802 { sc|dirty, sc, MF_MSG_CLEAN_SWAPCACHE, me_swapcache_clean },
803
804 { mlock|dirty, mlock|dirty, MF_MSG_DIRTY_MLOCKED_LRU, me_pagecache_dirty },
805 { mlock|dirty, mlock, MF_MSG_CLEAN_MLOCKED_LRU, me_pagecache_clean },
806
807 { unevict|dirty, unevict|dirty, MF_MSG_DIRTY_UNEVICTABLE_LRU, me_pagecache_dirty },
808 { unevict|dirty, unevict, MF_MSG_CLEAN_UNEVICTABLE_LRU, me_pagecache_clean },
809
810 { lru|dirty, lru|dirty, MF_MSG_DIRTY_LRU, me_pagecache_dirty },
811 { lru|dirty, lru, MF_MSG_CLEAN_LRU, me_pagecache_clean },
812
813 /*
814 * Catchall entry: must be at end.
815 */
816 { 0, 0, MF_MSG_UNKNOWN, me_unknown },
817 };
818
819 #undef dirty
820 #undef sc
821 #undef unevict
822 #undef mlock
823 #undef writeback
824 #undef lru
825 #undef head
826 #undef slab
827 #undef reserved
828
829 /*
830 * "Dirty/Clean" indication is not 100% accurate due to the possibility of
831 * setting PG_dirty outside page lock. See also comment above set_page_dirty().
832 */
833 static void action_result(unsigned long pfn, enum mf_action_page_type type,
834 enum mf_result result)
835 {
836 trace_memory_failure_event(pfn, type, result);
837
838 pr_err("Memory failure: %#lx: recovery action for %s: %s\n",
839 pfn, action_page_types[type], action_name[result]);
840 }
841
842 static int page_action(struct page_state *ps, struct page *p,
843 unsigned long pfn)
844 {
845 int result;
846 int count;
847
848 result = ps->action(p, pfn);
849
850 count = page_count(p) - 1;
851 if (ps->action == me_swapcache_dirty && result == MF_DELAYED)
852 count--;
853 if (count != 0) {
854 pr_err("Memory failure: %#lx: %s still referenced by %d users\n",
855 pfn, action_page_types[ps->type], count);
856 result = MF_FAILED;
857 }
858 action_result(pfn, ps->type, result);
859
860 /* Could do more checks here if page looks ok */
861 /*
862 * Could adjust zone counters here to correct for the missing page.
863 */
864
865 return (result == MF_RECOVERED || result == MF_DELAYED) ? 0 : -EBUSY;
866 }
867
868 /**
869 * get_hwpoison_page() - Get refcount for memory error handling:
870 * @page: raw error page (hit by memory error)
871 *
872 * Return: return 0 if failed to grab the refcount, otherwise true (some
873 * non-zero value.)
874 */
875 int get_hwpoison_page(struct page *page)
876 {
877 struct page *head = compound_head(page);
878
879 if (!PageHuge(head) && PageTransHuge(head)) {
880 /*
881 * Non anonymous thp exists only in allocation/free time. We
882 * can't handle such a case correctly, so let's give it up.
883 * This should be better than triggering BUG_ON when kernel
884 * tries to touch the "partially handled" page.
885 */
886 if (!PageAnon(head)) {
887 pr_err("Memory failure: %#lx: non anonymous thp\n",
888 page_to_pfn(page));
889 return 0;
890 }
891 }
892
893 if (get_page_unless_zero(head)) {
894 if (head == compound_head(page))
895 return 1;
896
897 pr_info("Memory failure: %#lx cannot catch tail\n",
898 page_to_pfn(page));
899 put_page(head);
900 }
901
902 return 0;
903 }
904 EXPORT_SYMBOL_GPL(get_hwpoison_page);
905
906 /*
907 * Do all that is necessary to remove user space mappings. Unmap
908 * the pages and send SIGBUS to the processes if the data was dirty.
909 */
910 static bool hwpoison_user_mappings(struct page *p, unsigned long pfn,
911 int trapno, int flags, struct page **hpagep)
912 {
913 enum ttu_flags ttu = TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS;
914 struct address_space *mapping;
915 LIST_HEAD(tokill);
916 bool unmap_success;
917 int kill = 1, forcekill;
918 struct page *hpage = *hpagep;
919 bool mlocked = PageMlocked(hpage);
920
921 /*
922 * Here we are interested only in user-mapped pages, so skip any
923 * other types of pages.
924 */
925 if (PageReserved(p) || PageSlab(p))
926 return true;
927 if (!(PageLRU(hpage) || PageHuge(p)))
928 return true;
929
930 /*
931 * This check implies we don't kill processes if their pages
932 * are in the swap cache early. Those are always late kills.
933 */
934 if (!page_mapped(hpage))
935 return true;
936
937 if (PageKsm(p)) {
938 pr_err("Memory failure: %#lx: can't handle KSM pages.\n", pfn);
939 return false;
940 }
941
942 if (PageSwapCache(p)) {
943 pr_err("Memory failure: %#lx: keeping poisoned page in swap cache\n",
944 pfn);
945 ttu |= TTU_IGNORE_HWPOISON;
946 }
947
948 /*
949 * Propagate the dirty bit from PTEs to struct page first, because we
950 * need this to decide if we should kill or just drop the page.
951 * XXX: the dirty test could be racy: set_page_dirty() may not always
952 * be called inside page lock (it's recommended but not enforced).
953 */
954 mapping = page_mapping(hpage);
955 if (!(flags & MF_MUST_KILL) && !PageDirty(hpage) && mapping &&
956 mapping_cap_writeback_dirty(mapping)) {
957 if (page_mkclean(hpage)) {
958 SetPageDirty(hpage);
959 } else {
960 kill = 0;
961 ttu |= TTU_IGNORE_HWPOISON;
962 pr_info("Memory failure: %#lx: corrupted page was clean: dropped without side effects\n",
963 pfn);
964 }
965 }
966
967 /*
968 * First collect all the processes that have the page
969 * mapped in dirty form. This has to be done before try_to_unmap,
970 * because ttu takes the rmap data structures down.
971 *
972 * Error handling: We ignore errors here because
973 * there's nothing that can be done.
974 */
975 if (kill)
976 collect_procs(hpage, &tokill, flags & MF_ACTION_REQUIRED);
977
978 unmap_success = try_to_unmap(hpage, ttu);
979 if (!unmap_success)
980 pr_err("Memory failure: %#lx: failed to unmap page (mapcount=%d)\n",
981 pfn, page_mapcount(hpage));
982
983 /*
984 * try_to_unmap() might put mlocked page in lru cache, so call
985 * shake_page() again to ensure that it's flushed.
986 */
987 if (mlocked)
988 shake_page(hpage, 0);
989
990 /*
991 * Now that the dirty bit has been propagated to the
992 * struct page and all unmaps done we can decide if
993 * killing is needed or not. Only kill when the page
994 * was dirty or the process is not restartable,
995 * otherwise the tokill list is merely
996 * freed. When there was a problem unmapping earlier
997 * use a more force-full uncatchable kill to prevent
998 * any accesses to the poisoned memory.
999 */
1000 forcekill = PageDirty(hpage) || (flags & MF_MUST_KILL);
1001 kill_procs(&tokill, forcekill, trapno, !unmap_success, p, pfn, flags);
1002
1003 return unmap_success;
1004 }
1005
1006 static void set_page_hwpoison_huge_page(struct page *hpage)
1007 {
1008 int i;
1009 int nr_pages = 1 << compound_order(hpage);
1010 for (i = 0; i < nr_pages; i++)
1011 SetPageHWPoison(hpage + i);
1012 }
1013
1014 static void clear_page_hwpoison_huge_page(struct page *hpage)
1015 {
1016 int i;
1017 int nr_pages = 1 << compound_order(hpage);
1018 for (i = 0; i < nr_pages; i++)
1019 ClearPageHWPoison(hpage + i);
1020 }
1021
1022 /**
1023 * memory_failure - Handle memory failure of a page.
1024 * @pfn: Page Number of the corrupted page
1025 * @trapno: Trap number reported in the signal to user space.
1026 * @flags: fine tune action taken
1027 *
1028 * This function is called by the low level machine check code
1029 * of an architecture when it detects hardware memory corruption
1030 * of a page. It tries its best to recover, which includes
1031 * dropping pages, killing processes etc.
1032 *
1033 * The function is primarily of use for corruptions that
1034 * happen outside the current execution context (e.g. when
1035 * detected by a background scrubber)
1036 *
1037 * Must run in process context (e.g. a work queue) with interrupts
1038 * enabled and no spinlocks hold.
1039 */
1040 int memory_failure(unsigned long pfn, int trapno, int flags)
1041 {
1042 struct page_state *ps;
1043 struct page *p;
1044 struct page *hpage;
1045 struct page *orig_head;
1046 int res;
1047 unsigned int nr_pages;
1048 unsigned long page_flags;
1049
1050 if (!sysctl_memory_failure_recovery)
1051 panic("Memory failure from trap %d on page %lx", trapno, pfn);
1052
1053 if (!pfn_valid(pfn)) {
1054 pr_err("Memory failure: %#lx: memory outside kernel control\n",
1055 pfn);
1056 return -ENXIO;
1057 }
1058
1059 p = pfn_to_page(pfn);
1060 orig_head = hpage = compound_head(p);
1061 if (TestSetPageHWPoison(p)) {
1062 pr_err("Memory failure: %#lx: already hardware poisoned\n",
1063 pfn);
1064 return 0;
1065 }
1066
1067 /*
1068 * Currently errors on hugetlbfs pages are measured in hugepage units,
1069 * so nr_pages should be 1 << compound_order. OTOH when errors are on
1070 * transparent hugepages, they are supposed to be split and error
1071 * measurement is done in normal page units. So nr_pages should be one
1072 * in this case.
1073 */
1074 if (PageHuge(p))
1075 nr_pages = 1 << compound_order(hpage);
1076 else /* normal page or thp */
1077 nr_pages = 1;
1078 num_poisoned_pages_add(nr_pages);
1079
1080 /*
1081 * We need/can do nothing about count=0 pages.
1082 * 1) it's a free page, and therefore in safe hand:
1083 * prep_new_page() will be the gate keeper.
1084 * 2) it's a free hugepage, which is also safe:
1085 * an affected hugepage will be dequeued from hugepage freelist,
1086 * so there's no concern about reusing it ever after.
1087 * 3) it's part of a non-compound high order page.
1088 * Implies some kernel user: cannot stop them from
1089 * R/W the page; let's pray that the page has been
1090 * used and will be freed some time later.
1091 * In fact it's dangerous to directly bump up page count from 0,
1092 * that may make page_freeze_refs()/page_unfreeze_refs() mismatch.
1093 */
1094 if (!(flags & MF_COUNT_INCREASED) && !get_hwpoison_page(p)) {
1095 if (is_free_buddy_page(p)) {
1096 action_result(pfn, MF_MSG_BUDDY, MF_DELAYED);
1097 return 0;
1098 } else if (PageHuge(hpage)) {
1099 /*
1100 * Check "filter hit" and "race with other subpage."
1101 */
1102 lock_page(hpage);
1103 if (PageHWPoison(hpage)) {
1104 if ((hwpoison_filter(p) && TestClearPageHWPoison(p))
1105 || (p != hpage && TestSetPageHWPoison(hpage))) {
1106 num_poisoned_pages_sub(nr_pages);
1107 unlock_page(hpage);
1108 return 0;
1109 }
1110 }
1111 set_page_hwpoison_huge_page(hpage);
1112 res = dequeue_hwpoisoned_huge_page(hpage);
1113 action_result(pfn, MF_MSG_FREE_HUGE,
1114 res ? MF_IGNORED : MF_DELAYED);
1115 unlock_page(hpage);
1116 return res;
1117 } else {
1118 action_result(pfn, MF_MSG_KERNEL_HIGH_ORDER, MF_IGNORED);
1119 return -EBUSY;
1120 }
1121 }
1122
1123 if (!PageHuge(p) && PageTransHuge(hpage)) {
1124 lock_page(p);
1125 if (!PageAnon(p) || unlikely(split_huge_page(p))) {
1126 unlock_page(p);
1127 if (!PageAnon(p))
1128 pr_err("Memory failure: %#lx: non anonymous thp\n",
1129 pfn);
1130 else
1131 pr_err("Memory failure: %#lx: thp split failed\n",
1132 pfn);
1133 if (TestClearPageHWPoison(p))
1134 num_poisoned_pages_sub(nr_pages);
1135 put_hwpoison_page(p);
1136 return -EBUSY;
1137 }
1138 unlock_page(p);
1139 VM_BUG_ON_PAGE(!page_count(p), p);
1140 hpage = compound_head(p);
1141 }
1142
1143 /*
1144 * We ignore non-LRU pages for good reasons.
1145 * - PG_locked is only well defined for LRU pages and a few others
1146 * - to avoid races with __SetPageLocked()
1147 * - to avoid races with __SetPageSlab*() (and more non-atomic ops)
1148 * The check (unnecessarily) ignores LRU pages being isolated and
1149 * walked by the page reclaim code, however that's not a big loss.
1150 */
1151 shake_page(p, 0);
1152 /* shake_page could have turned it free. */
1153 if (!PageLRU(p) && is_free_buddy_page(p)) {
1154 if (flags & MF_COUNT_INCREASED)
1155 action_result(pfn, MF_MSG_BUDDY, MF_DELAYED);
1156 else
1157 action_result(pfn, MF_MSG_BUDDY_2ND, MF_DELAYED);
1158 return 0;
1159 }
1160
1161 lock_page(hpage);
1162
1163 /*
1164 * The page could have changed compound pages during the locking.
1165 * If this happens just bail out.
1166 */
1167 if (PageCompound(p) && compound_head(p) != orig_head) {
1168 action_result(pfn, MF_MSG_DIFFERENT_COMPOUND, MF_IGNORED);
1169 res = -EBUSY;
1170 goto out;
1171 }
1172
1173 /*
1174 * We use page flags to determine what action should be taken, but
1175 * the flags can be modified by the error containment action. One
1176 * example is an mlocked page, where PG_mlocked is cleared by
1177 * page_remove_rmap() in try_to_unmap_one(). So to determine page status
1178 * correctly, we save a copy of the page flags at this time.
1179 */
1180 page_flags = p->flags;
1181
1182 /*
1183 * unpoison always clear PG_hwpoison inside page lock
1184 */
1185 if (!PageHWPoison(p)) {
1186 pr_err("Memory failure: %#lx: just unpoisoned\n", pfn);
1187 num_poisoned_pages_sub(nr_pages);
1188 unlock_page(hpage);
1189 put_hwpoison_page(hpage);
1190 return 0;
1191 }
1192 if (hwpoison_filter(p)) {
1193 if (TestClearPageHWPoison(p))
1194 num_poisoned_pages_sub(nr_pages);
1195 unlock_page(hpage);
1196 put_hwpoison_page(hpage);
1197 return 0;
1198 }
1199
1200 if (!PageHuge(p) && !PageTransTail(p) && !PageLRU(p))
1201 goto identify_page_state;
1202
1203 /*
1204 * For error on the tail page, we should set PG_hwpoison
1205 * on the head page to show that the hugepage is hwpoisoned
1206 */
1207 if (PageHuge(p) && PageTail(p) && TestSetPageHWPoison(hpage)) {
1208 action_result(pfn, MF_MSG_POISONED_HUGE, MF_IGNORED);
1209 unlock_page(hpage);
1210 put_hwpoison_page(hpage);
1211 return 0;
1212 }
1213 /*
1214 * Set PG_hwpoison on all pages in an error hugepage,
1215 * because containment is done in hugepage unit for now.
1216 * Since we have done TestSetPageHWPoison() for the head page with
1217 * page lock held, we can safely set PG_hwpoison bits on tail pages.
1218 */
1219 if (PageHuge(p))
1220 set_page_hwpoison_huge_page(hpage);
1221
1222 /*
1223 * It's very difficult to mess with pages currently under IO
1224 * and in many cases impossible, so we just avoid it here.
1225 */
1226 wait_on_page_writeback(p);
1227
1228 /*
1229 * Now take care of user space mappings.
1230 * Abort on fail: __delete_from_page_cache() assumes unmapped page.
1231 *
1232 * When the raw error page is thp tail page, hpage points to the raw
1233 * page after thp split.
1234 */
1235 if (!hwpoison_user_mappings(p, pfn, trapno, flags, &hpage)) {
1236 action_result(pfn, MF_MSG_UNMAP_FAILED, MF_IGNORED);
1237 res = -EBUSY;
1238 goto out;
1239 }
1240
1241 /*
1242 * Torn down by someone else?
1243 */
1244 if (PageLRU(p) && !PageSwapCache(p) && p->mapping == NULL) {
1245 action_result(pfn, MF_MSG_TRUNCATED_LRU, MF_IGNORED);
1246 res = -EBUSY;
1247 goto out;
1248 }
1249
1250 identify_page_state:
1251 res = -EBUSY;
1252 /*
1253 * The first check uses the current page flags which may not have any
1254 * relevant information. The second check with the saved page flagss is
1255 * carried out only if the first check can't determine the page status.
1256 */
1257 for (ps = error_states;; ps++)
1258 if ((p->flags & ps->mask) == ps->res)
1259 break;
1260
1261 page_flags |= (p->flags & (1UL << PG_dirty));
1262
1263 if (!ps->mask)
1264 for (ps = error_states;; ps++)
1265 if ((page_flags & ps->mask) == ps->res)
1266 break;
1267 res = page_action(ps, p, pfn);
1268 out:
1269 unlock_page(hpage);
1270 return res;
1271 }
1272 EXPORT_SYMBOL_GPL(memory_failure);
1273
1274 #define MEMORY_FAILURE_FIFO_ORDER 4
1275 #define MEMORY_FAILURE_FIFO_SIZE (1 << MEMORY_FAILURE_FIFO_ORDER)
1276
1277 struct memory_failure_entry {
1278 unsigned long pfn;
1279 int trapno;
1280 int flags;
1281 };
1282
1283 struct memory_failure_cpu {
1284 DECLARE_KFIFO(fifo, struct memory_failure_entry,
1285 MEMORY_FAILURE_FIFO_SIZE);
1286 spinlock_t lock;
1287 struct work_struct work;
1288 };
1289
1290 static DEFINE_PER_CPU(struct memory_failure_cpu, memory_failure_cpu);
1291
1292 /**
1293 * memory_failure_queue - Schedule handling memory failure of a page.
1294 * @pfn: Page Number of the corrupted page
1295 * @trapno: Trap number reported in the signal to user space.
1296 * @flags: Flags for memory failure handling
1297 *
1298 * This function is called by the low level hardware error handler
1299 * when it detects hardware memory corruption of a page. It schedules
1300 * the recovering of error page, including dropping pages, killing
1301 * processes etc.
1302 *
1303 * The function is primarily of use for corruptions that
1304 * happen outside the current execution context (e.g. when
1305 * detected by a background scrubber)
1306 *
1307 * Can run in IRQ context.
1308 */
1309 void memory_failure_queue(unsigned long pfn, int trapno, int flags)
1310 {
1311 struct memory_failure_cpu *mf_cpu;
1312 unsigned long proc_flags;
1313 struct memory_failure_entry entry = {
1314 .pfn = pfn,
1315 .trapno = trapno,
1316 .flags = flags,
1317 };
1318
1319 mf_cpu = &get_cpu_var(memory_failure_cpu);
1320 spin_lock_irqsave(&mf_cpu->lock, proc_flags);
1321 if (kfifo_put(&mf_cpu->fifo, entry))
1322 schedule_work_on(smp_processor_id(), &mf_cpu->work);
1323 else
1324 pr_err("Memory failure: buffer overflow when queuing memory failure at %#lx\n",
1325 pfn);
1326 spin_unlock_irqrestore(&mf_cpu->lock, proc_flags);
1327 put_cpu_var(memory_failure_cpu);
1328 }
1329 EXPORT_SYMBOL_GPL(memory_failure_queue);
1330
1331 static void memory_failure_work_func(struct work_struct *work)
1332 {
1333 struct memory_failure_cpu *mf_cpu;
1334 struct memory_failure_entry entry = { 0, };
1335 unsigned long proc_flags;
1336 int gotten;
1337
1338 mf_cpu = this_cpu_ptr(&memory_failure_cpu);
1339 for (;;) {
1340 spin_lock_irqsave(&mf_cpu->lock, proc_flags);
1341 gotten = kfifo_get(&mf_cpu->fifo, &entry);
1342 spin_unlock_irqrestore(&mf_cpu->lock, proc_flags);
1343 if (!gotten)
1344 break;
1345 if (entry.flags & MF_SOFT_OFFLINE)
1346 soft_offline_page(pfn_to_page(entry.pfn), entry.flags);
1347 else
1348 memory_failure(entry.pfn, entry.trapno, entry.flags);
1349 }
1350 }
1351
1352 static int __init memory_failure_init(void)
1353 {
1354 struct memory_failure_cpu *mf_cpu;
1355 int cpu;
1356
1357 for_each_possible_cpu(cpu) {
1358 mf_cpu = &per_cpu(memory_failure_cpu, cpu);
1359 spin_lock_init(&mf_cpu->lock);
1360 INIT_KFIFO(mf_cpu->fifo);
1361 INIT_WORK(&mf_cpu->work, memory_failure_work_func);
1362 }
1363
1364 return 0;
1365 }
1366 core_initcall(memory_failure_init);
1367
1368 #define unpoison_pr_info(fmt, pfn, rs) \
1369 ({ \
1370 if (__ratelimit(rs)) \
1371 pr_info(fmt, pfn); \
1372 })
1373
1374 /**
1375 * unpoison_memory - Unpoison a previously poisoned page
1376 * @pfn: Page number of the to be unpoisoned page
1377 *
1378 * Software-unpoison a page that has been poisoned by
1379 * memory_failure() earlier.
1380 *
1381 * This is only done on the software-level, so it only works
1382 * for linux injected failures, not real hardware failures
1383 *
1384 * Returns 0 for success, otherwise -errno.
1385 */
1386 int unpoison_memory(unsigned long pfn)
1387 {
1388 struct page *page;
1389 struct page *p;
1390 int freeit = 0;
1391 unsigned int nr_pages;
1392 static DEFINE_RATELIMIT_STATE(unpoison_rs, DEFAULT_RATELIMIT_INTERVAL,
1393 DEFAULT_RATELIMIT_BURST);
1394
1395 if (!pfn_valid(pfn))
1396 return -ENXIO;
1397
1398 p = pfn_to_page(pfn);
1399 page = compound_head(p);
1400
1401 if (!PageHWPoison(p)) {
1402 unpoison_pr_info("Unpoison: Page was already unpoisoned %#lx\n",
1403 pfn, &unpoison_rs);
1404 return 0;
1405 }
1406
1407 if (page_count(page) > 1) {
1408 unpoison_pr_info("Unpoison: Someone grabs the hwpoison page %#lx\n",
1409 pfn, &unpoison_rs);
1410 return 0;
1411 }
1412
1413 if (page_mapped(page)) {
1414 unpoison_pr_info("Unpoison: Someone maps the hwpoison page %#lx\n",
1415 pfn, &unpoison_rs);
1416 return 0;
1417 }
1418
1419 if (page_mapping(page)) {
1420 unpoison_pr_info("Unpoison: the hwpoison page has non-NULL mapping %#lx\n",
1421 pfn, &unpoison_rs);
1422 return 0;
1423 }
1424
1425 /*
1426 * unpoison_memory() can encounter thp only when the thp is being
1427 * worked by memory_failure() and the page lock is not held yet.
1428 * In such case, we yield to memory_failure() and make unpoison fail.
1429 */
1430 if (!PageHuge(page) && PageTransHuge(page)) {
1431 unpoison_pr_info("Unpoison: Memory failure is now running on %#lx\n",
1432 pfn, &unpoison_rs);
1433 return 0;
1434 }
1435
1436 nr_pages = 1 << compound_order(page);
1437
1438 if (!get_hwpoison_page(p)) {
1439 /*
1440 * Since HWPoisoned hugepage should have non-zero refcount,
1441 * race between memory failure and unpoison seems to happen.
1442 * In such case unpoison fails and memory failure runs
1443 * to the end.
1444 */
1445 if (PageHuge(page)) {
1446 unpoison_pr_info("Unpoison: Memory failure is now running on free hugepage %#lx\n",
1447 pfn, &unpoison_rs);
1448 return 0;
1449 }
1450 if (TestClearPageHWPoison(p))
1451 num_poisoned_pages_dec();
1452 unpoison_pr_info("Unpoison: Software-unpoisoned free page %#lx\n",
1453 pfn, &unpoison_rs);
1454 return 0;
1455 }
1456
1457 lock_page(page);
1458 /*
1459 * This test is racy because PG_hwpoison is set outside of page lock.
1460 * That's acceptable because that won't trigger kernel panic. Instead,
1461 * the PG_hwpoison page will be caught and isolated on the entrance to
1462 * the free buddy page pool.
1463 */
1464 if (TestClearPageHWPoison(page)) {
1465 unpoison_pr_info("Unpoison: Software-unpoisoned page %#lx\n",
1466 pfn, &unpoison_rs);
1467 num_poisoned_pages_sub(nr_pages);
1468 freeit = 1;
1469 if (PageHuge(page))
1470 clear_page_hwpoison_huge_page(page);
1471 }
1472 unlock_page(page);
1473
1474 put_hwpoison_page(page);
1475 if (freeit && !(pfn == my_zero_pfn(0) && page_count(p) == 1))
1476 put_hwpoison_page(page);
1477
1478 return 0;
1479 }
1480 EXPORT_SYMBOL(unpoison_memory);
1481
1482 static struct page *new_page(struct page *p, unsigned long private, int **x)
1483 {
1484 int nid = page_to_nid(p);
1485 if (PageHuge(p))
1486 return alloc_huge_page_node(page_hstate(compound_head(p)),
1487 nid);
1488 else
1489 return __alloc_pages_node(nid, GFP_HIGHUSER_MOVABLE, 0);
1490 }
1491
1492 /*
1493 * Safely get reference count of an arbitrary page.
1494 * Returns 0 for a free page, -EIO for a zero refcount page
1495 * that is not free, and 1 for any other page type.
1496 * For 1 the page is returned with increased page count, otherwise not.
1497 */
1498 static int __get_any_page(struct page *p, unsigned long pfn, int flags)
1499 {
1500 int ret;
1501
1502 if (flags & MF_COUNT_INCREASED)
1503 return 1;
1504
1505 /*
1506 * When the target page is a free hugepage, just remove it
1507 * from free hugepage list.
1508 */
1509 if (!get_hwpoison_page(p)) {
1510 if (PageHuge(p)) {
1511 pr_info("%s: %#lx free huge page\n", __func__, pfn);
1512 ret = 0;
1513 } else if (is_free_buddy_page(p)) {
1514 pr_info("%s: %#lx free buddy page\n", __func__, pfn);
1515 ret = 0;
1516 } else {
1517 pr_info("%s: %#lx: unknown zero refcount page type %lx\n",
1518 __func__, pfn, p->flags);
1519 ret = -EIO;
1520 }
1521 } else {
1522 /* Not a free page */
1523 ret = 1;
1524 }
1525 return ret;
1526 }
1527
1528 static int get_any_page(struct page *page, unsigned long pfn, int flags)
1529 {
1530 int ret = __get_any_page(page, pfn, flags);
1531
1532 if (ret == 1 && !PageHuge(page) &&
1533 !PageLRU(page) && !__PageMovable(page)) {
1534 /*
1535 * Try to free it.
1536 */
1537 put_hwpoison_page(page);
1538 shake_page(page, 1);
1539
1540 /*
1541 * Did it turn free?
1542 */
1543 ret = __get_any_page(page, pfn, 0);
1544 if (ret == 1 && !PageLRU(page)) {
1545 /* Drop page reference which is from __get_any_page() */
1546 put_hwpoison_page(page);
1547 pr_info("soft_offline: %#lx: unknown non LRU page type %lx (%pGp)\n",
1548 pfn, page->flags, &page->flags);
1549 return -EIO;
1550 }
1551 }
1552 return ret;
1553 }
1554
1555 static int soft_offline_huge_page(struct page *page, int flags)
1556 {
1557 int ret;
1558 unsigned long pfn = page_to_pfn(page);
1559 struct page *hpage = compound_head(page);
1560 LIST_HEAD(pagelist);
1561
1562 /*
1563 * This double-check of PageHWPoison is to avoid the race with
1564 * memory_failure(). See also comment in __soft_offline_page().
1565 */
1566 lock_page(hpage);
1567 if (PageHWPoison(hpage)) {
1568 unlock_page(hpage);
1569 put_hwpoison_page(hpage);
1570 pr_info("soft offline: %#lx hugepage already poisoned\n", pfn);
1571 return -EBUSY;
1572 }
1573 unlock_page(hpage);
1574
1575 ret = isolate_huge_page(hpage, &pagelist);
1576 /*
1577 * get_any_page() and isolate_huge_page() takes a refcount each,
1578 * so need to drop one here.
1579 */
1580 put_hwpoison_page(hpage);
1581 if (!ret) {
1582 pr_info("soft offline: %#lx hugepage failed to isolate\n", pfn);
1583 return -EBUSY;
1584 }
1585
1586 ret = migrate_pages(&pagelist, new_page, NULL, MPOL_MF_MOVE_ALL,
1587 MIGRATE_SYNC, MR_MEMORY_FAILURE);
1588 if (ret) {
1589 pr_info("soft offline: %#lx: migration failed %d, type %lx (%pGp)\n",
1590 pfn, ret, page->flags, &page->flags);
1591 /*
1592 * We know that soft_offline_huge_page() tries to migrate
1593 * only one hugepage pointed to by hpage, so we need not
1594 * run through the pagelist here.
1595 */
1596 putback_active_hugepage(hpage);
1597 if (ret > 0)
1598 ret = -EIO;
1599 } else {
1600 /* overcommit hugetlb page will be freed to buddy */
1601 if (PageHuge(page)) {
1602 set_page_hwpoison_huge_page(hpage);
1603 dequeue_hwpoisoned_huge_page(hpage);
1604 num_poisoned_pages_add(1 << compound_order(hpage));
1605 } else {
1606 SetPageHWPoison(page);
1607 num_poisoned_pages_inc();
1608 }
1609 }
1610 return ret;
1611 }
1612
1613 static int __soft_offline_page(struct page *page, int flags)
1614 {
1615 int ret;
1616 unsigned long pfn = page_to_pfn(page);
1617
1618 /*
1619 * Check PageHWPoison again inside page lock because PageHWPoison
1620 * is set by memory_failure() outside page lock. Note that
1621 * memory_failure() also double-checks PageHWPoison inside page lock,
1622 * so there's no race between soft_offline_page() and memory_failure().
1623 */
1624 lock_page(page);
1625 wait_on_page_writeback(page);
1626 if (PageHWPoison(page)) {
1627 unlock_page(page);
1628 put_hwpoison_page(page);
1629 pr_info("soft offline: %#lx page already poisoned\n", pfn);
1630 return -EBUSY;
1631 }
1632 /*
1633 * Try to invalidate first. This should work for
1634 * non dirty unmapped page cache pages.
1635 */
1636 ret = invalidate_inode_page(page);
1637 unlock_page(page);
1638 /*
1639 * RED-PEN would be better to keep it isolated here, but we
1640 * would need to fix isolation locking first.
1641 */
1642 if (ret == 1) {
1643 put_hwpoison_page(page);
1644 pr_info("soft_offline: %#lx: invalidated\n", pfn);
1645 SetPageHWPoison(page);
1646 num_poisoned_pages_inc();
1647 return 0;
1648 }
1649
1650 /*
1651 * Simple invalidation didn't work.
1652 * Try to migrate to a new page instead. migrate.c
1653 * handles a large number of cases for us.
1654 */
1655 if (PageLRU(page))
1656 ret = isolate_lru_page(page);
1657 else
1658 ret = isolate_movable_page(page, ISOLATE_UNEVICTABLE);
1659 /*
1660 * Drop page reference which is came from get_any_page()
1661 * successful isolate_lru_page() already took another one.
1662 */
1663 put_hwpoison_page(page);
1664 if (!ret) {
1665 LIST_HEAD(pagelist);
1666 /*
1667 * After isolated lru page, the PageLRU will be cleared,
1668 * so use !__PageMovable instead for LRU page's mapping
1669 * cannot have PAGE_MAPPING_MOVABLE.
1670 */
1671 if (!__PageMovable(page))
1672 inc_node_page_state(page, NR_ISOLATED_ANON +
1673 page_is_file_cache(page));
1674 list_add(&page->lru, &pagelist);
1675 ret = migrate_pages(&pagelist, new_page, NULL, MPOL_MF_MOVE_ALL,
1676 MIGRATE_SYNC, MR_MEMORY_FAILURE);
1677 if (ret) {
1678 if (!list_empty(&pagelist))
1679 putback_movable_pages(&pagelist);
1680
1681 pr_info("soft offline: %#lx: migration failed %d, type %lx (%pGp)\n",
1682 pfn, ret, page->flags, &page->flags);
1683 if (ret > 0)
1684 ret = -EIO;
1685 }
1686 } else {
1687 pr_info("soft offline: %#lx: isolation failed: %d, page count %d, type %lx (%pGp)\n",
1688 pfn, ret, page_count(page), page->flags, &page->flags);
1689 }
1690 return ret;
1691 }
1692
1693 static int soft_offline_in_use_page(struct page *page, int flags)
1694 {
1695 int ret;
1696 struct page *hpage = compound_head(page);
1697
1698 if (!PageHuge(page) && PageTransHuge(hpage)) {
1699 lock_page(hpage);
1700 if (!PageAnon(hpage) || unlikely(split_huge_page(hpage))) {
1701 unlock_page(hpage);
1702 if (!PageAnon(hpage))
1703 pr_info("soft offline: %#lx: non anonymous thp\n", page_to_pfn(page));
1704 else
1705 pr_info("soft offline: %#lx: thp split failed\n", page_to_pfn(page));
1706 put_hwpoison_page(hpage);
1707 return -EBUSY;
1708 }
1709 unlock_page(hpage);
1710 get_hwpoison_page(page);
1711 put_hwpoison_page(hpage);
1712 }
1713
1714 if (PageHuge(page))
1715 ret = soft_offline_huge_page(page, flags);
1716 else
1717 ret = __soft_offline_page(page, flags);
1718
1719 return ret;
1720 }
1721
1722 static void soft_offline_free_page(struct page *page)
1723 {
1724 if (PageHuge(page)) {
1725 struct page *hpage = compound_head(page);
1726
1727 set_page_hwpoison_huge_page(hpage);
1728 if (!dequeue_hwpoisoned_huge_page(hpage))
1729 num_poisoned_pages_add(1 << compound_order(hpage));
1730 } else {
1731 if (!TestSetPageHWPoison(page))
1732 num_poisoned_pages_inc();
1733 }
1734 }
1735
1736 /**
1737 * soft_offline_page - Soft offline a page.
1738 * @page: page to offline
1739 * @flags: flags. Same as memory_failure().
1740 *
1741 * Returns 0 on success, otherwise negated errno.
1742 *
1743 * Soft offline a page, by migration or invalidation,
1744 * without killing anything. This is for the case when
1745 * a page is not corrupted yet (so it's still valid to access),
1746 * but has had a number of corrected errors and is better taken
1747 * out.
1748 *
1749 * The actual policy on when to do that is maintained by
1750 * user space.
1751 *
1752 * This should never impact any application or cause data loss,
1753 * however it might take some time.
1754 *
1755 * This is not a 100% solution for all memory, but tries to be
1756 * ``good enough'' for the majority of memory.
1757 */
1758 int soft_offline_page(struct page *page, int flags)
1759 {
1760 int ret;
1761 unsigned long pfn = page_to_pfn(page);
1762
1763 if (PageHWPoison(page)) {
1764 pr_info("soft offline: %#lx page already poisoned\n", pfn);
1765 if (flags & MF_COUNT_INCREASED)
1766 put_hwpoison_page(page);
1767 return -EBUSY;
1768 }
1769
1770 get_online_mems();
1771 ret = get_any_page(page, pfn, flags);
1772 put_online_mems();
1773
1774 if (ret > 0)
1775 ret = soft_offline_in_use_page(page, flags);
1776 else if (ret == 0)
1777 soft_offline_free_page(page);
1778
1779 return ret;
1780 }