mm/fault, arch: Use pagefault_disable() to check for disabled pagefaults in the handler

Introduce faulthandler_disabled() and use it to check for irq context and
disabled pagefaults (via pagefault_disable()) in the pagefault handlers.

Please note that we keep the in_atomic() checks in place - to detect
whether in irq context (in which case preemption is always properly
disabled).

In contrast, preempt_disable() should never be used to disable pagefaults.
With !CONFIG_PREEMPT_COUNT, preempt_disable() doesn't modify the preempt
counter, and therefore the result of in_atomic() differs.
We validate that condition by using might_fault() checks when calling
might_sleep().

Therefore, add a comment to faulthandler_disabled(), describing why this
is needed.

faulthandler_disabled() and pagefault_disable() are defined in
linux/uaccess.h, so let's properly add that include to all relevant files.

This patch is based on a patch from Thomas Gleixner.

Reviewed-and-tested-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: David Hildenbrand <dahi@linux.vnet.ibm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: David.Laight@ACULAB.COM
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: airlied@linux.ie
Cc: akpm@linux-foundation.org
Cc: benh@kernel.crashing.org
Cc: bigeasy@linutronix.de
Cc: borntraeger@de.ibm.com
Cc: daniel.vetter@intel.com
Cc: heiko.carstens@de.ibm.com
Cc: herbert@gondor.apana.org.au
Cc: hocko@suse.cz
Cc: hughd@google.com
Cc: mst@redhat.com
Cc: paulus@samba.org
Cc: ralf@linux-mips.org
Cc: schwidefsky@de.ibm.com
Cc: yang.shi@windriver.com
Link: http://lkml.kernel.org/r/1431359540-32227-7-git-send-email-dahi@linux.vnet.ibm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/arch/alpha/mm/fault.c b/arch/alpha/mm/fault.c
index 9d0ac091a52a7d16cf1f78f402ab48c511924a24..4a905bd667e2ef71542e2585469404478860bce5 100644 (file)
--- a/arch/alpha/mm/fault.c
+++ b/arch/alpha/mm/fault.c
@@ -23,8 +23,7 @@
 #include <linux/smp.h>
 #include <linux/interrupt.h>
 #include <linux/module.h>
-
-#include <asm/uaccess.h>
+#include <linux/uaccess.h>
 
 extern void die_if_kernel(char *,struct pt_regs *,long, unsigned long *);
 
@@ -107,7 +106,7 @@ do_page_fault(unsigned long address, unsigned long mmcsr,
 
        /* If we're in an interrupt context, or have no user context,
           we must not take the fault.  */
-       if (!mm || in_atomic())
+       if (!mm || faulthandler_disabled())
                goto no_context;
 
 #ifdef CONFIG_ALPHA_LARGE_VMALLOC

diff --git a/arch/arc/mm/fault.c b/arch/arc/mm/fault.c
index 6a2e006cbcce1f1cd69866e0f0f9f94463d73dcb..d948e4e9d89c4ebe7e5676f449c9377b4fbe3535 100644 (file)
--- a/arch/arc/mm/fault.c
+++ b/arch/arc/mm/fault.c
@@ -86,7 +86,7 @@ void do_page_fault(unsigned long address, struct pt_regs *regs)
         * If we're in an interrupt or have no user
         * context, we must not take the fault..
         */
-       if (in_atomic() || !mm)
+       if (faulthandler_disabled() || !mm)
                goto no_context;
 
        if (user_mode(regs))

diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c
index 6333d9c178757fe4f365b8e765b2a9ea75e2b80b..0d629b8f973fc2ca63aacb59e5baaf718b194543 100644 (file)
--- a/arch/arm/mm/fault.c
+++ b/arch/arm/mm/fault.c
@@ -276,7 +276,7 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
         * If we're in an interrupt or have no user
         * context, we must not take the fault..
         */
-       if (in_atomic() || !mm)
+       if (faulthandler_disabled() || !mm)
                goto no_context;
 
        if (user_mode(regs))

diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
index 96da13167d4a5c77564952a6d3a0fdce35d6580f..0948d327d013651c7b9978023139ea9cd89ecaeb 100644 (file)
--- a/arch/arm64/mm/fault.c
+++ b/arch/arm64/mm/fault.c
@@ -211,7 +211,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
         * If we're in an interrupt or have no user context, we must not take
         * the fault.
         */
-       if (in_atomic() || !mm)
+       if (faulthandler_disabled() || !mm)
                goto no_context;
 
        if (user_mode(regs))

diff --git a/arch/avr32/mm/fault.c b/arch/avr32/mm/fault.c
index d223a8b57c1eaad282289e75089654153ab598d6..c03533937a9f0aa273a75c76ecb66f2731b2d39c 100644 (file)
--- a/arch/avr32/mm/fault.c
+++ b/arch/avr32/mm/fault.c
@@ -14,11 +14,11 @@
 #include <linux/pagemap.h>
 #include <linux/kdebug.h>
 #include <linux/kprobes.h>
+#include <linux/uaccess.h>
 
 #include <asm/mmu_context.h>
 #include <asm/sysreg.h>
 #include <asm/tlb.h>
-#include <asm/uaccess.h>
 
 #ifdef CONFIG_KPROBES
 static inline int notify_page_fault(struct pt_regs *regs, int trap)
@@ -81,7 +81,7 @@ asmlinkage void do_page_fault(unsigned long ecr, struct pt_regs *regs)
         * If we're in an interrupt or have no user context, we must
         * not take the fault...
         */
-       if (in_atomic() || !mm || regs->sr & SYSREG_BIT(GM))
+       if (faulthandler_disabled() || !mm || regs->sr & SYSREG_BIT(GM))
                goto no_context;
 
        local_irq_enable();

diff --git a/arch/cris/mm/fault.c b/arch/cris/mm/fault.c
index 83f12f2ed9e31b8705ed4f8f5ec1c4cbd8822fb4..3066d40a6db14425c162d399d89e5c6db66786fe 100644 (file)
--- a/arch/cris/mm/fault.c
+++ b/arch/cris/mm/fault.c
@@ -8,7 +8,7 @@
 #include <linux/interrupt.h>
 #include <linux/module.h>
 #include <linux/wait.h>
-#include <asm/uaccess.h>
+#include <linux/uaccess.h>
 #include <arch/system.h>
 
 extern int find_fixup_code(struct pt_regs *);
@@ -109,11 +109,11 @@ do_page_fault(unsigned long address, struct pt_regs *regs,
        info.si_code = SEGV_MAPERR;
 
        /*
-        * If we're in an interrupt or "atomic" operation or have no
+        * If we're in an interrupt, have pagefaults disabled or have no
         * user context, we must not take the fault.
         */
 
-       if (in_atomic() || !mm)
+       if (faulthandler_disabled() || !mm)
                goto no_context;
 
        if (user_mode(regs))

diff --git a/arch/frv/mm/fault.c b/arch/frv/mm/fault.c
index ec4917ddf67872aa46b60c6b067b0a67ec5417a4..61d99767fe1691e70286bf8d52a05aee506bbccc 100644 (file)
--- a/arch/frv/mm/fault.c
+++ b/arch/frv/mm/fault.c
@@ -19,9 +19,9 @@
 #include <linux/kernel.h>
 #include <linux/ptrace.h>
 #include <linux/hardirq.h>
+#include <linux/uaccess.h>
 
 #include <asm/pgtable.h>
-#include <asm/uaccess.h>
 #include <asm/gdb-stub.h>
 
 /*****************************************************************************/
@@ -78,7 +78,7 @@ asmlinkage void do_page_fault(int datammu, unsigned long esr0, unsigned long ear
         * If we're in an interrupt or have no user
         * context, we must not take the fault..
         */
-       if (in_atomic() || !mm)
+       if (faulthandler_disabled() || !mm)
                goto no_context;
 
        if (user_mode(__frame))

diff --git a/arch/ia64/mm/fault.c b/arch/ia64/mm/fault.c
index ba5ba7accd0d6bb4dbab34f7fc307c4306347f4a..70b40d1205a6b9b3ec7efcbc9e60ec64c2eff712 100644 (file)
--- a/arch/ia64/mm/fault.c
+++ b/arch/ia64/mm/fault.c
@@ -11,10 +11,10 @@
 #include <linux/kprobes.h>
 #include <linux/kdebug.h>
 #include <linux/prefetch.h>
+#include <linux/uaccess.h>
 
 #include <asm/pgtable.h>
 #include <asm/processor.h>
-#include <asm/uaccess.h>
 
 extern int die(char *, struct pt_regs *, long);
 
@@ -96,7 +96,7 @@ ia64_do_page_fault (unsigned long address, unsigned long isr, struct pt_regs *re
        /*
         * If we're in an interrupt or have no user context, we must not take the fault..
         */
-       if (in_atomic() || !mm)
+       if (faulthandler_disabled() || !mm)
                goto no_context;
 
 #ifdef CONFIG_VIRTUAL_MEM_MAP

diff --git a/arch/m32r/mm/fault.c b/arch/m32r/mm/fault.c
index e3d4d4890104cc27e2eb9de2f22cb6f53f939c90..8f9875b7933d5582277a777693a2e44c1483d362 100644 (file)
--- a/arch/m32r/mm/fault.c
+++ b/arch/m32r/mm/fault.c
@@ -24,9 +24,9 @@
 #include <linux/vt_kern.h>             /* For unblank_screen() */
 #include <linux/highmem.h>
 #include <linux/module.h>
+#include <linux/uaccess.h>
 
 #include <asm/m32r.h>
-#include <asm/uaccess.h>
 #include <asm/hardirq.h>
 #include <asm/mmu_context.h>
 #include <asm/tlbflush.h>
@@ -111,10 +111,10 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code,
        mm = tsk->mm;
 
        /*
-        * If we're in an interrupt or have no user context or are running in an
-        * atomic region then we must not take the fault..
+        * If we're in an interrupt or have no user context or have pagefaults
+        * disabled then we must not take the fault.
         */
-       if (in_atomic() || !mm)
+       if (faulthandler_disabled() || !mm)
                goto bad_area_nosemaphore;
 
        if (error_code & ACE_USERMODE)

diff --git a/arch/m68k/mm/fault.c b/arch/m68k/mm/fault.c
index b2f04aee46ecc2f7a5fb1db26d8e4279f6b6ea2e..6a94cdd0c8308cb70151d477872ccbeda2022ea8 100644 (file)
--- a/arch/m68k/mm/fault.c
+++ b/arch/m68k/mm/fault.c
@@ -10,10 +10,10 @@
 #include <linux/ptrace.h>
 #include <linux/interrupt.h>
 #include <linux/module.h>
+#include <linux/uaccess.h>
 
 #include <asm/setup.h>
 #include <asm/traps.h>
-#include <asm/uaccess.h>
 #include <asm/pgalloc.h>
 
 extern void die_if_kernel(char *, struct pt_regs *, long);
@@ -81,7 +81,7 @@ int do_page_fault(struct pt_regs *regs, unsigned long address,
         * If we're in an interrupt or have no user
         * context, we must not take the fault..
         */
-       if (in_atomic() || !mm)
+       if (faulthandler_disabled() || !mm)
                goto no_context;
 
        if (user_mode(regs))

diff --git a/arch/metag/mm/fault.c b/arch/metag/mm/fault.c
index 2de5dc695a87fa96d41a83e127166a7126d10df0..f57edca63609bf15f1a71bf5a90c8837d6a9fd13 100644 (file)
--- a/arch/metag/mm/fault.c
+++ b/arch/metag/mm/fault.c
@@ -105,7 +105,7 @@ int do_page_fault(struct pt_regs *regs, unsigned long address,
 
        mm = tsk->mm;
 
-       if (in_atomic() || !mm)
+       if (faulthandler_disabled() || !mm)
                goto no_context;
 
        if (user_mode(regs))

diff --git a/arch/microblaze/mm/fault.c b/arch/microblaze/mm/fault.c
index d46a5ebb7570e07869ea03b9b995374aa3bff82e..177dfc0036436284d4e016b0987e4516faa7f445 100644 (file)
--- a/arch/microblaze/mm/fault.c
+++ b/arch/microblaze/mm/fault.c
@@ -107,14 +107,14 @@ void do_page_fault(struct pt_regs *regs, unsigned long address,
        if ((error_code & 0x13) == 0x13 || (error_code & 0x11) == 0x11)
                is_write = 0;
 
-       if (unlikely(in_atomic() || !mm)) {
+       if (unlikely(faulthandler_disabled() || !mm)) {
                if (kernel_mode(regs))
                        goto bad_area_nosemaphore;
 
-               /* in_atomic() in user mode is really bad,
+               /* faulthandler_disabled() in user mode is really bad,
                   as is current->mm == NULL. */
-               pr_emerg("Page fault in user mode with in_atomic(), mm = %p\n",
-                                                                       mm);
+               pr_emerg("Page fault in user mode with faulthandler_disabled(), mm = %p\n",
+                        mm);
                pr_emerg("r15 = %lx  MSR = %lx\n",
                       regs->r15, regs->msr);
                die("Weird page fault", regs, SIGSEGV);

diff --git a/arch/mips/mm/fault.c b/arch/mips/mm/fault.c
index 7ff8637e530d7974d002594797d42044505a0467..36c0f26fac6b0780318958a59fc2665a444a10ea 100644 (file)
--- a/arch/mips/mm/fault.c
+++ b/arch/mips/mm/fault.c
@@ -21,10 +21,10 @@
 #include <linux/module.h>
 #include <linux/kprobes.h>
 #include <linux/perf_event.h>
+#include <linux/uaccess.h>
 
 #include <asm/branch.h>
 #include <asm/mmu_context.h>
-#include <asm/uaccess.h>
 #include <asm/ptrace.h>
 #include <asm/highmem.h>               /* For VMALLOC_END */
 #include <linux/kdebug.h>
@@ -94,7 +94,7 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write,
         * If we're in an interrupt or have no user
         * context, we must not take the fault..
         */
-       if (in_atomic() || !mm)
+       if (faulthandler_disabled() || !mm)
                goto bad_area_nosemaphore;
 
        if (user_mode(regs))

diff --git a/arch/mn10300/mm/fault.c b/arch/mn10300/mm/fault.c
index 0c2cc5d39c8e37ce1cfe5be191902bc435c41090..4a1d181ed32f7690a82cfcba3eb264f9311b0a0f 100644 (file)
--- a/arch/mn10300/mm/fault.c
+++ b/arch/mn10300/mm/fault.c
@@ -23,8 +23,8 @@
 #include <linux/interrupt.h>
 #include <linux/init.h>
 #include <linux/vt_kern.h>             /* For unblank_screen() */
+#include <linux/uaccess.h>
 
-#include <asm/uaccess.h>
 #include <asm/pgalloc.h>
 #include <asm/hardirq.h>
 #include <asm/cpu-regs.h>
@@ -168,7 +168,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long fault_code,
         * If we're in an interrupt or have no user
         * context, we must not take the fault..
         */
-       if (in_atomic() || !mm)
+       if (faulthandler_disabled() || !mm)
                goto no_context;
 
        if ((fault_code & MMUFCR_xFC_ACCESS) == MMUFCR_xFC_ACCESS_USR)

diff --git a/arch/nios2/mm/fault.c b/arch/nios2/mm/fault.c
index 0c9b6afe69e9094815cc1e73084422368b3a2e52..b51878b0c6b87362074c68832c1b4355f7c127cc 100644 (file)
--- a/arch/nios2/mm/fault.c
+++ b/arch/nios2/mm/fault.c
@@ -77,7 +77,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long cause,
         * If we're in an interrupt or have no user
         * context, we must not take the fault..
         */
-       if (in_atomic() || !mm)
+       if (faulthandler_disabled() || !mm)
                goto bad_area_nosemaphore;
 
        if (user_mode(regs))

diff --git a/arch/parisc/kernel/traps.c b/arch/parisc/kernel/traps.c
index 47ee620d15d27850ab8ebac1f739dfd3215dae9b..6548fd1d2e62defc2dfc6e8dfc125a42aa8e986e 100644 (file)
--- a/arch/parisc/kernel/traps.c
+++ b/arch/parisc/kernel/traps.c
@@ -26,9 +26,9 @@
 #include <linux/console.h>
 #include <linux/bug.h>
 #include <linux/ratelimit.h>
+#include <linux/uaccess.h>
 
 #include <asm/assembly.h>
-#include <asm/uaccess.h>
 #include <asm/io.h>
 #include <asm/irq.h>
 #include <asm/traps.h>
@@ -800,7 +800,7 @@ void notrace handle_interruption(int code, struct pt_regs *regs)
             * unless pagefault_disable() was called before.
             */
 
-           if (fault_space == 0 && !in_atomic())
+           if (fault_space == 0 && !faulthandler_disabled())
            {
                pdc_chassis_send_status(PDC_CHASSIS_DIRECT_PANIC);
                parisc_terminate("Kernel Fault", regs, code, fault_address);

diff --git a/arch/parisc/mm/fault.c b/arch/parisc/mm/fault.c
index e5120e653240c4fa52d4895c7d1d206d3d12e68c..15503adddf4f59695d34f3b5adb428250594bf66 100644 (file)
--- a/arch/parisc/mm/fault.c
+++ b/arch/parisc/mm/fault.c
@@ -15,8 +15,8 @@
 #include <linux/sched.h>
 #include <linux/interrupt.h>
 #include <linux/module.h>
+#include <linux/uaccess.h>
 
-#include <asm/uaccess.h>
 #include <asm/traps.h>
 
 /* Various important other fields */
@@ -207,7 +207,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long code,
        int fault;
        unsigned int flags;
 
-       if (in_atomic())
+       if (pagefault_disabled())
                goto no_context;
 
        tsk = current;

diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
index b396868d2aa7c48438f1577df35759991b165f6a..6d535973b200dde0f64ae19a3d47f1eaca6a4c99 100644 (file)
--- a/arch/powerpc/mm/fault.c
+++ b/arch/powerpc/mm/fault.c
@@ -33,13 +33,13 @@
 #include <linux/ratelimit.h>
 #include <linux/context_tracking.h>
 #include <linux/hugetlb.h>
+#include <linux/uaccess.h>
 
 #include <asm/firmware.h>
 #include <asm/page.h>
 #include <asm/pgtable.h>
 #include <asm/mmu.h>
 #include <asm/mmu_context.h>
-#include <asm/uaccess.h>
 #include <asm/tlbflush.h>
 #include <asm/siginfo.h>
 #include <asm/debug.h>
@@ -272,15 +272,16 @@ int __kprobes do_page_fault(struct pt_regs *regs, unsigned long address,
        if (!arch_irq_disabled_regs(regs))
                local_irq_enable();
 
-       if (in_atomic() || mm == NULL) {
+       if (faulthandler_disabled() || mm == NULL) {
                if (!user_mode(regs)) {
                        rc = SIGSEGV;
                        goto bail;
                }
-               /* in_atomic() in user mode is really bad,
+               /* faulthandler_disabled() in user mode is really bad,
                   as is current->mm == NULL. */
                printk(KERN_EMERG "Page fault in user mode with "
-                      "in_atomic() = %d mm = %p\n", in_atomic(), mm);
+                      "faulthandler_disabled() = %d mm = %p\n",
+                      faulthandler_disabled(), mm);
                printk(KERN_EMERG "NIP = %lx  MSR = %lx\n",
                       regs->nip, regs->msr);
                die("Weird page fault", regs, SIGSEGV);

diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c
index 76515bcea2f18f5f78e59b86c0bd331fed0380f2..4c8f5d7f9c23d74c960cd1172f266391ba8ec1a3 100644 (file)
--- a/arch/s390/mm/fault.c
+++ b/arch/s390/mm/fault.c
@@ -399,7 +399,7 @@ static inline int do_exception(struct pt_regs *regs, int access)
         * user context.
         */
        fault = VM_FAULT_BADCONTEXT;
-       if (unlikely(!user_space_fault(regs) || in_atomic() || !mm))
+       if (unlikely(!user_space_fault(regs) || faulthandler_disabled() || !mm))
                goto out;
 
        address = trans_exc_code & __FAIL_ADDR_MASK;

diff --git a/arch/score/mm/fault.c b/arch/score/mm/fault.c
index 6860beb2a280d0a4a65a67c89ad2201b33513068..37a6c2e0e96926f26902484969befaf860fc52fc 100644 (file)
--- a/arch/score/mm/fault.c
+++ b/arch/score/mm/fault.c
@@ -34,6 +34,7 @@
 #include <linux/string.h>
 #include <linux/types.h>
 #include <linux/ptrace.h>
+#include <linux/uaccess.h>
 
 /*
  * This routine handles page faults.  It determines the address,
@@ -73,7 +74,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long write,
        * If we're in an interrupt or have no user
        * context, we must not take the fault..
        */
-       if (in_atomic() || !mm)
+       if (pagefault_disabled() || !mm)
                goto bad_area_nosemaphore;
 
        if (user_mode(regs))

diff --git a/arch/sh/mm/fault.c b/arch/sh/mm/fault.c
index a58fec9b55e016df85cdfb7c214cc385e300479c..79d8276377d1e2f62e6c9231f735ef0f1c5ca22e 100644 (file)
--- a/arch/sh/mm/fault.c
+++ b/arch/sh/mm/fault.c
@@ -17,6 +17,7 @@
 #include <linux/kprobes.h>
 #include <linux/perf_event.h>
 #include <linux/kdebug.h>
+#include <linux/uaccess.h>
 #include <asm/io_trapped.h>
 #include <asm/mmu_context.h>
 #include <asm/tlbflush.h>
@@ -438,9 +439,9 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
 
        /*
         * If we're in an interrupt, have no user context or are running
-        * in an atomic region then we must not take the fault:
+        * with pagefaults disabled then we must not take the fault:
         */
-       if (unlikely(in_atomic() || !mm)) {
+       if (unlikely(faulthandler_disabled() || !mm)) {
                bad_area_nosemaphore(regs, error_code, address);
                return;
        }

diff --git a/arch/sparc/mm/fault_32.c b/arch/sparc/mm/fault_32.c
index 70d817154fe8bfd04aeaa71f45f15667f4962c23..c399e7b3b035250d66ed4522d2da190dc6169aa3 100644 (file)
--- a/arch/sparc/mm/fault_32.c
+++ b/arch/sparc/mm/fault_32.c
@@ -21,6 +21,7 @@
 #include <linux/perf_event.h>
 #include <linux/interrupt.h>
 #include <linux/kdebug.h>
+#include <linux/uaccess.h>
 
 #include <asm/page.h>
 #include <asm/pgtable.h>
@@ -29,7 +30,6 @@
 #include <asm/setup.h>
 #include <asm/smp.h>
 #include <asm/traps.h>
-#include <asm/uaccess.h>
 
 #include "mm_32.h"
 
@@ -196,7 +196,7 @@ asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write,
         * If we're in an interrupt or have no user
         * context, we must not take the fault..
         */
-       if (in_atomic() || !mm)
+       if (pagefault_disabled() || !mm)
                goto no_context;
 
        perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);

diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c
index 4798232494294a7ece0bef232216dd4a26408d88..e9268ea1a68de5364d468973e61b3cbd232c19c9 100644 (file)
--- a/arch/sparc/mm/fault_64.c
+++ b/arch/sparc/mm/fault_64.c
@@ -22,12 +22,12 @@
 #include <linux/kdebug.h>
 #include <linux/percpu.h>
 #include <linux/context_tracking.h>
+#include <linux/uaccess.h>
 
 #include <asm/page.h>
 #include <asm/pgtable.h>
 #include <asm/openprom.h>
 #include <asm/oplib.h>
-#include <asm/uaccess.h>
 #include <asm/asi.h>
 #include <asm/lsu.h>
 #include <asm/sections.h>
@@ -330,7 +330,7 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs)
         * If we're in an interrupt or have no user
         * context, we must not take the fault..
         */
-       if (in_atomic() || !mm)
+       if (faulthandler_disabled() || !mm)
                goto intr_or_no_mm;
 
        perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);

diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index 4ca0d6ba5ec8331c67f43f8515eb3737526208bb..cee9b77ddd05ac38061b49b0940591b3ac112489 100644 (file)
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -2706,7 +2706,7 @@ void hugetlb_setup(struct pt_regs *regs)
        struct mm_struct *mm = current->mm;
        struct tsb_config *tp;
 
-       if (in_atomic() || !mm) {
+       if (faulthandler_disabled() || !mm) {
                const struct exception_table_entry *entry;
 
                entry = search_exception_tables(regs->tpc);

diff --git a/arch/tile/mm/fault.c b/arch/tile/mm/fault.c
index e83cc999da029b469fde1f2e7f2b4d0f5a96dbb6..3f4f58d34a92b6029615756892c1fe207d6f6701 100644 (file)
--- a/arch/tile/mm/fault.c
+++ b/arch/tile/mm/fault.c
@@ -354,9 +354,9 @@ static int handle_page_fault(struct pt_regs *regs,
 
        /*
         * If we're in an interrupt, have no user context or are running in an
-        * atomic region then we must not take the fault.
+        * region with pagefaults disabled then we must not take the fault.
         */
-       if (in_atomic() || !mm) {
+       if (pagefault_disabled() || !mm) {
                vma = NULL;  /* happy compiler */
                goto bad_area_nosemaphore;
        }

diff --git a/arch/um/kernel/trap.c b/arch/um/kernel/trap.c
index 8e4daf44e9805ed2380b72826a699ed139819ee3..f9c9e5a6bebaacba7b3eae6ade67dc243e2da5eb 100644 (file)
--- a/arch/um/kernel/trap.c
+++ b/arch/um/kernel/trap.c
@@ -35,10 +35,10 @@ int handle_page_fault(unsigned long address, unsigned long ip,
        *code_out = SEGV_MAPERR;
 
        /*
-        * If the fault was during atomic operation, don't take the fault, just
+        * If the fault was with pagefaults disabled, don't take the fault, just
         * fail.
         */
-       if (in_atomic())
+       if (faulthandler_disabled())
                goto out_nosemaphore;
 
        if (is_user)

diff --git a/arch/unicore32/mm/fault.c b/arch/unicore32/mm/fault.c
index 0dc922dba9154d7cfcfe5352ec8ec77169e8082f..afccef5529ccb8d44409519aa075476af747722e 100644 (file)
--- a/arch/unicore32/mm/fault.c
+++ b/arch/unicore32/mm/fault.c
@@ -218,7 +218,7 @@ static int do_pf(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
         * If we're in an interrupt or have no user
         * context, we must not take the fault..
         */
-       if (in_atomic() || !mm)
+       if (faulthandler_disabled() || !mm)
                goto no_context;
 
        if (user_mode(regs))

diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index 181c53bac3a7ee8881b8844bae66b951d9beecde..9dc909841739bf24b01d7d2e574c4870cc409cfd 100644 (file)
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -13,6 +13,7 @@
 #include <linux/hugetlb.h>             /* hstate_index_to_shift        */
 #include <linux/prefetch.h>            /* prefetchw                    */
 #include <linux/context_tracking.h>    /* exception_enter(), ...       */
+#include <linux/uaccess.h>             /* faulthandler_disabled()      */
 
 #include <asm/traps.h>                 /* dotraplinkage, ...           */
 #include <asm/pgalloc.h>               /* pgd_*(), ...                 */
@@ -1126,9 +1127,9 @@ __do_page_fault(struct pt_regs *regs, unsigned long error_code,
 
        /*
         * If we're in an interrupt, have no user context or are running
-        * in an atomic region then we must not take the fault:
+        * in a region with pagefaults disabled then we must not take the fault
         */
-       if (unlikely(in_atomic() || !mm)) {
+       if (unlikely(faulthandler_disabled() || !mm)) {
                bad_area_nosemaphore(regs, error_code, address);
                return;
        }

diff --git a/arch/xtensa/mm/fault.c b/arch/xtensa/mm/fault.c
index 9e3571a6535c3b1bbc8535195ee40405fe9c42c0..83a44a33cfa11221f99ba5f8d836e02a0c4c9f92 100644 (file)
--- a/arch/xtensa/mm/fault.c
+++ b/arch/xtensa/mm/fault.c
@@ -15,10 +15,10 @@
 #include <linux/mm.h>
 #include <linux/module.h>
 #include <linux/hardirq.h>
+#include <linux/uaccess.h>
 #include <asm/mmu_context.h>
 #include <asm/cacheflush.h>
 #include <asm/hardirq.h>
-#include <asm/uaccess.h>
 #include <asm/pgalloc.h>
 
 DEFINE_PER_CPU(unsigned long, asid_cache) = ASID_USER_FIRST;
@@ -57,7 +57,7 @@ void do_page_fault(struct pt_regs *regs)
        /* If we're in an interrupt or have no user
         * context, we must not take the fault..
         */
-       if (in_atomic() || !mm) {
+       if (faulthandler_disabled() || !mm) {
                bad_page_fault(regs, address, SIGSEGV);
                return;
        }

diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h
index 23290cc93a2473a804a52b7e7eac48616ff87429..90786d2d74e51a1d35aff6ba90de63b55c5fce64 100644 (file)
--- a/include/linux/uaccess.h
+++ b/include/linux/uaccess.h
@@ -59,6 +59,18 @@ static inline void pagefault_enable(void)
  */
 #define pagefault_disabled() (current->pagefault_disabled != 0)
 
+/*
+ * The pagefault handler is in general disabled by pagefault_disable() or
+ * when in irq context (via in_atomic()).
+ *
+ * This function should only be used by the fault handlers. Other users should
+ * stick to pagefault_disabled().
+ * Please NEVER use preempt_disable() to disable the fault handler. With
+ * !CONFIG_PREEMPT_COUNT, this is like a NOP. So the handler won't be disabled.
+ * in_atomic() will report different values based on !CONFIG_PREEMPT_COUNT.
+ */
+#define faulthandler_disabled() (pagefault_disabled() || in_atomic())
+
 #ifndef ARCH_HAS_NOCACHE_UACCESS
 
 static inline unsigned long __copy_from_user_inatomic_nocache(void *to,