x86/mm: Optimize RESTORE_CR3

[mirror_ubuntu-bionic-kernel.git] / arch / x86 / entry / calling.h

diff --git a/arch/x86/entry/calling.h b/arch/x86/entry/calling.h
index 7894e5c0eef73503281c55048ccccdc8cfdcf83d..45a63e00a6af9a12b4739246d6844ba94f766e71 100644 (file)
--- a/arch/x86/entry/calling.h
+++ b/arch/x86/entry/calling.h
@@ -281,8 +281,34 @@ For 32-bit we have the following conventions - kernel is built with
 .Ldone_\@:
 .endm
 
-.macro RESTORE_CR3 save_reg:req
+.macro RESTORE_CR3 scratch_reg:req save_reg:req
        ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
+
+       ALTERNATIVE "jmp .Lwrcr3_\@", "", X86_FEATURE_PCID
+
+       /*
+        * KERNEL pages can always resume with NOFLUSH as we do
+        * explicit flushes.
+        */
+       bt      $X86_CR3_PTI_SWITCH_BIT, \save_reg
+       jnc     .Lnoflush_\@
+
+       /*
+        * Check if there's a pending flush for the user ASID we're
+        * about to set.
+        */
+       movq    \save_reg, \scratch_reg
+       andq    $(0x7FF), \scratch_reg
+       bt      \scratch_reg, THIS_CPU_user_pcid_flush_mask
+       jnc     .Lnoflush_\@
+
+       btr     \scratch_reg, THIS_CPU_user_pcid_flush_mask
+       jmp     .Lwrcr3_\@
+
+.Lnoflush_\@:
+       SET_NOFLUSH_BIT \save_reg
+
+.Lwrcr3_\@:
        /*
         * The CR3 write could be avoided when not changing its value,
         * but would require a CR3 read *and* a scratch register.
@@ -301,7 +327,7 @@ For 32-bit we have the following conventions - kernel is built with
 .endm
 .macro SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg:req save_reg:req
 .endm
-.macro RESTORE_CR3 save_reg:req
+.macro RESTORE_CR3 scratch_reg:req save_reg:req
 .endm
 
 #endif