Merge remote-tracking branch 'regulator/topic/palmas' into v3.9-rc8

[mirror_ubuntu-hirsute-kernel.git] / arch / sparc / mm / ultra.S

diff --git a/arch/sparc/mm/ultra.S b/arch/sparc/mm/ultra.S
index f8e13d421fcbf415dd49a50bc2ed4aaf2b3b6494..432aa0cb1b38caa75076938700c60761699dc920 100644 (file)
--- a/arch/sparc/mm/ultra.S
+++ b/arch/sparc/mm/ultra.S
@@ -52,6 +52,33 @@ __flush_tlb_mm:              /* 18 insns */
        nop
        nop
 
+       .align          32
+       .globl          __flush_tlb_page
+__flush_tlb_page:      /* 22 insns */
+       /* %o0 = context, %o1 = vaddr */
+       rdpr            %pstate, %g7
+       andn            %g7, PSTATE_IE, %g2
+       wrpr            %g2, %pstate
+       mov             SECONDARY_CONTEXT, %o4
+       ldxa            [%o4] ASI_DMMU, %g2
+       stxa            %o0, [%o4] ASI_DMMU
+       andcc           %o1, 1, %g0
+       andn            %o1, 1, %o3
+       be,pn           %icc, 1f
+        or             %o3, 0x10, %o3
+       stxa            %g0, [%o3] ASI_IMMU_DEMAP
+1:     stxa            %g0, [%o3] ASI_DMMU_DEMAP
+       membar          #Sync
+       stxa            %g2, [%o4] ASI_DMMU
+       sethi           %hi(KERNBASE), %o4
+       flush           %o4
+       retl
+        wrpr           %g7, 0x0, %pstate
+       nop
+       nop
+       nop
+       nop
+
        .align          32
        .globl          __flush_tlb_pending
 __flush_tlb_pending:   /* 26 insns */
@@ -203,6 +230,31 @@ __cheetah_flush_tlb_mm: /* 19 insns */
        retl
         wrpr           %g7, 0x0, %pstate
 
+__cheetah_flush_tlb_page:      /* 22 insns */
+       /* %o0 = context, %o1 = vaddr */
+       rdpr            %pstate, %g7
+       andn            %g7, PSTATE_IE, %g2
+       wrpr            %g2, 0x0, %pstate
+       wrpr            %g0, 1, %tl
+       mov             PRIMARY_CONTEXT, %o4
+       ldxa            [%o4] ASI_DMMU, %g2
+       srlx            %g2, CTX_PGSZ1_NUC_SHIFT, %o3
+       sllx            %o3, CTX_PGSZ1_NUC_SHIFT, %o3
+       or              %o0, %o3, %o0   /* Preserve nucleus page size fields */
+       stxa            %o0, [%o4] ASI_DMMU
+       andcc           %o1, 1, %g0
+       be,pn           %icc, 1f
+        andn           %o1, 1, %o3
+       stxa            %g0, [%o3] ASI_IMMU_DEMAP
+1:     stxa            %g0, [%o3] ASI_DMMU_DEMAP       
+       membar          #Sync
+       stxa            %g2, [%o4] ASI_DMMU
+       sethi           %hi(KERNBASE), %o4
+       flush           %o4
+       wrpr            %g0, 0, %tl
+       retl
+        wrpr           %g7, 0x0, %pstate
+
 __cheetah_flush_tlb_pending:   /* 27 insns */
        /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
        rdpr            %pstate, %g7
@@ -269,6 +321,20 @@ __hypervisor_flush_tlb_mm: /* 10 insns */
        retl
         nop
 
+__hypervisor_flush_tlb_page: /* 11 insns */
+       /* %o0 = context, %o1 = vaddr */
+       mov             %o0, %g2
+       mov             %o1, %o0              /* ARG0: vaddr + IMMU-bit */
+       mov             %g2, %o1              /* ARG1: mmu context */
+       mov             HV_MMU_ALL, %o2       /* ARG2: flags */
+       srlx            %o0, PAGE_SHIFT, %o0
+       sllx            %o0, PAGE_SHIFT, %o0
+       ta              HV_MMU_UNMAP_ADDR_TRAP
+       brnz,pn         %o0, __hypervisor_tlb_tl0_error
+        mov            HV_MMU_UNMAP_ADDR_TRAP, %o1
+       retl
+        nop
+
 __hypervisor_flush_tlb_pending: /* 16 insns */
        /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
        sllx            %o1, 3, %g1
@@ -339,6 +405,13 @@ cheetah_patch_cachetlbops:
        call            tlb_patch_one
         mov            19, %o2
 
+       sethi           %hi(__flush_tlb_page), %o0
+       or              %o0, %lo(__flush_tlb_page), %o0
+       sethi           %hi(__cheetah_flush_tlb_page), %o1
+       or              %o1, %lo(__cheetah_flush_tlb_page), %o1
+       call            tlb_patch_one
+        mov            22, %o2
+
        sethi           %hi(__flush_tlb_pending), %o0
        or              %o0, %lo(__flush_tlb_pending), %o0
        sethi           %hi(__cheetah_flush_tlb_pending), %o1
@@ -397,10 +470,9 @@ xcall_flush_tlb_mm:        /* 21 insns */
        nop
        nop
 
-       .globl          xcall_flush_tlb_pending
-xcall_flush_tlb_pending:       /* 21 insns */
-       /* %g5=context, %g1=nr, %g7=vaddrs[] */
-       sllx            %g1, 3, %g1
+       .globl          xcall_flush_tlb_page
+xcall_flush_tlb_page:  /* 17 insns */
+       /* %g5=context, %g1=vaddr */
        mov             PRIMARY_CONTEXT, %g4
        ldxa            [%g4] ASI_DMMU, %g2
        srlx            %g2, CTX_PGSZ1_NUC_SHIFT, %g4
@@ -408,20 +480,16 @@ xcall_flush_tlb_pending:  /* 21 insns */
        or              %g5, %g4, %g5
        mov             PRIMARY_CONTEXT, %g4
        stxa            %g5, [%g4] ASI_DMMU
-1:     sub             %g1, (1 << 3), %g1
-       ldx             [%g7 + %g1], %g5
-       andcc           %g5, 0x1, %g0
+       andcc           %g1, 0x1, %g0
        be,pn           %icc, 2f
-
-        andn           %g5, 0x1, %g5
+        andn           %g1, 0x1, %g5
        stxa            %g0, [%g5] ASI_IMMU_DEMAP
 2:     stxa            %g0, [%g5] ASI_DMMU_DEMAP
        membar          #Sync
-       brnz,pt         %g1, 1b
-        nop
        stxa            %g2, [%g4] ASI_DMMU
        retry
        nop
+       nop
 
        .globl          xcall_flush_tlb_kernel_range
 xcall_flush_tlb_kernel_range:  /* 25 insns */
@@ -656,15 +724,13 @@ __hypervisor_xcall_flush_tlb_mm: /* 21 insns */
        membar          #Sync
        retry
 
-       .globl          __hypervisor_xcall_flush_tlb_pending
-__hypervisor_xcall_flush_tlb_pending: /* 21 insns */
-       /* %g5=ctx, %g1=nr, %g7=vaddrs[], %g2,%g3,%g4,g6=scratch */
-       sllx            %g1, 3, %g1
+       .globl          __hypervisor_xcall_flush_tlb_page
+__hypervisor_xcall_flush_tlb_page: /* 17 insns */
+       /* %g5=ctx, %g1=vaddr */
        mov             %o0, %g2
        mov             %o1, %g3
        mov             %o2, %g4
-1:     sub             %g1, (1 << 3), %g1
-       ldx             [%g7 + %g1], %o0        /* ARG0: virtual address */
+       mov             %g1, %o0                /* ARG0: virtual address */
        mov             %g5, %o1                /* ARG1: mmu context */
        mov             HV_MMU_ALL, %o2         /* ARG2: flags */
        srlx            %o0, PAGE_SHIFT, %o0
@@ -673,8 +739,6 @@ __hypervisor_xcall_flush_tlb_pending: /* 21 insns */
        mov             HV_MMU_UNMAP_ADDR_TRAP, %g6
        brnz,a,pn       %o0, __hypervisor_tlb_xcall_error
         mov            %o0, %g5
-       brnz,pt         %g1, 1b
-        nop
        mov             %g2, %o0
        mov             %g3, %o1
        mov             %g4, %o2
@@ -757,6 +821,13 @@ hypervisor_patch_cachetlbops:
        call            tlb_patch_one
         mov            10, %o2
 
+       sethi           %hi(__flush_tlb_page), %o0
+       or              %o0, %lo(__flush_tlb_page), %o0
+       sethi           %hi(__hypervisor_flush_tlb_page), %o1
+       or              %o1, %lo(__hypervisor_flush_tlb_page), %o1
+       call            tlb_patch_one
+        mov            11, %o2
+
        sethi           %hi(__flush_tlb_pending), %o0
        or              %o0, %lo(__flush_tlb_pending), %o0
        sethi           %hi(__hypervisor_flush_tlb_pending), %o1
@@ -788,12 +859,12 @@ hypervisor_patch_cachetlbops:
        call            tlb_patch_one
         mov            21, %o2
 
-       sethi           %hi(xcall_flush_tlb_pending), %o0
-       or              %o0, %lo(xcall_flush_tlb_pending), %o0
-       sethi           %hi(__hypervisor_xcall_flush_tlb_pending), %o1
-       or              %o1, %lo(__hypervisor_xcall_flush_tlb_pending), %o1
+       sethi           %hi(xcall_flush_tlb_page), %o0
+       or              %o0, %lo(xcall_flush_tlb_page), %o0
+       sethi           %hi(__hypervisor_xcall_flush_tlb_page), %o1
+       or              %o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1
        call            tlb_patch_one
-        mov            21, %o2
+        mov            17, %o2
 
        sethi           %hi(xcall_flush_tlb_kernel_range), %o0
        or              %o0, %lo(xcall_flush_tlb_kernel_range), %o0