arch/sparc/mm/ultra.S

   1 /*
   2  * ultra.S: Don't expand these all over the place...
   3  *
   4  * Copyright (C) 1997, 2000, 2008 David S. Miller (davem@davemloft.net)
   5  */
   6
   7 #include <asm/asi.h>
   8 #include <asm/pgtable.h>
   9 #include <asm/page.h>
  10 #include <asm/spitfire.h>
  11 #include <asm/mmu_context.h>
  12 #include <asm/mmu.h>
  13 #include <asm/pil.h>
  14 #include <asm/head.h>
  15 #include <asm/thread_info.h>
  16 #include <asm/cacheflush.h>
  17 #include <asm/hypervisor.h>
  18 #include <asm/cpudata.h>
  19
  20         /* Basically, most of the Spitfire vs. Cheetah madness
  21          * has to do with the fact that Cheetah does not support
  22          * IMMU flushes out of the secondary context.  Someone needs
  23          * to throw a south lake birthday party for the folks
  24          * in Microelectronics who refused to fix this shit.
  25          */
  26
  27         /* This file is meant to be read efficiently by the CPU, not humans.
  28          * Staraj sie tego nikomu nie pierdolnac...
  29          */
  30         .text
  31         .align          32
  32         .globl          __flush_tlb_mm
  33 __flush_tlb_mm:         /* 19 insns */
  34         /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */
  35         ldxa            [%o1] ASI_DMMU, %g2
  36         cmp             %g2, %o0
  37         bne,pn          %icc, __spitfire_flush_tlb_mm_slow
  38          mov            0x50, %g3
  39         stxa            %g0, [%g3] ASI_DMMU_DEMAP
  40         stxa            %g0, [%g3] ASI_IMMU_DEMAP
  41         sethi           %hi(KERNBASE), %g3
  42         flush           %g3
  43         retl
  44          nop
  45         nop
  46         nop
  47         nop
  48         nop
  49         nop
  50         nop
  51         nop
  52         nop
  53         nop
  54
  55         .align          32
  56         .globl          __flush_tlb_page
  57 __flush_tlb_page:       /* 22 insns */
  58         /* %o0 = context, %o1 = vaddr */
  59         rdpr            %pstate, %g7
  60         andn            %g7, PSTATE_IE, %g2
  61         wrpr            %g2, %pstate
  62         mov             SECONDARY_CONTEXT, %o4
  63         ldxa            [%o4] ASI_DMMU, %g2
  64         stxa            %o0, [%o4] ASI_DMMU
  65         andcc           %o1, 1, %g0
  66         andn            %o1, 1, %o3
  67         be,pn           %icc, 1f
  68          or             %o3, 0x10, %o3
  69         stxa            %g0, [%o3] ASI_IMMU_DEMAP
  70 1:      stxa            %g0, [%o3] ASI_DMMU_DEMAP
  71         membar          #Sync
  72         stxa            %g2, [%o4] ASI_DMMU
  73         sethi           %hi(KERNBASE), %o4
  74         flush           %o4
  75         retl
  76          wrpr           %g7, 0x0, %pstate
  77         nop
  78         nop
  79         nop
  80         nop
  81
  82         .align          32
  83         .globl          __flush_tlb_pending
  84 __flush_tlb_pending:    /* 27 insns */
  85         /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
  86         rdpr            %pstate, %g7
  87         sllx            %o1, 3, %o1
  88         andn            %g7, PSTATE_IE, %g2
  89         wrpr            %g2, %pstate
  90         mov             SECONDARY_CONTEXT, %o4
  91         ldxa            [%o4] ASI_DMMU, %g2
  92         stxa            %o0, [%o4] ASI_DMMU
  93 1:      sub             %o1, (1 << 3), %o1
  94         ldx             [%o2 + %o1], %o3
  95         andcc           %o3, 1, %g0
  96         andn            %o3, 1, %o3
  97         be,pn           %icc, 2f
  98          or             %o3, 0x10, %o3
  99         stxa            %g0, [%o3] ASI_IMMU_DEMAP
 100 2:      stxa            %g0, [%o3] ASI_DMMU_DEMAP
 101         membar          #Sync
 102         brnz,pt         %o1, 1b
 103          nop
 104         stxa            %g2, [%o4] ASI_DMMU
 105         sethi           %hi(KERNBASE), %o4
 106         flush           %o4
 107         retl
 108          wrpr           %g7, 0x0, %pstate
 109         nop
 110         nop
 111         nop
 112         nop
 113
 114         .align          32
 115         .globl          __flush_tlb_kernel_range
 116 __flush_tlb_kernel_range:       /* 31 insns */
 117         /* %o0=start, %o1=end */
 118         cmp             %o0, %o1
 119         be,pn           %xcc, 2f
 120          sub            %o1, %o0, %o3
 121         srlx            %o3, 18, %o4
 122         brnz,pn         %o4, __spitfire_flush_tlb_kernel_range_slow
 123          sethi          %hi(PAGE_SIZE), %o4
 124         sub             %o3, %o4, %o3
 125         or              %o0, 0x20, %o0          ! Nucleus
 126 1:      stxa            %g0, [%o0 + %o3] ASI_DMMU_DEMAP
 127         stxa            %g0, [%o0 + %o3] ASI_IMMU_DEMAP
 128         membar          #Sync
 129         brnz,pt         %o3, 1b
 130          sub            %o3, %o4, %o3
 131 2:      sethi           %hi(KERNBASE), %o3
 132         flush           %o3
 133         retl
 134          nop
 135         nop
 136         nop
 137         nop
 138         nop
 139         nop
 140         nop
 141         nop
 142         nop
 143         nop
 144         nop
 145         nop
 146         nop
 147         nop
 148         nop
 149
 150 __spitfire_flush_tlb_kernel_range_slow:
 151         mov             63 * 8, %o4
 152 1:      ldxa            [%o4] ASI_ITLB_DATA_ACCESS, %o3
 153         andcc           %o3, 0x40, %g0                  /* _PAGE_L_4U */
 154         bne,pn          %xcc, 2f
 155          mov            TLB_TAG_ACCESS, %o3
 156         stxa            %g0, [%o3] ASI_IMMU
 157         stxa            %g0, [%o4] ASI_ITLB_DATA_ACCESS
 158         membar          #Sync
 159 2:      ldxa            [%o4] ASI_DTLB_DATA_ACCESS, %o3
 160         andcc           %o3, 0x40, %g0
 161         bne,pn          %xcc, 2f
 162          mov            TLB_TAG_ACCESS, %o3
 163         stxa            %g0, [%o3] ASI_DMMU
 164         stxa            %g0, [%o4] ASI_DTLB_DATA_ACCESS
 165         membar          #Sync
 166 2:      sub             %o4, 8, %o4
 167         brgez,pt        %o4, 1b
 168          nop
 169         retl
 170          nop
 171
 172 __spitfire_flush_tlb_mm_slow:
 173         rdpr            %pstate, %g1
 174         wrpr            %g1, PSTATE_IE, %pstate
 175         stxa            %o0, [%o1] ASI_DMMU
 176         stxa            %g0, [%g3] ASI_DMMU_DEMAP
 177         stxa            %g0, [%g3] ASI_IMMU_DEMAP
 178         flush           %g6
 179         stxa            %g2, [%o1] ASI_DMMU
 180         sethi           %hi(KERNBASE), %o1
 181         flush           %o1
 182         retl
 183          wrpr           %g1, 0, %pstate
 184
 185 /*
 186  * The following code flushes one page_size worth.
 187  */
 188         .section .kprobes.text, "ax"
 189         .align          32
 190         .globl          __flush_icache_page
 191 __flush_icache_page:    /* %o0 = phys_page */
 192         srlx            %o0, PAGE_SHIFT, %o0
 193         sethi           %hi(PAGE_OFFSET), %g1
 194         sllx            %o0, PAGE_SHIFT, %o0
 195         sethi           %hi(PAGE_SIZE), %g2
 196         ldx             [%g1 + %lo(PAGE_OFFSET)], %g1
 197         add             %o0, %g1, %o0
 198 1:      subcc           %g2, 32, %g2
 199         bne,pt          %icc, 1b
 200          flush          %o0 + %g2
 201         retl
 202          nop
 203
 204 #ifdef DCACHE_ALIASING_POSSIBLE
 205
 206 #if (PAGE_SHIFT != 13)
 207 #error only page shift of 13 is supported by dcache flush
 208 #endif
 209
 210 #define DTAG_MASK 0x3
 211
 212         /* This routine is Spitfire specific so the hardcoded
 213          * D-cache size and line-size are OK.
 214          */
 215         .align          64
 216         .globl          __flush_dcache_page
 217 __flush_dcache_page:    /* %o0=kaddr, %o1=flush_icache */
 218         sethi           %hi(PAGE_OFFSET), %g1
 219         ldx             [%g1 + %lo(PAGE_OFFSET)], %g1
 220         sub             %o0, %g1, %o0                   ! physical address
 221         srlx            %o0, 11, %o0                    ! make D-cache TAG
 222         sethi           %hi(1 << 14), %o2               ! D-cache size
 223         sub             %o2, (1 << 5), %o2              ! D-cache line size
 224 1:      ldxa            [%o2] ASI_DCACHE_TAG, %o3       ! load D-cache TAG
 225         andcc           %o3, DTAG_MASK, %g0             ! Valid?
 226         be,pn           %xcc, 2f                        ! Nope, branch
 227          andn           %o3, DTAG_MASK, %o3             ! Clear valid bits
 228         cmp             %o3, %o0                        ! TAG match?
 229         bne,pt          %xcc, 2f                        ! Nope, branch
 230          nop
 231         stxa            %g0, [%o2] ASI_DCACHE_TAG       ! Invalidate TAG
 232         membar          #Sync
 233 2:      brnz,pt         %o2, 1b
 234          sub            %o2, (1 << 5), %o2              ! D-cache line size
 235
 236         /* The I-cache does not snoop local stores so we
 237          * better flush that too when necessary.
 238          */
 239         brnz,pt         %o1, __flush_icache_page
 240          sllx           %o0, 11, %o0
 241         retl
 242          nop
 243
 244 #endif /* DCACHE_ALIASING_POSSIBLE */
 245
 246         .previous
 247
 248         /* Cheetah specific versions, patched at boot time. */
 249 __cheetah_flush_tlb_mm: /* 19 insns */
 250         rdpr            %pstate, %g7
 251         andn            %g7, PSTATE_IE, %g2
 252         wrpr            %g2, 0x0, %pstate
 253         wrpr            %g0, 1, %tl
 254         mov             PRIMARY_CONTEXT, %o2
 255         mov             0x40, %g3
 256         ldxa            [%o2] ASI_DMMU, %g2
 257         srlx            %g2, CTX_PGSZ1_NUC_SHIFT, %o1
 258         sllx            %o1, CTX_PGSZ1_NUC_SHIFT, %o1
 259         or              %o0, %o1, %o0   /* Preserve nucleus page size fields */
 260         stxa            %o0, [%o2] ASI_DMMU
 261         stxa            %g0, [%g3] ASI_DMMU_DEMAP
 262         stxa            %g0, [%g3] ASI_IMMU_DEMAP
 263         stxa            %g2, [%o2] ASI_DMMU
 264         sethi           %hi(KERNBASE), %o2
 265         flush           %o2
 266         wrpr            %g0, 0, %tl
 267         retl
 268          wrpr           %g7, 0x0, %pstate
 269
 270 __cheetah_flush_tlb_page:       /* 22 insns */
 271         /* %o0 = context, %o1 = vaddr */
 272         rdpr            %pstate, %g7
 273         andn            %g7, PSTATE_IE, %g2
 274         wrpr            %g2, 0x0, %pstate
 275         wrpr            %g0, 1, %tl
 276         mov             PRIMARY_CONTEXT, %o4
 277         ldxa            [%o4] ASI_DMMU, %g2
 278         srlx            %g2, CTX_PGSZ1_NUC_SHIFT, %o3
 279         sllx            %o3, CTX_PGSZ1_NUC_SHIFT, %o3
 280         or              %o0, %o3, %o0   /* Preserve nucleus page size fields */
 281         stxa            %o0, [%o4] ASI_DMMU
 282         andcc           %o1, 1, %g0
 283         be,pn           %icc, 1f
 284          andn           %o1, 1, %o3
 285         stxa            %g0, [%o3] ASI_IMMU_DEMAP
 286 1:      stxa            %g0, [%o3] ASI_DMMU_DEMAP
 287         membar          #Sync
 288         stxa            %g2, [%o4] ASI_DMMU
 289         sethi           %hi(KERNBASE), %o4
 290         flush           %o4
 291         wrpr            %g0, 0, %tl
 292         retl
 293          wrpr           %g7, 0x0, %pstate
 294
 295 __cheetah_flush_tlb_pending:    /* 27 insns */
 296         /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
 297         rdpr            %pstate, %g7
 298         sllx            %o1, 3, %o1
 299         andn            %g7, PSTATE_IE, %g2
 300         wrpr            %g2, 0x0, %pstate
 301         wrpr            %g0, 1, %tl
 302         mov             PRIMARY_CONTEXT, %o4
 303         ldxa            [%o4] ASI_DMMU, %g2
 304         srlx            %g2, CTX_PGSZ1_NUC_SHIFT, %o3
 305         sllx            %o3, CTX_PGSZ1_NUC_SHIFT, %o3
 306         or              %o0, %o3, %o0   /* Preserve nucleus page size fields */
 307         stxa            %o0, [%o4] ASI_DMMU
 308 1:      sub             %o1, (1 << 3), %o1
 309         ldx             [%o2 + %o1], %o3
 310         andcc           %o3, 1, %g0
 311         be,pn           %icc, 2f
 312          andn           %o3, 1, %o3
 313         stxa            %g0, [%o3] ASI_IMMU_DEMAP
 314 2:      stxa            %g0, [%o3] ASI_DMMU_DEMAP
 315         membar          #Sync
 316         brnz,pt         %o1, 1b
 317          nop
 318         stxa            %g2, [%o4] ASI_DMMU
 319         sethi           %hi(KERNBASE), %o4
 320         flush           %o4
 321         wrpr            %g0, 0, %tl
 322         retl
 323          wrpr           %g7, 0x0, %pstate
 324
 325 __cheetah_flush_tlb_kernel_range:       /* 31 insns */
 326         /* %o0=start, %o1=end */
 327         cmp             %o0, %o1
 328         be,pn           %xcc, 2f
 329          sub            %o1, %o0, %o3
 330         srlx            %o3, 18, %o4
 331         brnz,pn         %o4, 3f
 332          sethi          %hi(PAGE_SIZE), %o4
 333         sub             %o3, %o4, %o3
 334         or              %o0, 0x20, %o0          ! Nucleus
 335 1:      stxa            %g0, [%o0 + %o3] ASI_DMMU_DEMAP
 336         stxa            %g0, [%o0 + %o3] ASI_IMMU_DEMAP
 337         membar          #Sync
 338         brnz,pt         %o3, 1b
 339          sub            %o3, %o4, %o3
 340 2:      sethi           %hi(KERNBASE), %o3
 341         flush           %o3
 342         retl
 343          nop
 344 3:      mov             0x80, %o4
 345         stxa            %g0, [%o4] ASI_DMMU_DEMAP
 346         membar          #Sync
 347         stxa            %g0, [%o4] ASI_IMMU_DEMAP
 348         membar          #Sync
 349         retl
 350          nop
 351         nop
 352         nop
 353         nop
 354         nop
 355         nop
 356         nop
 357         nop
 358
 359 #ifdef DCACHE_ALIASING_POSSIBLE
 360 __cheetah_flush_dcache_page: /* 11 insns */
 361         sethi           %hi(PAGE_OFFSET), %g1
 362         ldx             [%g1 + %lo(PAGE_OFFSET)], %g1
 363         sub             %o0, %g1, %o0
 364         sethi           %hi(PAGE_SIZE), %o4
 365 1:      subcc           %o4, (1 << 5), %o4
 366         stxa            %g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE
 367         membar          #Sync
 368         bne,pt          %icc, 1b
 369          nop
 370         retl            /* I-cache flush never needed on Cheetah, see callers. */
 371          nop
 372 #endif /* DCACHE_ALIASING_POSSIBLE */
 373
 374         /* Hypervisor specific versions, patched at boot time.  */
 375 __hypervisor_tlb_tl0_error:
 376         save            %sp, -192, %sp
 377         mov             %i0, %o0
 378         call            hypervisor_tlbop_error
 379          mov            %i1, %o1
 380         ret
 381          restore
 382
 383 __hypervisor_flush_tlb_mm: /* 19 insns */
 384         mov             %o0, %o2        /* ARG2: mmu context */
 385         mov             0, %o0          /* ARG0: CPU lists unimplemented */
 386         mov             0, %o1          /* ARG1: CPU lists unimplemented */
 387         mov             HV_MMU_ALL, %o3 /* ARG3: flags */
 388         mov             HV_FAST_MMU_DEMAP_CTX, %o5
 389         ta              HV_FAST_TRAP
 390         brnz,pn         %o0, 1f
 391          mov            HV_FAST_MMU_DEMAP_CTX, %o1
 392         retl
 393          nop
 394 1:      sethi           %hi(__hypervisor_tlb_tl0_error), %o5
 395         jmpl            %o5 + %lo(__hypervisor_tlb_tl0_error), %g0
 396          nop
 397         nop
 398         nop
 399         nop
 400         nop
 401         nop
 402         nop
 403
 404 __hypervisor_flush_tlb_page: /* 22 insns */
 405         /* %o0 = context, %o1 = vaddr */
 406         mov             %o0, %g2
 407         mov             %o1, %o0              /* ARG0: vaddr + IMMU-bit */
 408         mov             %g2, %o1              /* ARG1: mmu context */
 409         mov             HV_MMU_ALL, %o2       /* ARG2: flags */
 410         srlx            %o0, PAGE_SHIFT, %o0
 411         sllx            %o0, PAGE_SHIFT, %o0
 412         ta              HV_MMU_UNMAP_ADDR_TRAP
 413         brnz,pn         %o0, 1f
 414          mov            HV_MMU_UNMAP_ADDR_TRAP, %o1
 415         retl
 416          nop
 417 1:      sethi           %hi(__hypervisor_tlb_tl0_error), %o2
 418         jmpl            %o2 + %lo(__hypervisor_tlb_tl0_error), %g0
 419          nop
 420         nop
 421         nop
 422         nop
 423         nop
 424         nop
 425         nop
 426         nop
 427         nop
 428
 429 __hypervisor_flush_tlb_pending: /* 27 insns */
 430         /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
 431         sllx            %o1, 3, %g1
 432         mov             %o2, %g2
 433         mov             %o0, %g3
 434 1:      sub             %g1, (1 << 3), %g1
 435         ldx             [%g2 + %g1], %o0      /* ARG0: vaddr + IMMU-bit */
 436         mov             %g3, %o1              /* ARG1: mmu context */
 437         mov             HV_MMU_ALL, %o2       /* ARG2: flags */
 438         srlx            %o0, PAGE_SHIFT, %o0
 439         sllx            %o0, PAGE_SHIFT, %o0
 440         ta              HV_MMU_UNMAP_ADDR_TRAP
 441         brnz,pn         %o0, 1f
 442          mov            HV_MMU_UNMAP_ADDR_TRAP, %o1
 443         brnz,pt         %g1, 1b
 444          nop
 445         retl
 446          nop
 447 1:      sethi           %hi(__hypervisor_tlb_tl0_error), %o2
 448         jmpl            %o2 + %lo(__hypervisor_tlb_tl0_error), %g0
 449          nop
 450         nop
 451         nop
 452         nop
 453         nop
 454         nop
 455         nop
 456         nop
 457         nop
 458
 459 __hypervisor_flush_tlb_kernel_range: /* 31 insns */
 460         /* %o0=start, %o1=end */
 461         cmp             %o0, %o1
 462         be,pn           %xcc, 2f
 463          sub            %o1, %o0, %g2
 464         srlx            %g2, 18, %g3
 465         brnz,pn         %g3, 4f
 466          mov            %o0, %g1
 467         sethi           %hi(PAGE_SIZE), %g3
 468         sub             %g2, %g3, %g2
 469 1:      add             %g1, %g2, %o0   /* ARG0: virtual address */
 470         mov             0, %o1          /* ARG1: mmu context */
 471         mov             HV_MMU_ALL, %o2 /* ARG2: flags */
 472         ta              HV_MMU_UNMAP_ADDR_TRAP
 473         brnz,pn         %o0, 3f
 474          mov            HV_MMU_UNMAP_ADDR_TRAP, %o1
 475         brnz,pt         %g2, 1b
 476          sub            %g2, %g3, %g2
 477 2:      retl
 478          nop
 479 3:      sethi           %hi(__hypervisor_tlb_tl0_error), %o2
 480         jmpl            %o2 + %lo(__hypervisor_tlb_tl0_error), %g0
 481          nop
 482 4:      mov             0, %o0          /* ARG0: CPU lists unimplemented */
 483         mov             0, %o1          /* ARG1: CPU lists unimplemented */
 484         mov             0, %o2          /* ARG2: mmu context == nucleus */
 485         mov             HV_MMU_ALL, %o3 /* ARG3: flags */
 486         mov             HV_FAST_MMU_DEMAP_CTX, %o5
 487         ta              HV_FAST_TRAP
 488         brnz,pn         %o0, 3b
 489          mov            HV_FAST_MMU_DEMAP_CTX, %o1
 490         retl
 491          nop
 492
 493 #ifdef DCACHE_ALIASING_POSSIBLE
 494         /* XXX Niagara and friends have an 8K cache, so no aliasing is
 495          * XXX possible, but nothing explicit in the Hypervisor API
 496          * XXX guarantees this.
 497          */
 498 __hypervisor_flush_dcache_page: /* 2 insns */
 499         retl
 500          nop
 501 #endif
 502
 503 tlb_patch_one:
 504 1:      lduw            [%o1], %g1
 505         stw             %g1, [%o0]
 506         flush           %o0
 507         subcc           %o2, 1, %o2
 508         add             %o1, 4, %o1
 509         bne,pt          %icc, 1b
 510          add            %o0, 4, %o0
 511         retl
 512          nop
 513
 514 #ifdef CONFIG_SMP
 515         /* These are all called by the slaves of a cross call, at
 516          * trap level 1, with interrupts fully disabled.
 517          *
 518          * Register usage:
 519          *   %g5        mm->context     (all tlb flushes)
 520          *   %g1        address arg 1   (tlb page and range flushes)
 521          *   %g7        address arg 2   (tlb range flush only)
 522          *
 523          *   %g6        scratch 1
 524          *   %g2        scratch 2
 525          *   %g3        scratch 3
 526          *   %g4        scratch 4
 527          */
 528         .align          32
 529         .globl          xcall_flush_tlb_mm
 530 xcall_flush_tlb_mm:     /* 24 insns */
 531         mov             PRIMARY_CONTEXT, %g2
 532         ldxa            [%g2] ASI_DMMU, %g3
 533         srlx            %g3, CTX_PGSZ1_NUC_SHIFT, %g4
 534         sllx            %g4, CTX_PGSZ1_NUC_SHIFT, %g4
 535         or              %g5, %g4, %g5   /* Preserve nucleus page size fields */
 536         stxa            %g5, [%g2] ASI_DMMU
 537         mov             0x40, %g4
 538         stxa            %g0, [%g4] ASI_DMMU_DEMAP
 539         stxa            %g0, [%g4] ASI_IMMU_DEMAP
 540         stxa            %g3, [%g2] ASI_DMMU
 541         retry
 542         nop
 543         nop
 544         nop
 545         nop
 546         nop
 547         nop
 548         nop
 549         nop
 550         nop
 551         nop
 552         nop
 553         nop
 554         nop
 555
 556         .globl          xcall_flush_tlb_page
 557 xcall_flush_tlb_page:   /* 20 insns */
 558         /* %g5=context, %g1=vaddr */
 559         mov             PRIMARY_CONTEXT, %g4
 560         ldxa            [%g4] ASI_DMMU, %g2
 561         srlx            %g2, CTX_PGSZ1_NUC_SHIFT, %g4
 562         sllx            %g4, CTX_PGSZ1_NUC_SHIFT, %g4
 563         or              %g5, %g4, %g5
 564         mov             PRIMARY_CONTEXT, %g4
 565         stxa            %g5, [%g4] ASI_DMMU
 566         andcc           %g1, 0x1, %g0
 567         be,pn           %icc, 2f
 568          andn           %g1, 0x1, %g5
 569         stxa            %g0, [%g5] ASI_IMMU_DEMAP
 570 2:      stxa            %g0, [%g5] ASI_DMMU_DEMAP
 571         membar          #Sync
 572         stxa            %g2, [%g4] ASI_DMMU
 573         retry
 574         nop
 575         nop
 576         nop
 577         nop
 578         nop
 579
 580         .globl          xcall_flush_tlb_kernel_range
 581 xcall_flush_tlb_kernel_range:   /* 44 insns */
 582         sethi           %hi(PAGE_SIZE - 1), %g2
 583         or              %g2, %lo(PAGE_SIZE - 1), %g2
 584         andn            %g1, %g2, %g1
 585         andn            %g7, %g2, %g7
 586         sub             %g7, %g1, %g3
 587         srlx            %g3, 18, %g2
 588         brnz,pn         %g2, 2f
 589          add            %g2, 1, %g2
 590         sub             %g3, %g2, %g3
 591         or              %g1, 0x20, %g1          ! Nucleus
 592 1:      stxa            %g0, [%g1 + %g3] ASI_DMMU_DEMAP
 593         stxa            %g0, [%g1 + %g3] ASI_IMMU_DEMAP
 594         membar          #Sync
 595         brnz,pt         %g3, 1b
 596          sub            %g3, %g2, %g3
 597         retry
 598 2:      mov             63 * 8, %g1
 599 1:      ldxa            [%g1] ASI_ITLB_DATA_ACCESS, %g2
 600         andcc           %g2, 0x40, %g0                  /* _PAGE_L_4U */
 601         bne,pn          %xcc, 2f
 602          mov            TLB_TAG_ACCESS, %g2
 603         stxa            %g0, [%g2] ASI_IMMU
 604         stxa            %g0, [%g1] ASI_ITLB_DATA_ACCESS
 605         membar          #Sync
 606 2:      ldxa            [%g1] ASI_DTLB_DATA_ACCESS, %g2
 607         andcc           %g2, 0x40, %g0
 608         bne,pn          %xcc, 2f
 609          mov            TLB_TAG_ACCESS, %g2
 610         stxa            %g0, [%g2] ASI_DMMU
 611         stxa            %g0, [%g1] ASI_DTLB_DATA_ACCESS
 612         membar          #Sync
 613 2:      sub             %g1, 8, %g1
 614         brgez,pt        %g1, 1b
 615          nop
 616         retry
 617         nop
 618         nop
 619         nop
 620         nop
 621         nop
 622         nop
 623         nop
 624         nop
 625         nop
 626
 627         /* This runs in a very controlled environment, so we do
 628          * not need to worry about BH races etc.
 629          */
 630         .globl          xcall_sync_tick
 631 xcall_sync_tick:
 632
 633 661:    rdpr            %pstate, %g2
 634         wrpr            %g2, PSTATE_IG | PSTATE_AG, %pstate
 635         .section        .sun4v_2insn_patch, "ax"
 636         .word           661b
 637         nop
 638         nop
 639         .previous
 640
 641         rdpr            %pil, %g2
 642         wrpr            %g0, PIL_NORMAL_MAX, %pil
 643         sethi           %hi(109f), %g7
 644         b,pt            %xcc, etrap_irq
 645 109:     or             %g7, %lo(109b), %g7
 646 #ifdef CONFIG_TRACE_IRQFLAGS
 647         call            trace_hardirqs_off
 648          nop
 649 #endif
 650         call            smp_synchronize_tick_client
 651          nop
 652         b               rtrap_xcall
 653          ldx            [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1
 654
 655         .globl          xcall_fetch_glob_regs
 656 xcall_fetch_glob_regs:
 657         sethi           %hi(global_cpu_snapshot), %g1
 658         or              %g1, %lo(global_cpu_snapshot), %g1
 659         __GET_CPUID(%g2)
 660         sllx            %g2, 6, %g3
 661         add             %g1, %g3, %g1
 662         rdpr            %tstate, %g7
 663         stx             %g7, [%g1 + GR_SNAP_TSTATE]
 664         rdpr            %tpc, %g7
 665         stx             %g7, [%g1 + GR_SNAP_TPC]
 666         rdpr            %tnpc, %g7
 667         stx             %g7, [%g1 + GR_SNAP_TNPC]
 668         stx             %o7, [%g1 + GR_SNAP_O7]
 669         stx             %i7, [%g1 + GR_SNAP_I7]
 670         /* Don't try this at home kids... */
 671         rdpr            %cwp, %g3
 672         sub             %g3, 1, %g7
 673         wrpr            %g7, %cwp
 674         mov             %i7, %g7
 675         wrpr            %g3, %cwp
 676         stx             %g7, [%g1 + GR_SNAP_RPC]
 677         sethi           %hi(trap_block), %g7
 678         or              %g7, %lo(trap_block), %g7
 679         sllx            %g2, TRAP_BLOCK_SZ_SHIFT, %g2
 680         add             %g7, %g2, %g7
 681         ldx             [%g7 + TRAP_PER_CPU_THREAD], %g3
 682         stx             %g3, [%g1 + GR_SNAP_THREAD]
 683         retry
 684
 685         .globl          xcall_fetch_glob_pmu
 686 xcall_fetch_glob_pmu:
 687         sethi           %hi(global_cpu_snapshot), %g1
 688         or              %g1, %lo(global_cpu_snapshot), %g1
 689         __GET_CPUID(%g2)
 690         sllx            %g2, 6, %g3
 691         add             %g1, %g3, %g1
 692         rd              %pic, %g7
 693         stx             %g7, [%g1 + (4 * 8)]
 694         rd              %pcr, %g7
 695         stx             %g7, [%g1 + (0 * 8)]
 696         retry
 697
 698         .globl          xcall_fetch_glob_pmu_n4
 699 xcall_fetch_glob_pmu_n4:
 700         sethi           %hi(global_cpu_snapshot), %g1
 701         or              %g1, %lo(global_cpu_snapshot), %g1
 702         __GET_CPUID(%g2)
 703         sllx            %g2, 6, %g3
 704         add             %g1, %g3, %g1
 705
 706         ldxa            [%g0] ASI_PIC, %g7
 707         stx             %g7, [%g1 + (4 * 8)]
 708         mov             0x08, %g3
 709         ldxa            [%g3] ASI_PIC, %g7
 710         stx             %g7, [%g1 + (5 * 8)]
 711         mov             0x10, %g3
 712         ldxa            [%g3] ASI_PIC, %g7
 713         stx             %g7, [%g1 + (6 * 8)]
 714         mov             0x18, %g3
 715         ldxa            [%g3] ASI_PIC, %g7
 716         stx             %g7, [%g1 + (7 * 8)]
 717
 718         mov             %o0, %g2
 719         mov             %o1, %g3
 720         mov             %o5, %g7
 721
 722         mov             HV_FAST_VT_GET_PERFREG, %o5
 723         mov             3, %o0
 724         ta              HV_FAST_TRAP
 725         stx             %o1, [%g1 + (3 * 8)]
 726         mov             HV_FAST_VT_GET_PERFREG, %o5
 727         mov             2, %o0
 728         ta              HV_FAST_TRAP
 729         stx             %o1, [%g1 + (2 * 8)]
 730         mov             HV_FAST_VT_GET_PERFREG, %o5
 731         mov             1, %o0
 732         ta              HV_FAST_TRAP
 733         stx             %o1, [%g1 + (1 * 8)]
 734         mov             HV_FAST_VT_GET_PERFREG, %o5
 735         mov             0, %o0
 736         ta              HV_FAST_TRAP
 737         stx             %o1, [%g1 + (0 * 8)]
 738
 739         mov             %g2, %o0
 740         mov             %g3, %o1
 741         mov             %g7, %o5
 742
 743         retry
 744
 745 __cheetah_xcall_flush_tlb_kernel_range: /* 44 insns */
 746         sethi           %hi(PAGE_SIZE - 1), %g2
 747         or              %g2, %lo(PAGE_SIZE - 1), %g2
 748         andn            %g1, %g2, %g1
 749         andn            %g7, %g2, %g7
 750         sub             %g7, %g1, %g3
 751         srlx            %g3, 18, %g2
 752         brnz,pn         %g2, 2f
 753          add            %g2, 1, %g2
 754         sub             %g3, %g2, %g3
 755         or              %g1, 0x20, %g1          ! Nucleus
 756 1:      stxa            %g0, [%g1 + %g3] ASI_DMMU_DEMAP
 757         stxa            %g0, [%g1 + %g3] ASI_IMMU_DEMAP
 758         membar          #Sync
 759         brnz,pt         %g3, 1b
 760          sub            %g3, %g2, %g3
 761         retry
 762 2:      mov             0x80, %g2
 763         stxa            %g0, [%g2] ASI_DMMU_DEMAP
 764         membar          #Sync
 765         stxa            %g0, [%g2] ASI_IMMU_DEMAP
 766         membar          #Sync
 767         retry
 768         nop
 769         nop
 770         nop
 771         nop
 772         nop
 773         nop
 774         nop
 775         nop
 776         nop
 777         nop
 778         nop
 779         nop
 780         nop
 781         nop
 782         nop
 783         nop
 784         nop
 785         nop
 786         nop
 787         nop
 788         nop
 789         nop
 790
 791 #ifdef DCACHE_ALIASING_POSSIBLE
 792         .align          32
 793         .globl          xcall_flush_dcache_page_cheetah
 794 xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */
 795         sethi           %hi(PAGE_SIZE), %g3
 796 1:      subcc           %g3, (1 << 5), %g3
 797         stxa            %g0, [%g1 + %g3] ASI_DCACHE_INVALIDATE
 798         membar          #Sync
 799         bne,pt          %icc, 1b
 800          nop
 801         retry
 802         nop
 803 #endif /* DCACHE_ALIASING_POSSIBLE */
 804
 805         .globl          xcall_flush_dcache_page_spitfire
 806 xcall_flush_dcache_page_spitfire: /* %g1 == physical page address
 807                                      %g7 == kernel page virtual address
 808                                      %g5 == (page->mapping != NULL)  */
 809 #ifdef DCACHE_ALIASING_POSSIBLE
 810         srlx            %g1, (13 - 2), %g1      ! Form tag comparitor
 811         sethi           %hi(L1DCACHE_SIZE), %g3 ! D$ size == 16K
 812         sub             %g3, (1 << 5), %g3      ! D$ linesize == 32
 813 1:      ldxa            [%g3] ASI_DCACHE_TAG, %g2
 814         andcc           %g2, 0x3, %g0
 815         be,pn           %xcc, 2f
 816          andn           %g2, 0x3, %g2
 817         cmp             %g2, %g1
 818
 819         bne,pt          %xcc, 2f
 820          nop
 821         stxa            %g0, [%g3] ASI_DCACHE_TAG
 822         membar          #Sync
 823 2:      cmp             %g3, 0
 824         bne,pt          %xcc, 1b
 825          sub            %g3, (1 << 5), %g3
 826
 827         brz,pn          %g5, 2f
 828 #endif /* DCACHE_ALIASING_POSSIBLE */
 829          sethi          %hi(PAGE_SIZE), %g3
 830
 831 1:      flush           %g7
 832         subcc           %g3, (1 << 5), %g3
 833         bne,pt          %icc, 1b
 834          add            %g7, (1 << 5), %g7
 835
 836 2:      retry
 837         nop
 838         nop
 839
 840         /* %g5: error
 841          * %g6: tlb op
 842          */
 843 __hypervisor_tlb_xcall_error:
 844         mov     %g5, %g4
 845         mov     %g6, %g5
 846         ba,pt   %xcc, etrap
 847          rd     %pc, %g7
 848         mov     %l4, %o0
 849         call    hypervisor_tlbop_error_xcall
 850          mov    %l5, %o1
 851         ba,a,pt %xcc, rtrap
 852
 853         .globl          __hypervisor_xcall_flush_tlb_mm
 854 __hypervisor_xcall_flush_tlb_mm: /* 24 insns */
 855         /* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */
 856         mov             %o0, %g2
 857         mov             %o1, %g3
 858         mov             %o2, %g4
 859         mov             %o3, %g1
 860         mov             %o5, %g7
 861         clr             %o0             /* ARG0: CPU lists unimplemented */
 862         clr             %o1             /* ARG1: CPU lists unimplemented */
 863         mov             %g5, %o2        /* ARG2: mmu context */
 864         mov             HV_MMU_ALL, %o3 /* ARG3: flags */
 865         mov             HV_FAST_MMU_DEMAP_CTX, %o5
 866         ta              HV_FAST_TRAP
 867         mov             HV_FAST_MMU_DEMAP_CTX, %g6
 868         brnz,pn         %o0, 1f
 869          mov            %o0, %g5
 870         mov             %g2, %o0
 871         mov             %g3, %o1
 872         mov             %g4, %o2
 873         mov             %g1, %o3
 874         mov             %g7, %o5
 875         membar          #Sync
 876         retry
 877 1:      sethi           %hi(__hypervisor_tlb_xcall_error), %g4
 878         jmpl            %g4 + %lo(__hypervisor_tlb_xcall_error), %g0
 879          nop
 880
 881         .globl          __hypervisor_xcall_flush_tlb_page
 882 __hypervisor_xcall_flush_tlb_page: /* 20 insns */
 883         /* %g5=ctx, %g1=vaddr */
 884         mov             %o0, %g2
 885         mov             %o1, %g3
 886         mov             %o2, %g4
 887         mov             %g1, %o0                /* ARG0: virtual address */
 888         mov             %g5, %o1                /* ARG1: mmu context */
 889         mov             HV_MMU_ALL, %o2         /* ARG2: flags */
 890         srlx            %o0, PAGE_SHIFT, %o0
 891         sllx            %o0, PAGE_SHIFT, %o0
 892         ta              HV_MMU_UNMAP_ADDR_TRAP
 893         mov             HV_MMU_UNMAP_ADDR_TRAP, %g6
 894         brnz,a,pn       %o0, 1f
 895          mov            %o0, %g5
 896         mov             %g2, %o0
 897         mov             %g3, %o1
 898         mov             %g4, %o2
 899         membar          #Sync
 900         retry
 901 1:      sethi           %hi(__hypervisor_tlb_xcall_error), %g4
 902         jmpl            %g4 + %lo(__hypervisor_tlb_xcall_error), %g0
 903          nop
 904
 905         .globl          __hypervisor_xcall_flush_tlb_kernel_range
 906 __hypervisor_xcall_flush_tlb_kernel_range: /* 44 insns */
 907         /* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */
 908         sethi           %hi(PAGE_SIZE - 1), %g2
 909         or              %g2, %lo(PAGE_SIZE - 1), %g2
 910         andn            %g1, %g2, %g1
 911         andn            %g7, %g2, %g7
 912         sub             %g7, %g1, %g3
 913         srlx            %g3, 18, %g7
 914         add             %g2, 1, %g2
 915         sub             %g3, %g2, %g3
 916         mov             %o0, %g2
 917         mov             %o1, %g4
 918         brnz,pn         %g7, 2f
 919          mov            %o2, %g7
 920 1:      add             %g1, %g3, %o0   /* ARG0: virtual address */
 921         mov             0, %o1          /* ARG1: mmu context */
 922         mov             HV_MMU_ALL, %o2 /* ARG2: flags */
 923         ta              HV_MMU_UNMAP_ADDR_TRAP
 924         mov             HV_MMU_UNMAP_ADDR_TRAP, %g6
 925         brnz,pn         %o0, 1f
 926          mov            %o0, %g5
 927         sethi           %hi(PAGE_SIZE), %o2
 928         brnz,pt         %g3, 1b
 929          sub            %g3, %o2, %g3
 930 5:      mov             %g2, %o0
 931         mov             %g4, %o1
 932         mov             %g7, %o2
 933         membar          #Sync
 934         retry
 935 1:      sethi           %hi(__hypervisor_tlb_xcall_error), %g4
 936         jmpl            %g4 + %lo(__hypervisor_tlb_xcall_error), %g0
 937          nop
 938 2:      mov             %o3, %g1
 939         mov             %o5, %g3
 940         mov             0, %o0          /* ARG0: CPU lists unimplemented */
 941         mov             0, %o1          /* ARG1: CPU lists unimplemented */
 942         mov             0, %o2          /* ARG2: mmu context == nucleus */
 943         mov             HV_MMU_ALL, %o3 /* ARG3: flags */
 944         mov             HV_FAST_MMU_DEMAP_CTX, %o5
 945         ta              HV_FAST_TRAP
 946         mov             %g1, %o3
 947         brz,pt          %o0, 5b
 948          mov            %g3, %o5
 949         mov             HV_FAST_MMU_DEMAP_CTX, %g6
 950         ba,pt           %xcc, 1b
 951          clr            %g5
 952
 953         /* These just get rescheduled to PIL vectors. */
 954         .globl          xcall_call_function
 955 xcall_call_function:
 956         wr              %g0, (1 << PIL_SMP_CALL_FUNC), %set_softint
 957         retry
 958
 959         .globl          xcall_call_function_single
 960 xcall_call_function_single:
 961         wr              %g0, (1 << PIL_SMP_CALL_FUNC_SNGL), %set_softint
 962         retry
 963
 964         .globl          xcall_receive_signal
 965 xcall_receive_signal:
 966         wr              %g0, (1 << PIL_SMP_RECEIVE_SIGNAL), %set_softint
 967         retry
 968
 969         .globl          xcall_capture
 970 xcall_capture:
 971         wr              %g0, (1 << PIL_SMP_CAPTURE), %set_softint
 972         retry
 973
 974         .globl          xcall_new_mmu_context_version
 975 xcall_new_mmu_context_version:
 976         wr              %g0, (1 << PIL_SMP_CTX_NEW_VERSION), %set_softint
 977         retry
 978
 979 #ifdef CONFIG_KGDB
 980         .globl          xcall_kgdb_capture
 981 xcall_kgdb_capture:
 982         wr              %g0, (1 << PIL_KGDB_CAPTURE), %set_softint
 983         retry
 984 #endif
 985
 986 #endif /* CONFIG_SMP */
 987
 988         .globl          cheetah_patch_cachetlbops
 989 cheetah_patch_cachetlbops:
 990         save            %sp, -128, %sp
 991
 992         sethi           %hi(__flush_tlb_mm), %o0
 993         or              %o0, %lo(__flush_tlb_mm), %o0
 994         sethi           %hi(__cheetah_flush_tlb_mm), %o1
 995         or              %o1, %lo(__cheetah_flush_tlb_mm), %o1
 996         call            tlb_patch_one
 997          mov            19, %o2
 998
 999         sethi           %hi(__flush_tlb_page), %o0
1000         or              %o0, %lo(__flush_tlb_page), %o0
1001         sethi           %hi(__cheetah_flush_tlb_page), %o1
1002         or              %o1, %lo(__cheetah_flush_tlb_page), %o1
1003         call            tlb_patch_one
1004          mov            22, %o2
1005
1006         sethi           %hi(__flush_tlb_pending), %o0
1007         or              %o0, %lo(__flush_tlb_pending), %o0
1008         sethi           %hi(__cheetah_flush_tlb_pending), %o1
1009         or              %o1, %lo(__cheetah_flush_tlb_pending), %o1
1010         call            tlb_patch_one
1011          mov            27, %o2
1012
1013         sethi           %hi(__flush_tlb_kernel_range), %o0
1014         or              %o0, %lo(__flush_tlb_kernel_range), %o0
1015         sethi           %hi(__cheetah_flush_tlb_kernel_range), %o1
1016         or              %o1, %lo(__cheetah_flush_tlb_kernel_range), %o1
1017         call            tlb_patch_one
1018          mov            31, %o2
1019
1020 #ifdef DCACHE_ALIASING_POSSIBLE
1021         sethi           %hi(__flush_dcache_page), %o0
1022         or              %o0, %lo(__flush_dcache_page), %o0
1023         sethi           %hi(__cheetah_flush_dcache_page), %o1
1024         or              %o1, %lo(__cheetah_flush_dcache_page), %o1
1025         call            tlb_patch_one
1026          mov            11, %o2
1027 #endif /* DCACHE_ALIASING_POSSIBLE */
1028
1029 #ifdef CONFIG_SMP
1030         sethi           %hi(xcall_flush_tlb_kernel_range), %o0
1031         or              %o0, %lo(xcall_flush_tlb_kernel_range), %o0
1032         sethi           %hi(__cheetah_xcall_flush_tlb_kernel_range), %o1
1033         or              %o1, %lo(__cheetah_xcall_flush_tlb_kernel_range), %o1
1034         call            tlb_patch_one
1035          mov            44, %o2
1036 #endif /* CONFIG_SMP */
1037
1038         ret
1039          restore
1040
1041         .globl          hypervisor_patch_cachetlbops
1042 hypervisor_patch_cachetlbops:
1043         save            %sp, -128, %sp
1044
1045         sethi           %hi(__flush_tlb_mm), %o0
1046         or              %o0, %lo(__flush_tlb_mm), %o0
1047         sethi           %hi(__hypervisor_flush_tlb_mm), %o1
1048         or              %o1, %lo(__hypervisor_flush_tlb_mm), %o1
1049         call            tlb_patch_one
1050          mov            19, %o2
1051
1052         sethi           %hi(__flush_tlb_page), %o0
1053         or              %o0, %lo(__flush_tlb_page), %o0
1054         sethi           %hi(__hypervisor_flush_tlb_page), %o1
1055         or              %o1, %lo(__hypervisor_flush_tlb_page), %o1
1056         call            tlb_patch_one
1057          mov            22, %o2
1058
1059         sethi           %hi(__flush_tlb_pending), %o0
1060         or              %o0, %lo(__flush_tlb_pending), %o0
1061         sethi           %hi(__hypervisor_flush_tlb_pending), %o1
1062         or              %o1, %lo(__hypervisor_flush_tlb_pending), %o1
1063         call            tlb_patch_one
1064          mov            27, %o2
1065
1066         sethi           %hi(__flush_tlb_kernel_range), %o0
1067         or              %o0, %lo(__flush_tlb_kernel_range), %o0
1068         sethi           %hi(__hypervisor_flush_tlb_kernel_range), %o1
1069         or              %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1
1070         call            tlb_patch_one
1071          mov            31, %o2
1072
1073 #ifdef DCACHE_ALIASING_POSSIBLE
1074         sethi           %hi(__flush_dcache_page), %o0
1075         or              %o0, %lo(__flush_dcache_page), %o0
1076         sethi           %hi(__hypervisor_flush_dcache_page), %o1
1077         or              %o1, %lo(__hypervisor_flush_dcache_page), %o1
1078         call            tlb_patch_one
1079          mov            2, %o2
1080 #endif /* DCACHE_ALIASING_POSSIBLE */
1081
1082 #ifdef CONFIG_SMP
1083         sethi           %hi(xcall_flush_tlb_mm), %o0
1084         or              %o0, %lo(xcall_flush_tlb_mm), %o0
1085         sethi           %hi(__hypervisor_xcall_flush_tlb_mm), %o1
1086         or              %o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1
1087         call            tlb_patch_one
1088          mov            24, %o2
1089
1090         sethi           %hi(xcall_flush_tlb_page), %o0
1091         or              %o0, %lo(xcall_flush_tlb_page), %o0
1092         sethi           %hi(__hypervisor_xcall_flush_tlb_page), %o1
1093         or              %o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1
1094         call            tlb_patch_one
1095          mov            20, %o2
1096
1097         sethi           %hi(xcall_flush_tlb_kernel_range), %o0
1098         or              %o0, %lo(xcall_flush_tlb_kernel_range), %o0
1099         sethi           %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1
1100         or              %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1
1101         call            tlb_patch_one
1102          mov            44, %o2
1103 #endif /* CONFIG_SMP */
1104
1105         ret
1106          restore