arch/sparc64/mm/ultra.S

   1 /* $Id: ultra.S,v 1.72 2002/02/09 19:49:31 davem Exp $
   2  * ultra.S: Don't expand these all over the place...
   3  *
   4  * Copyright (C) 1997, 2000 David S. Miller (davem@redhat.com)
   5  */
   6
   7 #include <linux/config.h>
   8 #include <asm/asi.h>
   9 #include <asm/pgtable.h>
  10 #include <asm/page.h>
  11 #include <asm/spitfire.h>
  12 #include <asm/mmu_context.h>
  13 #include <asm/mmu.h>
  14 #include <asm/pil.h>
  15 #include <asm/head.h>
  16 #include <asm/thread_info.h>
  17 #include <asm/cacheflush.h>
  18
  19         /* Basically, most of the Spitfire vs. Cheetah madness
  20          * has to do with the fact that Cheetah does not support
  21          * IMMU flushes out of the secondary context.  Someone needs
  22          * to throw a south lake birthday party for the folks
  23          * in Microelectronics who refused to fix this shit.
  24          */
  25
  26         /* This file is meant to be read efficiently by the CPU, not humans.
  27          * Staraj sie tego nikomu nie pierdolnac...
  28          */
  29         .text
  30         .align          32
  31         .globl          __flush_tlb_mm
  32 __flush_tlb_mm: /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */
  33         ldxa            [%o1] ASI_DMMU, %g2
  34         cmp             %g2, %o0
  35         bne,pn          %icc, __spitfire_flush_tlb_mm_slow
  36          mov            0x50, %g3
  37         stxa            %g0, [%g3] ASI_DMMU_DEMAP
  38         stxa            %g0, [%g3] ASI_IMMU_DEMAP
  39         retl
  40          flush          %g6
  41         nop
  42         nop
  43         nop
  44         nop
  45         nop
  46         nop
  47         nop
  48         nop
  49         nop
  50         nop
  51
  52         .align          32
  53         .globl          __flush_tlb_pending
  54 __flush_tlb_pending:
  55         /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
  56         rdpr            %pstate, %g7
  57         sllx            %o1, 3, %o1
  58         andn            %g7, PSTATE_IE, %g2
  59         wrpr            %g2, %pstate
  60         mov             SECONDARY_CONTEXT, %o4
  61         ldxa            [%o4] ASI_DMMU, %g2
  62         stxa            %o0, [%o4] ASI_DMMU
  63 1:      sub             %o1, (1 << 3), %o1
  64         ldx             [%o2 + %o1], %o3
  65         andcc           %o3, 1, %g0
  66         andn            %o3, 1, %o3
  67         be,pn           %icc, 2f
  68          or             %o3, 0x10, %o3
  69         stxa            %g0, [%o3] ASI_IMMU_DEMAP
  70 2:      stxa            %g0, [%o3] ASI_DMMU_DEMAP
  71         membar          #Sync
  72         brnz,pt         %o1, 1b
  73          nop
  74         stxa            %g2, [%o4] ASI_DMMU
  75         flush           %g6
  76         retl
  77          wrpr           %g7, 0x0, %pstate
  78         nop
  79         nop
  80         nop
  81         nop
  82
  83         .align          32
  84         .globl          __flush_tlb_kernel_range
  85 __flush_tlb_kernel_range:       /* %o0=start, %o1=end */
  86         cmp             %o0, %o1
  87         be,pn           %xcc, 2f
  88          sethi          %hi(PAGE_SIZE), %o4
  89         sub             %o1, %o0, %o3
  90         sub             %o3, %o4, %o3
  91         or              %o0, 0x20, %o0          ! Nucleus
  92 1:      stxa            %g0, [%o0 + %o3] ASI_DMMU_DEMAP
  93         stxa            %g0, [%o0 + %o3] ASI_IMMU_DEMAP
  94         membar          #Sync
  95         brnz,pt         %o3, 1b
  96          sub            %o3, %o4, %o3
  97 2:      retl
  98          flush          %g6
  99
 100 __spitfire_flush_tlb_mm_slow:
 101         rdpr            %pstate, %g1
 102         wrpr            %g1, PSTATE_IE, %pstate
 103         stxa            %o0, [%o1] ASI_DMMU
 104         stxa            %g0, [%g3] ASI_DMMU_DEMAP
 105         stxa            %g0, [%g3] ASI_IMMU_DEMAP
 106         flush           %g6
 107         stxa            %g2, [%o1] ASI_DMMU
 108         flush           %g6
 109         retl
 110          wrpr           %g1, 0, %pstate
 111
 112 /*
 113  * The following code flushes one page_size worth.
 114  */
 115 #if (PAGE_SHIFT == 13)
 116 #define ITAG_MASK 0xfe
 117 #elif (PAGE_SHIFT == 16)
 118 #define ITAG_MASK 0x7fe
 119 #else
 120 #error unsupported PAGE_SIZE
 121 #endif
 122         .align          32
 123         .globl          __flush_icache_page
 124 __flush_icache_page:    /* %o0 = phys_page */
 125         membar          #StoreStore
 126         srlx            %o0, PAGE_SHIFT, %o0
 127         sethi           %uhi(PAGE_OFFSET), %g1
 128         sllx            %o0, PAGE_SHIFT, %o0
 129         sethi           %hi(PAGE_SIZE), %g2
 130         sllx            %g1, 32, %g1
 131         add             %o0, %g1, %o0
 132 1:      subcc           %g2, 32, %g2
 133         bne,pt          %icc, 1b
 134          flush          %o0 + %g2
 135         retl
 136          nop
 137
 138 #ifdef DCACHE_ALIASING_POSSIBLE
 139
 140 #if (PAGE_SHIFT != 13)
 141 #error only page shift of 13 is supported by dcache flush
 142 #endif
 143
 144 #define DTAG_MASK 0x3
 145
 146         .align          64
 147         .globl          __flush_dcache_page
 148 __flush_dcache_page:    /* %o0=kaddr, %o1=flush_icache */
 149         sethi           %uhi(PAGE_OFFSET), %g1
 150         sllx            %g1, 32, %g1
 151         sub             %o0, %g1, %o0
 152         clr             %o4
 153         srlx            %o0, 11, %o0
 154         sethi           %hi(1 << 14), %o2
 155 1:      ldxa            [%o4] ASI_DCACHE_TAG, %o3       ! LSU   Group
 156         add             %o4, (1 << 5), %o4              ! IEU0
 157         ldxa            [%o4] ASI_DCACHE_TAG, %g1       ! LSU   Group
 158         add             %o4, (1 << 5), %o4              ! IEU0
 159         ldxa            [%o4] ASI_DCACHE_TAG, %g2       ! LSU   Group   o3 available
 160         add             %o4, (1 << 5), %o4              ! IEU0
 161         andn            %o3, DTAG_MASK, %o3             ! IEU1
 162         ldxa            [%o4] ASI_DCACHE_TAG, %g3       ! LSU   Group
 163         add             %o4, (1 << 5), %o4              ! IEU0
 164         andn            %g1, DTAG_MASK, %g1             ! IEU1
 165         cmp             %o0, %o3                        ! IEU1  Group
 166         be,a,pn         %xcc, dflush1                   ! CTI
 167          sub            %o4, (4 << 5), %o4              ! IEU0  (Group)
 168         cmp             %o0, %g1                        ! IEU1  Group
 169         andn            %g2, DTAG_MASK, %g2             ! IEU0
 170         be,a,pn         %xcc, dflush2                   ! CTI
 171          sub            %o4, (3 << 5), %o4              ! IEU0  (Group)
 172         cmp             %o0, %g2                        ! IEU1  Group
 173         andn            %g3, DTAG_MASK, %g3             ! IEU0
 174         be,a,pn         %xcc, dflush3                   ! CTI
 175          sub            %o4, (2 << 5), %o4              ! IEU0  (Group)
 176         cmp             %o0, %g3                        ! IEU1  Group
 177         be,a,pn         %xcc, dflush4                   ! CTI
 178          sub            %o4, (1 << 5), %o4              ! IEU0
 179 2:      cmp             %o4, %o2                        ! IEU1  Group
 180         bne,pt          %xcc, 1b                        ! CTI
 181          nop                                            ! IEU0
 182
 183         /* The I-cache does not snoop local stores so we
 184          * better flush that too when necessary.
 185          */
 186         brnz,pt         %o1, __flush_icache_page
 187          sllx           %o0, 11, %o0
 188         retl
 189          nop
 190
 191 dflush1:stxa            %g0, [%o4] ASI_DCACHE_TAG
 192         add             %o4, (1 << 5), %o4
 193 dflush2:stxa            %g0, [%o4] ASI_DCACHE_TAG
 194         add             %o4, (1 << 5), %o4
 195 dflush3:stxa            %g0, [%o4] ASI_DCACHE_TAG
 196         add             %o4, (1 << 5), %o4
 197 dflush4:stxa            %g0, [%o4] ASI_DCACHE_TAG
 198         add             %o4, (1 << 5), %o4
 199         membar          #Sync
 200         ba,pt           %xcc, 2b
 201          nop
 202 #endif /* DCACHE_ALIASING_POSSIBLE */
 203
 204         .align          32
 205 __prefill_dtlb:
 206         rdpr            %pstate, %g7
 207         wrpr            %g7, PSTATE_IE, %pstate
 208         mov             TLB_TAG_ACCESS, %g1
 209         stxa            %o5, [%g1] ASI_DMMU
 210         stxa            %o2, [%g0] ASI_DTLB_DATA_IN
 211         flush           %g6
 212         retl
 213          wrpr           %g7, %pstate
 214 __prefill_itlb:
 215         rdpr            %pstate, %g7
 216         wrpr            %g7, PSTATE_IE, %pstate
 217         mov             TLB_TAG_ACCESS, %g1
 218         stxa            %o5, [%g1] ASI_IMMU
 219         stxa            %o2, [%g0] ASI_ITLB_DATA_IN
 220         flush           %g6
 221         retl
 222          wrpr           %g7, %pstate
 223
 224         .globl          __update_mmu_cache
 225 __update_mmu_cache:     /* %o0=hw_context, %o1=address, %o2=pte, %o3=fault_code */
 226         srlx            %o1, PAGE_SHIFT, %o1
 227         andcc           %o3, FAULT_CODE_DTLB, %g0
 228         sllx            %o1, PAGE_SHIFT, %o5
 229         bne,pt          %xcc, __prefill_dtlb
 230          or             %o5, %o0, %o5
 231         ba,a,pt         %xcc, __prefill_itlb
 232
 233         /* Cheetah specific versions, patched at boot time. */
 234 __cheetah_flush_tlb_mm: /* 18 insns */
 235         rdpr            %pstate, %g7
 236         andn            %g7, PSTATE_IE, %g2
 237         wrpr            %g2, 0x0, %pstate
 238         wrpr            %g0, 1, %tl
 239         mov             PRIMARY_CONTEXT, %o2
 240         mov             0x40, %g3
 241         ldxa            [%o2] ASI_DMMU, %g2
 242         srlx            %g2, CTX_PGSZ1_NUC_SHIFT, %o1
 243         sllx            %o1, CTX_PGSZ1_NUC_SHIFT, %o1
 244         or              %o0, %o1, %o0   /* Preserve nucleus page size fields */
 245         stxa            %o0, [%o2] ASI_DMMU
 246         stxa            %g0, [%g3] ASI_DMMU_DEMAP
 247         stxa            %g0, [%g3] ASI_IMMU_DEMAP
 248         stxa            %g2, [%o2] ASI_DMMU
 249         flush           %g6
 250         wrpr            %g0, 0, %tl
 251         retl
 252          wrpr           %g7, 0x0, %pstate
 253
 254 __cheetah_flush_tlb_pending:    /* 26 insns */
 255         /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
 256         rdpr            %pstate, %g7
 257         sllx            %o1, 3, %o1
 258         andn            %g7, PSTATE_IE, %g2
 259         wrpr            %g2, 0x0, %pstate
 260         wrpr            %g0, 1, %tl
 261         mov             PRIMARY_CONTEXT, %o4
 262         ldxa            [%o4] ASI_DMMU, %g2
 263         srlx            %g2, CTX_PGSZ1_NUC_SHIFT, %o3
 264         sllx            %o3, CTX_PGSZ1_NUC_SHIFT, %o3
 265         or              %o0, %o3, %o0   /* Preserve nucleus page size fields */
 266         stxa            %o0, [%o4] ASI_DMMU
 267 1:      sub             %o1, (1 << 3), %o1
 268         ldx             [%o2 + %o1], %o3
 269         andcc           %o3, 1, %g0
 270         be,pn           %icc, 2f
 271          andn           %o3, 1, %o3
 272         stxa            %g0, [%o3] ASI_IMMU_DEMAP
 273 2:      stxa            %g0, [%o3] ASI_DMMU_DEMAP
 274         membar          #Sync
 275         brnz,pt         %o1, 1b
 276          nop
 277         stxa            %g2, [%o4] ASI_DMMU
 278         flush           %g6
 279         wrpr            %g0, 0, %tl
 280         retl
 281          wrpr           %g7, 0x0, %pstate
 282
 283 #ifdef DCACHE_ALIASING_POSSIBLE
 284 flush_dcpage_cheetah: /* 11 insns */
 285         sethi           %uhi(PAGE_OFFSET), %g1
 286         sllx            %g1, 32, %g1
 287         sub             %o0, %g1, %o0
 288         sethi           %hi(PAGE_SIZE), %o4
 289 1:      subcc           %o4, (1 << 5), %o4
 290         stxa            %g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE
 291         membar          #Sync
 292         bne,pt          %icc, 1b
 293          nop
 294         retl            /* I-cache flush never needed on Cheetah, see callers. */
 295          nop
 296 #endif /* DCACHE_ALIASING_POSSIBLE */
 297
 298 cheetah_patch_one:
 299 1:      lduw            [%o1], %g1
 300         stw             %g1, [%o0]
 301         flush           %o0
 302         subcc           %o2, 1, %o2
 303         add             %o1, 4, %o1
 304         bne,pt          %icc, 1b
 305          add            %o0, 4, %o0
 306         retl
 307          nop
 308
 309         .globl          cheetah_patch_cachetlbops
 310 cheetah_patch_cachetlbops:
 311         save            %sp, -128, %sp
 312
 313         sethi           %hi(__flush_tlb_mm), %o0
 314         or              %o0, %lo(__flush_tlb_mm), %o0
 315         sethi           %hi(__cheetah_flush_tlb_mm), %o1
 316         or              %o1, %lo(__cheetah_flush_tlb_mm), %o1
 317         call            cheetah_patch_one
 318          mov            18, %o2
 319
 320         sethi           %hi(__flush_tlb_pending), %o0
 321         or              %o0, %lo(__flush_tlb_pending), %o0
 322         sethi           %hi(__cheetah_flush_tlb_pending), %o1
 323         or              %o1, %lo(__cheetah_flush_tlb_pending), %o1
 324         call            cheetah_patch_one
 325          mov            26, %o2
 326
 327 #ifdef DCACHE_ALIASING_POSSIBLE
 328         sethi           %hi(__flush_dcache_page), %o0
 329         or              %o0, %lo(__flush_dcache_page), %o0
 330         sethi           %hi(flush_dcpage_cheetah), %o1
 331         or              %o1, %lo(flush_dcpage_cheetah), %o1
 332         call            cheetah_patch_one
 333          mov            11, %o2
 334 #endif /* DCACHE_ALIASING_POSSIBLE */
 335
 336         ret
 337          restore
 338
 339 #ifdef CONFIG_SMP
 340         /* These are all called by the slaves of a cross call, at
 341          * trap level 1, with interrupts fully disabled.
 342          *
 343          * Register usage:
 344          *   %g5        mm->context     (all tlb flushes)
 345          *   %g1        address arg 1   (tlb page and range flushes)
 346          *   %g7        address arg 2   (tlb range flush only)
 347          *
 348          *   %g6        ivector table, don't touch
 349          *   %g2        scratch 1
 350          *   %g3        scratch 2
 351          *   %g4        scratch 3
 352          *
 353          * TODO: Make xcall TLB range flushes use the tricks above... -DaveM
 354          */
 355         .align          32
 356         .globl          xcall_flush_tlb_mm
 357 xcall_flush_tlb_mm:
 358         mov             PRIMARY_CONTEXT, %g2
 359         ldxa            [%g2] ASI_DMMU, %g3
 360         srlx            %g3, CTX_PGSZ1_NUC_SHIFT, %g4
 361         sllx            %g4, CTX_PGSZ1_NUC_SHIFT, %g4
 362         or              %g5, %g4, %g5   /* Preserve nucleus page size fields */
 363         stxa            %g5, [%g2] ASI_DMMU
 364         mov             0x40, %g4
 365         stxa            %g0, [%g4] ASI_DMMU_DEMAP
 366         stxa            %g0, [%g4] ASI_IMMU_DEMAP
 367         stxa            %g3, [%g2] ASI_DMMU
 368         retry
 369
 370         .globl          xcall_flush_tlb_pending
 371 xcall_flush_tlb_pending:
 372         /* %g5=context, %g1=nr, %g7=vaddrs[] */
 373         sllx            %g1, 3, %g1
 374         mov             PRIMARY_CONTEXT, %g4
 375         ldxa            [%g4] ASI_DMMU, %g2
 376         srlx            %g2, CTX_PGSZ1_NUC_SHIFT, %g4
 377         sllx            %g4, CTX_PGSZ1_NUC_SHIFT, %g4
 378         or              %g5, %g4, %g5
 379         mov             PRIMARY_CONTEXT, %g4
 380         stxa            %g5, [%g4] ASI_DMMU
 381 1:      sub             %g1, (1 << 3), %g1
 382         ldx             [%g7 + %g1], %g5
 383         andcc           %g5, 0x1, %g0
 384         be,pn           %icc, 2f
 385
 386          andn           %g5, 0x1, %g5
 387         stxa            %g0, [%g5] ASI_IMMU_DEMAP
 388 2:      stxa            %g0, [%g5] ASI_DMMU_DEMAP
 389         membar          #Sync
 390         brnz,pt         %g1, 1b
 391          nop
 392         stxa            %g2, [%g4] ASI_DMMU
 393         retry
 394
 395         .globl          xcall_flush_tlb_kernel_range
 396 xcall_flush_tlb_kernel_range:
 397         sethi           %hi(PAGE_SIZE - 1), %g2
 398         or              %g2, %lo(PAGE_SIZE - 1), %g2
 399         andn            %g1, %g2, %g1
 400         andn            %g7, %g2, %g7
 401         sub             %g7, %g1, %g3
 402         add             %g2, 1, %g2
 403         sub             %g3, %g2, %g3
 404         or              %g1, 0x20, %g1          ! Nucleus
 405 1:      stxa            %g0, [%g1 + %g3] ASI_DMMU_DEMAP
 406         stxa            %g0, [%g1 + %g3] ASI_IMMU_DEMAP
 407         membar          #Sync
 408         brnz,pt         %g3, 1b
 409          sub            %g3, %g2, %g3
 410         retry
 411         nop
 412         nop
 413
 414         /* This runs in a very controlled environment, so we do
 415          * not need to worry about BH races etc.
 416          */
 417         .globl          xcall_sync_tick
 418 xcall_sync_tick:
 419         rdpr            %pstate, %g2
 420         wrpr            %g2, PSTATE_IG | PSTATE_AG, %pstate
 421         rdpr            %pil, %g2
 422         wrpr            %g0, 15, %pil
 423         sethi           %hi(109f), %g7
 424         b,pt            %xcc, etrap_irq
 425 109:     or             %g7, %lo(109b), %g7
 426         call            smp_synchronize_tick_client
 427          nop
 428         clr             %l6
 429         b               rtrap_xcall
 430          ldx            [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1
 431
 432         /* NOTE: This is SPECIAL!!  We do etrap/rtrap however
 433          *       we choose to deal with the "BH's run with
 434          *       %pil==15" problem (described in asm/pil.h)
 435          *       by just invoking rtrap directly past where
 436          *       BH's are checked for.
 437          *
 438          *       We do it like this because we do not want %pil==15
 439          *       lockups to prevent regs being reported.
 440          */
 441         .globl          xcall_report_regs
 442 xcall_report_regs:
 443         rdpr            %pstate, %g2
 444         wrpr            %g2, PSTATE_IG | PSTATE_AG, %pstate
 445         rdpr            %pil, %g2
 446         wrpr            %g0, 15, %pil
 447         sethi           %hi(109f), %g7
 448         b,pt            %xcc, etrap_irq
 449 109:     or             %g7, %lo(109b), %g7
 450         call            __show_regs
 451          add            %sp, PTREGS_OFF, %o0
 452         clr             %l6
 453         /* Has to be a non-v9 branch due to the large distance. */
 454         b               rtrap_xcall
 455          ldx            [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1
 456
 457 #ifdef DCACHE_ALIASING_POSSIBLE
 458         .align          32
 459         .globl          xcall_flush_dcache_page_cheetah
 460 xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */
 461         sethi           %hi(PAGE_SIZE), %g3
 462 1:      subcc           %g3, (1 << 5), %g3
 463         stxa            %g0, [%g1 + %g3] ASI_DCACHE_INVALIDATE
 464         membar          #Sync
 465         bne,pt          %icc, 1b
 466          nop
 467         retry
 468         nop
 469 #endif /* DCACHE_ALIASING_POSSIBLE */
 470
 471         .globl          xcall_flush_dcache_page_spitfire
 472 xcall_flush_dcache_page_spitfire: /* %g1 == physical page address
 473                                      %g7 == kernel page virtual address
 474                                      %g5 == (page->mapping != NULL)  */
 475 #ifdef DCACHE_ALIASING_POSSIBLE
 476         srlx            %g1, (13 - 2), %g1      ! Form tag comparitor
 477         sethi           %hi(L1DCACHE_SIZE), %g3 ! D$ size == 16K
 478         sub             %g3, (1 << 5), %g3      ! D$ linesize == 32
 479 1:      ldxa            [%g3] ASI_DCACHE_TAG, %g2
 480         andcc           %g2, 0x3, %g0
 481         be,pn           %xcc, 2f
 482          andn           %g2, 0x3, %g2
 483         cmp             %g2, %g1
 484
 485         bne,pt          %xcc, 2f
 486          nop
 487         stxa            %g0, [%g3] ASI_DCACHE_TAG
 488         membar          #Sync
 489 2:      cmp             %g3, 0
 490         bne,pt          %xcc, 1b
 491          sub            %g3, (1 << 5), %g3
 492
 493         brz,pn          %g5, 2f
 494 #endif /* DCACHE_ALIASING_POSSIBLE */
 495          sethi          %hi(PAGE_SIZE), %g3
 496
 497 1:      flush           %g7
 498         subcc           %g3, (1 << 5), %g3
 499         bne,pt          %icc, 1b
 500          add            %g7, (1 << 5), %g7
 501
 502 2:      retry
 503         nop
 504         nop
 505
 506         .globl          xcall_promstop
 507 xcall_promstop:
 508         rdpr            %pstate, %g2
 509         wrpr            %g2, PSTATE_IG | PSTATE_AG, %pstate
 510         rdpr            %pil, %g2
 511         wrpr            %g0, 15, %pil
 512         sethi           %hi(109f), %g7
 513         b,pt            %xcc, etrap_irq
 514 109:     or             %g7, %lo(109b), %g7
 515         flushw
 516         call            prom_stopself
 517          nop
 518         /* We should not return, just spin if we do... */
 519 1:      b,a,pt          %xcc, 1b
 520         nop
 521
 522         .data
 523
 524 errata32_hwbug:
 525         .xword  0
 526
 527         .text
 528
 529         /* These two are not performance critical... */
 530         .globl          xcall_flush_tlb_all_spitfire
 531 xcall_flush_tlb_all_spitfire:
 532         /* Spitfire Errata #32 workaround. */
 533         sethi           %hi(errata32_hwbug), %g4
 534         stx             %g0, [%g4 + %lo(errata32_hwbug)]
 535
 536         clr             %g2
 537         clr             %g3
 538 1:      ldxa            [%g3] ASI_DTLB_DATA_ACCESS, %g4
 539         and             %g4, _PAGE_L, %g5
 540         brnz,pn         %g5, 2f
 541          mov            TLB_TAG_ACCESS, %g7
 542
 543         stxa            %g0, [%g7] ASI_DMMU
 544         membar          #Sync
 545         stxa            %g0, [%g3] ASI_DTLB_DATA_ACCESS
 546         membar          #Sync
 547
 548         /* Spitfire Errata #32 workaround. */
 549         sethi           %hi(errata32_hwbug), %g4
 550         stx             %g0, [%g4 + %lo(errata32_hwbug)]
 551
 552 2:      ldxa            [%g3] ASI_ITLB_DATA_ACCESS, %g4
 553         and             %g4, _PAGE_L, %g5
 554         brnz,pn         %g5, 2f
 555          mov            TLB_TAG_ACCESS, %g7
 556
 557         stxa            %g0, [%g7] ASI_IMMU
 558         membar          #Sync
 559         stxa            %g0, [%g3] ASI_ITLB_DATA_ACCESS
 560         membar          #Sync
 561
 562         /* Spitfire Errata #32 workaround. */
 563         sethi           %hi(errata32_hwbug), %g4
 564         stx             %g0, [%g4 + %lo(errata32_hwbug)]
 565
 566 2:      add             %g2, 1, %g2
 567         cmp             %g2, SPITFIRE_HIGHEST_LOCKED_TLBENT
 568         ble,pt          %icc, 1b
 569          sll            %g2, 3, %g3
 570         flush           %g6
 571         retry
 572
 573         .globl          xcall_flush_tlb_all_cheetah
 574 xcall_flush_tlb_all_cheetah:
 575         mov             0x80, %g2
 576         stxa            %g0, [%g2] ASI_DMMU_DEMAP
 577         stxa            %g0, [%g2] ASI_IMMU_DEMAP
 578         retry
 579
 580         /* These just get rescheduled to PIL vectors. */
 581         .globl          xcall_call_function
 582 xcall_call_function:
 583         wr              %g0, (1 << PIL_SMP_CALL_FUNC), %set_softint
 584         retry
 585
 586         .globl          xcall_receive_signal
 587 xcall_receive_signal:
 588         wr              %g0, (1 << PIL_SMP_RECEIVE_SIGNAL), %set_softint
 589         retry
 590
 591         .globl          xcall_capture
 592 xcall_capture:
 593         wr              %g0, (1 << PIL_SMP_CAPTURE), %set_softint
 594         retry
 595
 596 #endif /* CONFIG_SMP */