projects / mirror_ubuntu-bionic-kernel.git / blame
| Commit | Line | Data |
|---|---|---|
| b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
| b00dc837 | 2 | /* |
| 1da177e4 LT |
3 | * ultra.S: Don't expand these all over the place... |
| 4 | * | |
| 93dae5b7 | 5 | * Copyright (C) 1997, 2000, 2008 David S. Miller (davem@davemloft.net) |
| 1da177e4 LT |
6 | */ |
| 7 | ||
| 1da177e4 LT |
8 | #include <asm/asi.h> |
| 9 | #include <asm/pgtable.h> | |
| 10 | #include <asm/page.h> | |
| 11 | #include <asm/spitfire.h> | |
| 12 | #include <asm/mmu_context.h> | |
| 2ef27778 | 13 | #include <asm/mmu.h> |
| 1da177e4 LT |
14 | #include <asm/pil.h> |
| 15 | #include <asm/head.h> | |
| 16 | #include <asm/thread_info.h> | |
| 17 | #include <asm/cacheflush.h> | |
| 52bf082f | 18 | #include <asm/hypervisor.h> |
| 93dae5b7 | 19 | #include <asm/cpudata.h> |
| 1da177e4 LT |
20 | |
| 21 | /* Basically, most of the Spitfire vs. Cheetah madness | |
| 22 | * has to do with the fact that Cheetah does not support | |
| 23 | * IMMU flushes out of the secondary context. Someone needs | |
| 24 | * to throw a south lake birthday party for the folks | |
| 25 | * in Microelectronics who refused to fix this shit. | |
| 26 | */ | |
| 27 | ||
| 28 | /* This file is meant to be read efficiently by the CPU, not humans. | |
| 29 | * Staraj sie tego nikomu nie pierdolnac... | |
| 30 | */ | |
| 31 | .text | |
| 32 | .align 32 | |
| 33 | .globl __flush_tlb_mm | |
| b429ae4d | 34 | __flush_tlb_mm: /* 19 insns */ |
| 52bf082f | 35 | /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */ |
| 1da177e4 LT |
36 | ldxa [%o1] ASI_DMMU, %g2 |
| 37 | cmp %g2, %o0 | |
| 38 | bne,pn %icc, __spitfire_flush_tlb_mm_slow | |
| 39 | mov 0x50, %g3 | |
| 40 | stxa %g0, [%g3] ASI_DMMU_DEMAP | |
| 41 | stxa %g0, [%g3] ASI_IMMU_DEMAP | |
| 4da808c3 DM |
42 | sethi %hi(KERNBASE), %g3 |
| 43 | flush %g3 | |
| 1da177e4 | 44 | retl |
| 4da808c3 | 45 | nop |
| 1da177e4 LT |
46 | nop |
| 47 | nop | |
| 48 | nop | |
| 49 | nop | |
| 50 | nop | |
| 51 | nop | |
| 52 | nop | |
| 2ef27778 DM |
53 | nop |
| 54 | nop | |
| 1da177e4 | 55 | |
| f36391d2 DM |
56 | .align 32 |
| 57 | .globl __flush_tlb_page | |
| 58 | __flush_tlb_page: /* 22 insns */ | |
| 59 | /* %o0 = context, %o1 = vaddr */ | |
| 60 | rdpr %pstate, %g7 | |
| 61 | andn %g7, PSTATE_IE, %g2 | |
| 62 | wrpr %g2, %pstate | |
| 63 | mov SECONDARY_CONTEXT, %o4 | |
| 64 | ldxa [%o4] ASI_DMMU, %g2 | |
| 65 | stxa %o0, [%o4] ASI_DMMU | |
| 66 | andcc %o1, 1, %g0 | |
| 67 | andn %o1, 1, %o3 | |
| 68 | be,pn %icc, 1f | |
| 69 | or %o3, 0x10, %o3 | |
| 70 | stxa %g0, [%o3] ASI_IMMU_DEMAP | |
| 71 | 1: stxa %g0, [%o3] ASI_DMMU_DEMAP | |
| 72 | membar #Sync | |
| 73 | stxa %g2, [%o4] ASI_DMMU | |
| 74 | sethi %hi(KERNBASE), %o4 | |
| 75 | flush %o4 | |
| 76 | retl | |
| 77 | wrpr %g7, 0x0, %pstate | |
| 78 | nop | |
| 79 | nop | |
| 80 | nop | |
| 81 | nop | |
| 82 | ||
| 1da177e4 LT |
83 | .align 32 |
| 84 | .globl __flush_tlb_pending | |
| b429ae4d | 85 | __flush_tlb_pending: /* 27 insns */ |
| 1da177e4 LT |
86 | /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ |
| 87 | rdpr %pstate, %g7 | |
| 88 | sllx %o1, 3, %o1 | |
| 89 | andn %g7, PSTATE_IE, %g2 | |
| 90 | wrpr %g2, %pstate | |
| 91 | mov SECONDARY_CONTEXT, %o4 | |
| 92 | ldxa [%o4] ASI_DMMU, %g2 | |
| 93 | stxa %o0, [%o4] ASI_DMMU | |
| 94 | 1: sub %o1, (1 << 3), %o1 | |
| 95 | ldx [%o2 + %o1], %o3 | |
| 96 | andcc %o3, 1, %g0 | |
| 97 | andn %o3, 1, %o3 | |
| 98 | be,pn %icc, 2f | |
| 99 | or %o3, 0x10, %o3 | |
| 100 | stxa %g0, [%o3] ASI_IMMU_DEMAP | |
| 101 | 2: stxa %g0, [%o3] ASI_DMMU_DEMAP | |
| 102 | membar #Sync | |
| 103 | brnz,pt %o1, 1b | |
| 104 | nop | |
| 105 | stxa %g2, [%o4] ASI_DMMU | |
| 4da808c3 DM |
106 | sethi %hi(KERNBASE), %o4 |
| 107 | flush %o4 | |
| 1da177e4 LT |
108 | retl |
| 109 | wrpr %g7, 0x0, %pstate | |
| fef43da4 | 110 | nop |
| 2ef27778 DM |
111 | nop |
| 112 | nop | |
| 113 | nop | |
| 1da177e4 LT |
114 | |
| 115 | .align 32 | |
| 116 | .globl __flush_tlb_kernel_range | |
| a74ad5e6 | 117 | __flush_tlb_kernel_range: /* 31 insns */ |
| 52bf082f | 118 | /* %o0=start, %o1=end */ |
| 1da177e4 LT |
119 | cmp %o0, %o1 |
| 120 | be,pn %xcc, 2f | |
| a74ad5e6 DM |
121 | sub %o1, %o0, %o3 |
| 122 | srlx %o3, 18, %o4 | |
| 123 | brnz,pn %o4, __spitfire_flush_tlb_kernel_range_slow | |
| 1da177e4 | 124 | sethi %hi(PAGE_SIZE), %o4 |
| 1da177e4 LT |
125 | sub %o3, %o4, %o3 |
| 126 | or %o0, 0x20, %o0 ! Nucleus | |
| 127 | 1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP | |
| 128 | stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP | |
| 129 | membar #Sync | |
| 130 | brnz,pt %o3, 1b | |
| 131 | sub %o3, %o4, %o3 | |
| 4da808c3 DM |
132 | 2: sethi %hi(KERNBASE), %o3 |
| 133 | flush %o3 | |
| 134 | retl | |
| 135 | nop | |
| 52bf082f | 136 | nop |
| b429ae4d DM |
137 | nop |
| 138 | nop | |
| 139 | nop | |
| a74ad5e6 DM |
140 | nop |
| 141 | nop | |
| 142 | nop | |
| 143 | nop | |
| 144 | nop | |
| 145 | nop | |
| 146 | nop | |
| 147 | nop | |
| 148 | nop | |
| 149 | nop | |
| 150 | ||
| 151 | __spitfire_flush_tlb_kernel_range_slow: | |
| 152 | mov 63 * 8, %o4 | |
| 153 | 1: ldxa [%o4] ASI_ITLB_DATA_ACCESS, %o3 | |
| 154 | andcc %o3, 0x40, %g0 /* _PAGE_L_4U */ | |
| 155 | bne,pn %xcc, 2f | |
| 156 | mov TLB_TAG_ACCESS, %o3 | |
| 157 | stxa %g0, [%o3] ASI_IMMU | |
| 158 | stxa %g0, [%o4] ASI_ITLB_DATA_ACCESS | |
| 159 | membar #Sync | |
| 160 | 2: ldxa [%o4] ASI_DTLB_DATA_ACCESS, %o3 | |
| 161 | andcc %o3, 0x40, %g0 | |
| 162 | bne,pn %xcc, 2f | |
| 163 | mov TLB_TAG_ACCESS, %o3 | |
| 164 | stxa %g0, [%o3] ASI_DMMU | |
| 165 | stxa %g0, [%o4] ASI_DTLB_DATA_ACCESS | |
| 166 | membar #Sync | |
| 167 | 2: sub %o4, 8, %o4 | |
| 168 | brgez,pt %o4, 1b | |
| 169 | nop | |
| 170 | retl | |
| 171 | nop | |
| 1da177e4 LT |
172 | |
| 173 | __spitfire_flush_tlb_mm_slow: | |
| 174 | rdpr %pstate, %g1 | |
| 175 | wrpr %g1, PSTATE_IE, %pstate | |
| 176 | stxa %o0, [%o1] ASI_DMMU | |
| 177 | stxa %g0, [%g3] ASI_DMMU_DEMAP | |
| 178 | stxa %g0, [%g3] ASI_IMMU_DEMAP | |
| 179 | flush %g6 | |
| 180 | stxa %g2, [%o1] ASI_DMMU | |
| 4da808c3 DM |
181 | sethi %hi(KERNBASE), %o1 |
| 182 | flush %o1 | |
| 1da177e4 LT |
183 | retl |
| 184 | wrpr %g1, 0, %pstate | |
| 185 | ||
| 186 | /* | |
| 187 | * The following code flushes one page_size worth. | |
| 188 | */ | |
| 83005161 | 189 | .section .kprobes.text, "ax" |
| 1da177e4 LT |
190 | .align 32 |
| 191 | .globl __flush_icache_page | |
| 192 | __flush_icache_page: /* %o0 = phys_page */ | |
| 1da177e4 | 193 | srlx %o0, PAGE_SHIFT, %o0 |
| b2d43834 | 194 | sethi %hi(PAGE_OFFSET), %g1 |
| 1da177e4 LT |
195 | sllx %o0, PAGE_SHIFT, %o0 |
| 196 | sethi %hi(PAGE_SIZE), %g2 | |
| b2d43834 | 197 | ldx [%g1 + %lo(PAGE_OFFSET)], %g1 |
| 1da177e4 LT |
198 | add %o0, %g1, %o0 |
| 199 | 1: subcc %g2, 32, %g2 | |
| 200 | bne,pt %icc, 1b | |
| 201 | flush %o0 + %g2 | |
| 202 | retl | |
| 203 | nop | |
| 204 | ||
| 205 | #ifdef DCACHE_ALIASING_POSSIBLE | |
| 206 | ||
| 207 | #if (PAGE_SHIFT != 13) | |
| 208 | #error only page shift of 13 is supported by dcache flush | |
| 209 | #endif | |
| 210 | ||
| 211 | #define DTAG_MASK 0x3 | |
| 212 | ||
| c5bd50a9 DM |
213 | /* This routine is Spitfire specific so the hardcoded |
| 214 | * D-cache size and line-size are OK. | |
| 215 | */ | |
| 1da177e4 LT |
216 | .align 64 |
| 217 | .globl __flush_dcache_page | |
| 218 | __flush_dcache_page: /* %o0=kaddr, %o1=flush_icache */ | |
| b2d43834 DM |
219 | sethi %hi(PAGE_OFFSET), %g1 |
| 220 | ldx [%g1 + %lo(PAGE_OFFSET)], %g1 | |
| c5bd50a9 DM |
221 | sub %o0, %g1, %o0 ! physical address |
| 222 | srlx %o0, 11, %o0 ! make D-cache TAG | |
| 223 | sethi %hi(1 << 14), %o2 ! D-cache size | |
| 224 | sub %o2, (1 << 5), %o2 ! D-cache line size | |
| 225 | 1: ldxa [%o2] ASI_DCACHE_TAG, %o3 ! load D-cache TAG | |
| 226 | andcc %o3, DTAG_MASK, %g0 ! Valid? | |
| 227 | be,pn %xcc, 2f ! Nope, branch | |
| 228 | andn %o3, DTAG_MASK, %o3 ! Clear valid bits | |
| 229 | cmp %o3, %o0 ! TAG match? | |
| 230 | bne,pt %xcc, 2f ! Nope, branch | |
| 231 | nop | |
| 232 | stxa %g0, [%o2] ASI_DCACHE_TAG ! Invalidate TAG | |
| 233 | membar #Sync | |
| 234 | 2: brnz,pt %o2, 1b | |
| 235 | sub %o2, (1 << 5), %o2 ! D-cache line size | |
| 1da177e4 LT |
236 | |
| 237 | /* The I-cache does not snoop local stores so we | |
| 238 | * better flush that too when necessary. | |
| 239 | */ | |
| 240 | brnz,pt %o1, __flush_icache_page | |
| 241 | sllx %o0, 11, %o0 | |
| 242 | retl | |
| 243 | nop | |
| 244 | ||
| 1da177e4 LT |
245 | #endif /* DCACHE_ALIASING_POSSIBLE */ |
| 246 | ||
| c5bd50a9 DM |
247 | .previous |
| 248 | ||
| 2ef27778 | 249 | /* Cheetah specific versions, patched at boot time. */ |
| 4da808c3 | 250 | __cheetah_flush_tlb_mm: /* 19 insns */ |
| 1da177e4 LT |
251 | rdpr %pstate, %g7 |
| 252 | andn %g7, PSTATE_IE, %g2 | |
| 253 | wrpr %g2, 0x0, %pstate | |
| 254 | wrpr %g0, 1, %tl | |
| 255 | mov PRIMARY_CONTEXT, %o2 | |
| 256 | mov 0x40, %g3 | |
| 257 | ldxa [%o2] ASI_DMMU, %g2 | |
| 2ef27778 DM |
258 | srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o1 |
| 259 | sllx %o1, CTX_PGSZ1_NUC_SHIFT, %o1 | |
| 260 | or %o0, %o1, %o0 /* Preserve nucleus page size fields */ | |
| 1da177e4 LT |
261 | stxa %o0, [%o2] ASI_DMMU |
| 262 | stxa %g0, [%g3] ASI_DMMU_DEMAP | |
| 263 | stxa %g0, [%g3] ASI_IMMU_DEMAP | |
| 264 | stxa %g2, [%o2] ASI_DMMU | |
| 4da808c3 DM |
265 | sethi %hi(KERNBASE), %o2 |
| 266 | flush %o2 | |
| 1da177e4 LT |
267 | wrpr %g0, 0, %tl |
| 268 | retl | |
| 269 | wrpr %g7, 0x0, %pstate | |
| 270 | ||
| f36391d2 DM |
271 | __cheetah_flush_tlb_page: /* 22 insns */ |
| 272 | /* %o0 = context, %o1 = vaddr */ | |
| 273 | rdpr %pstate, %g7 | |
| 274 | andn %g7, PSTATE_IE, %g2 | |
| 275 | wrpr %g2, 0x0, %pstate | |
| 276 | wrpr %g0, 1, %tl | |
| 277 | mov PRIMARY_CONTEXT, %o4 | |
| 278 | ldxa [%o4] ASI_DMMU, %g2 | |
| 279 | srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o3 | |
| 280 | sllx %o3, CTX_PGSZ1_NUC_SHIFT, %o3 | |
| 281 | or %o0, %o3, %o0 /* Preserve nucleus page size fields */ | |
| 282 | stxa %o0, [%o4] ASI_DMMU | |
| 283 | andcc %o1, 1, %g0 | |
| 284 | be,pn %icc, 1f | |
| 285 | andn %o1, 1, %o3 | |
| 286 | stxa %g0, [%o3] ASI_IMMU_DEMAP | |
| 287 | 1: stxa %g0, [%o3] ASI_DMMU_DEMAP | |
| 288 | membar #Sync | |
| 289 | stxa %g2, [%o4] ASI_DMMU | |
| 290 | sethi %hi(KERNBASE), %o4 | |
| 291 | flush %o4 | |
| 292 | wrpr %g0, 0, %tl | |
| 293 | retl | |
| 294 | wrpr %g7, 0x0, %pstate | |
| 295 | ||
| 4da808c3 | 296 | __cheetah_flush_tlb_pending: /* 27 insns */ |
| 1da177e4 LT |
297 | /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ |
| 298 | rdpr %pstate, %g7 | |
| 299 | sllx %o1, 3, %o1 | |
| 300 | andn %g7, PSTATE_IE, %g2 | |
| 301 | wrpr %g2, 0x0, %pstate | |
| 302 | wrpr %g0, 1, %tl | |
| 303 | mov PRIMARY_CONTEXT, %o4 | |
| 304 | ldxa [%o4] ASI_DMMU, %g2 | |
| 2ef27778 DM |
305 | srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o3 |
| 306 | sllx %o3, CTX_PGSZ1_NUC_SHIFT, %o3 | |
| 307 | or %o0, %o3, %o0 /* Preserve nucleus page size fields */ | |
| 1da177e4 LT |
308 | stxa %o0, [%o4] ASI_DMMU |
| 309 | 1: sub %o1, (1 << 3), %o1 | |
| 310 | ldx [%o2 + %o1], %o3 | |
| 311 | andcc %o3, 1, %g0 | |
| 312 | be,pn %icc, 2f | |
| 313 | andn %o3, 1, %o3 | |
| 314 | stxa %g0, [%o3] ASI_IMMU_DEMAP | |
| 315 | 2: stxa %g0, [%o3] ASI_DMMU_DEMAP | |
| b445e26c | 316 | membar #Sync |
| 1da177e4 | 317 | brnz,pt %o1, 1b |
| b445e26c | 318 | nop |
| 1da177e4 | 319 | stxa %g2, [%o4] ASI_DMMU |
| 4da808c3 DM |
320 | sethi %hi(KERNBASE), %o4 |
| 321 | flush %o4 | |
| 1da177e4 LT |
322 | wrpr %g0, 0, %tl |
| 323 | retl | |
| 324 | wrpr %g7, 0x0, %pstate | |
| 325 | ||
| a74ad5e6 DM |
326 | __cheetah_flush_tlb_kernel_range: /* 31 insns */ |
| 327 | /* %o0=start, %o1=end */ | |
| 328 | cmp %o0, %o1 | |
| 329 | be,pn %xcc, 2f | |
| 330 | sub %o1, %o0, %o3 | |
| 331 | srlx %o3, 18, %o4 | |
| 332 | brnz,pn %o4, 3f | |
| 333 | sethi %hi(PAGE_SIZE), %o4 | |
| 334 | sub %o3, %o4, %o3 | |
| 335 | or %o0, 0x20, %o0 ! Nucleus | |
| 336 | 1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP | |
| 337 | stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP | |
| 338 | membar #Sync | |
| 339 | brnz,pt %o3, 1b | |
| 340 | sub %o3, %o4, %o3 | |
| 341 | 2: sethi %hi(KERNBASE), %o3 | |
| 342 | flush %o3 | |
| 343 | retl | |
| 344 | nop | |
| 345 | 3: mov 0x80, %o4 | |
| 346 | stxa %g0, [%o4] ASI_DMMU_DEMAP | |
| 347 | membar #Sync | |
| 348 | stxa %g0, [%o4] ASI_IMMU_DEMAP | |
| 349 | membar #Sync | |
| 350 | retl | |
| 351 | nop | |
| 352 | nop | |
| 353 | nop | |
| 354 | nop | |
| 355 | nop | |
| 356 | nop | |
| 357 | nop | |
| 358 | nop | |
| 359 | ||
| 1da177e4 | 360 | #ifdef DCACHE_ALIASING_POSSIBLE |
| c5bd50a9 | 361 | __cheetah_flush_dcache_page: /* 11 insns */ |
| b2d43834 DM |
362 | sethi %hi(PAGE_OFFSET), %g1 |
| 363 | ldx [%g1 + %lo(PAGE_OFFSET)], %g1 | |
| 1da177e4 LT |
364 | sub %o0, %g1, %o0 |
| 365 | sethi %hi(PAGE_SIZE), %o4 | |
| 366 | 1: subcc %o4, (1 << 5), %o4 | |
| 367 | stxa %g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE | |
| 368 | membar #Sync | |
| 369 | bne,pt %icc, 1b | |
| 370 | nop | |
| 371 | retl /* I-cache flush never needed on Cheetah, see callers. */ | |
| 372 | nop | |
| 373 | #endif /* DCACHE_ALIASING_POSSIBLE */ | |
| 374 | ||
| 52bf082f | 375 | /* Hypervisor specific versions, patched at boot time. */ |
| 2a3a5f5d DM |
376 | __hypervisor_tlb_tl0_error: |
| 377 | save %sp, -192, %sp | |
| 378 | mov %i0, %o0 | |
| 379 | call hypervisor_tlbop_error | |
| 380 | mov %i1, %o1 | |
| 381 | ret | |
| 382 | restore | |
| 383 | ||
| b429ae4d | 384 | __hypervisor_flush_tlb_mm: /* 19 insns */ |
| 52bf082f DM |
385 | mov %o0, %o2 /* ARG2: mmu context */ |
| 386 | mov 0, %o0 /* ARG0: CPU lists unimplemented */ | |
| 387 | mov 0, %o1 /* ARG1: CPU lists unimplemented */ | |
| 388 | mov HV_MMU_ALL, %o3 /* ARG3: flags */ | |
| 389 | mov HV_FAST_MMU_DEMAP_CTX, %o5 | |
| 390 | ta HV_FAST_TRAP | |
| b429ae4d | 391 | brnz,pn %o0, 1f |
| 2a3a5f5d | 392 | mov HV_FAST_MMU_DEMAP_CTX, %o1 |
| 52bf082f DM |
393 | retl |
| 394 | nop | |
| b429ae4d DM |
395 | 1: sethi %hi(__hypervisor_tlb_tl0_error), %o5 |
| 396 | jmpl %o5 + %lo(__hypervisor_tlb_tl0_error), %g0 | |
| 397 | nop | |
| 398 | nop | |
| 399 | nop | |
| 400 | nop | |
| 401 | nop | |
| 402 | nop | |
| 403 | nop | |
| 52bf082f | 404 | |
| b429ae4d | 405 | __hypervisor_flush_tlb_page: /* 22 insns */ |
| f36391d2 DM |
406 | /* %o0 = context, %o1 = vaddr */ |
| 407 | mov %o0, %g2 | |
| 408 | mov %o1, %o0 /* ARG0: vaddr + IMMU-bit */ | |
| 409 | mov %g2, %o1 /* ARG1: mmu context */ | |
| 410 | mov HV_MMU_ALL, %o2 /* ARG2: flags */ | |
| 411 | srlx %o0, PAGE_SHIFT, %o0 | |
| 412 | sllx %o0, PAGE_SHIFT, %o0 | |
| 413 | ta HV_MMU_UNMAP_ADDR_TRAP | |
| b429ae4d | 414 | brnz,pn %o0, 1f |
| f36391d2 DM |
415 | mov HV_MMU_UNMAP_ADDR_TRAP, %o1 |
| 416 | retl | |
| 417 | nop | |
| b429ae4d DM |
418 | 1: sethi %hi(__hypervisor_tlb_tl0_error), %o2 |
| 419 | jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 | |
| 420 | nop | |
| 421 | nop | |
| 422 | nop | |
| 423 | nop | |
| 424 | nop | |
| 425 | nop | |
| 426 | nop | |
| 427 | nop | |
| 428 | nop | |
| f36391d2 | 429 | |
| 830cda3f | 430 | __hypervisor_flush_tlb_pending: /* 27 insns */ |
| 52bf082f DM |
431 | /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ |
| 432 | sllx %o1, 3, %g1 | |
| 433 | mov %o2, %g2 | |
| 434 | mov %o0, %g3 | |
| 435 | 1: sub %g1, (1 << 3), %g1 | |
| 436 | ldx [%g2 + %g1], %o0 /* ARG0: vaddr + IMMU-bit */ | |
| 437 | mov %g3, %o1 /* ARG1: mmu context */ | |
| 2a3a5f5d DM |
438 | mov HV_MMU_ALL, %o2 /* ARG2: flags */ |
| 439 | srlx %o0, PAGE_SHIFT, %o0 | |
| 440 | sllx %o0, PAGE_SHIFT, %o0 | |
| 52bf082f | 441 | ta HV_MMU_UNMAP_ADDR_TRAP |
| b429ae4d | 442 | brnz,pn %o0, 1f |
| 2a3a5f5d | 443 | mov HV_MMU_UNMAP_ADDR_TRAP, %o1 |
| 52bf082f DM |
444 | brnz,pt %g1, 1b |
| 445 | nop | |
| 446 | retl | |
| 447 | nop | |
| b429ae4d DM |
448 | 1: sethi %hi(__hypervisor_tlb_tl0_error), %o2 |
| 449 | jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 | |
| 450 | nop | |
| 451 | nop | |
| 452 | nop | |
| 453 | nop | |
| 454 | nop | |
| 455 | nop | |
| 456 | nop | |
| 457 | nop | |
| 458 | nop | |
| 52bf082f | 459 | |
| a74ad5e6 | 460 | __hypervisor_flush_tlb_kernel_range: /* 31 insns */ |
| 52bf082f DM |
461 | /* %o0=start, %o1=end */ |
| 462 | cmp %o0, %o1 | |
| 463 | be,pn %xcc, 2f | |
| a74ad5e6 DM |
464 | sub %o1, %o0, %g2 |
| 465 | srlx %g2, 18, %g3 | |
| 466 | brnz,pn %g3, 4f | |
| 467 | mov %o0, %g1 | |
| 468 | sethi %hi(PAGE_SIZE), %g3 | |
| 52bf082f DM |
469 | sub %g2, %g3, %g2 |
| 470 | 1: add %g1, %g2, %o0 /* ARG0: virtual address */ | |
| 471 | mov 0, %o1 /* ARG1: mmu context */ | |
| 472 | mov HV_MMU_ALL, %o2 /* ARG2: flags */ | |
| 473 | ta HV_MMU_UNMAP_ADDR_TRAP | |
| b429ae4d | 474 | brnz,pn %o0, 3f |
| 2a3a5f5d | 475 | mov HV_MMU_UNMAP_ADDR_TRAP, %o1 |
| 52bf082f DM |
476 | brnz,pt %g2, 1b |
| 477 | sub %g2, %g3, %g2 | |
| 478 | 2: retl | |
| 479 | nop | |
| b429ae4d DM |
480 | 3: sethi %hi(__hypervisor_tlb_tl0_error), %o2 |
| 481 | jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 | |
| 482 | nop | |
| a74ad5e6 DM |
483 | 4: mov 0, %o0 /* ARG0: CPU lists unimplemented */ |
| 484 | mov 0, %o1 /* ARG1: CPU lists unimplemented */ | |
| 485 | mov 0, %o2 /* ARG2: mmu context == nucleus */ | |
| 486 | mov HV_MMU_ALL, %o3 /* ARG3: flags */ | |
| 487 | mov HV_FAST_MMU_DEMAP_CTX, %o5 | |
| 488 | ta HV_FAST_TRAP | |
| 489 | brnz,pn %o0, 3b | |
| 490 | mov HV_FAST_MMU_DEMAP_CTX, %o1 | |
| 491 | retl | |
| 492 | nop | |
| 52bf082f DM |
493 | |
| 494 | #ifdef DCACHE_ALIASING_POSSIBLE | |
| 495 | /* XXX Niagara and friends have an 8K cache, so no aliasing is | |
| 496 | * XXX possible, but nothing explicit in the Hypervisor API | |
| 497 | * XXX guarantees this. | |
| 498 | */ | |
| 499 | __hypervisor_flush_dcache_page: /* 2 insns */ | |
| 500 | retl | |
| 501 | nop | |
| 502 | #endif | |
| 503 | ||
| 504 | tlb_patch_one: | |
| 1da177e4 LT |
505 | 1: lduw [%o1], %g1 |
| 506 | stw %g1, [%o0] | |
| 507 | flush %o0 | |
| 508 | subcc %o2, 1, %o2 | |
| 509 | add %o1, 4, %o1 | |
| 510 | bne,pt %icc, 1b | |
| 511 | add %o0, 4, %o0 | |
| 512 | retl | |
| 513 | nop | |
| 514 | ||
| 1da177e4 LT |
515 | #ifdef CONFIG_SMP |
| 516 | /* These are all called by the slaves of a cross call, at | |
| 517 | * trap level 1, with interrupts fully disabled. | |
| 518 | * | |
| 519 | * Register usage: | |
| 520 | * %g5 mm->context (all tlb flushes) | |
| 521 | * %g1 address arg 1 (tlb page and range flushes) | |
| 522 | * %g7 address arg 2 (tlb range flush only) | |
| 523 | * | |
| 56fb4df6 DM |
524 | * %g6 scratch 1 |
| 525 | * %g2 scratch 2 | |
| 526 | * %g3 scratch 3 | |
| 527 | * %g4 scratch 4 | |
| 1da177e4 LT |
528 | */ |
| 529 | .align 32 | |
| 530 | .globl xcall_flush_tlb_mm | |
| a236441b | 531 | xcall_flush_tlb_mm: /* 24 insns */ |
| 1da177e4 | 532 | mov PRIMARY_CONTEXT, %g2 |
| 1da177e4 | 533 | ldxa [%g2] ASI_DMMU, %g3 |
| 2ef27778 DM |
534 | srlx %g3, CTX_PGSZ1_NUC_SHIFT, %g4 |
| 535 | sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4 | |
| 536 | or %g5, %g4, %g5 /* Preserve nucleus page size fields */ | |
| 1da177e4 | 537 | stxa %g5, [%g2] ASI_DMMU |
| 2ef27778 | 538 | mov 0x40, %g4 |
| 1da177e4 LT |
539 | stxa %g0, [%g4] ASI_DMMU_DEMAP |
| 540 | stxa %g0, [%g4] ASI_IMMU_DEMAP | |
| 541 | stxa %g3, [%g2] ASI_DMMU | |
| 542 | retry | |
| 52bf082f DM |
543 | nop |
| 544 | nop | |
| 545 | nop | |
| 546 | nop | |
| 547 | nop | |
| 548 | nop | |
| 549 | nop | |
| 2a3a5f5d DM |
550 | nop |
| 551 | nop | |
| 552 | nop | |
| a236441b DM |
553 | nop |
| 554 | nop | |
| 555 | nop | |
| 1da177e4 | 556 | |
| f36391d2 | 557 | .globl xcall_flush_tlb_page |
| a236441b | 558 | xcall_flush_tlb_page: /* 20 insns */ |
| f36391d2 | 559 | /* %g5=context, %g1=vaddr */ |
| 1da177e4 LT |
560 | mov PRIMARY_CONTEXT, %g4 |
| 561 | ldxa [%g4] ASI_DMMU, %g2 | |
| 2ef27778 DM |
562 | srlx %g2, CTX_PGSZ1_NUC_SHIFT, %g4 |
| 563 | sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4 | |
| 564 | or %g5, %g4, %g5 | |
| 565 | mov PRIMARY_CONTEXT, %g4 | |
| 1da177e4 | 566 | stxa %g5, [%g4] ASI_DMMU |
| f36391d2 | 567 | andcc %g1, 0x1, %g0 |
| 1da177e4 | 568 | be,pn %icc, 2f |
| f36391d2 | 569 | andn %g1, 0x1, %g5 |
| 1da177e4 LT |
570 | stxa %g0, [%g5] ASI_IMMU_DEMAP |
| 571 | 2: stxa %g0, [%g5] ASI_DMMU_DEMAP | |
| 572 | membar #Sync | |
| 1da177e4 LT |
573 | stxa %g2, [%g4] ASI_DMMU |
| 574 | retry | |
| 2a3a5f5d | 575 | nop |
| f36391d2 | 576 | nop |
| a236441b DM |
577 | nop |
| 578 | nop | |
| 579 | nop | |
| 1da177e4 LT |
580 | |
| 581 | .globl xcall_flush_tlb_kernel_range | |
| a74ad5e6 | 582 | xcall_flush_tlb_kernel_range: /* 44 insns */ |
| 1da177e4 LT |
583 | sethi %hi(PAGE_SIZE - 1), %g2 |
| 584 | or %g2, %lo(PAGE_SIZE - 1), %g2 | |
| 585 | andn %g1, %g2, %g1 | |
| 586 | andn %g7, %g2, %g7 | |
| 587 | sub %g7, %g1, %g3 | |
| a74ad5e6 DM |
588 | srlx %g3, 18, %g2 |
| 589 | brnz,pn %g2, 2f | |
| 590 | add %g2, 1, %g2 | |
| 1da177e4 LT |
591 | sub %g3, %g2, %g3 |
| 592 | or %g1, 0x20, %g1 ! Nucleus | |
| 593 | 1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP | |
| 594 | stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP | |
| 595 | membar #Sync | |
| 596 | brnz,pt %g3, 1b | |
| 597 | sub %g3, %g2, %g3 | |
| 598 | retry | |
| a74ad5e6 DM |
599 | 2: mov 63 * 8, %g1 |
| 600 | 1: ldxa [%g1] ASI_ITLB_DATA_ACCESS, %g2 | |
| 601 | andcc %g2, 0x40, %g0 /* _PAGE_L_4U */ | |
| 602 | bne,pn %xcc, 2f | |
| 603 | mov TLB_TAG_ACCESS, %g2 | |
| 604 | stxa %g0, [%g2] ASI_IMMU | |
| 605 | stxa %g0, [%g1] ASI_ITLB_DATA_ACCESS | |
| 606 | membar #Sync | |
| 607 | 2: ldxa [%g1] ASI_DTLB_DATA_ACCESS, %g2 | |
| 608 | andcc %g2, 0x40, %g0 | |
| 609 | bne,pn %xcc, 2f | |
| 610 | mov TLB_TAG_ACCESS, %g2 | |
| 611 | stxa %g0, [%g2] ASI_DMMU | |
| 612 | stxa %g0, [%g1] ASI_DTLB_DATA_ACCESS | |
| 613 | membar #Sync | |
| 614 | 2: sub %g1, 8, %g1 | |
| 615 | brgez,pt %g1, 1b | |
| 616 | nop | |
| 617 | retry | |
| 52bf082f DM |
618 | nop |
| 619 | nop | |
| 620 | nop | |
| 2a3a5f5d DM |
621 | nop |
| 622 | nop | |
| 623 | nop | |
| a236441b DM |
624 | nop |
| 625 | nop | |
| 626 | nop | |
| 1da177e4 LT |
627 | |
| 628 | /* This runs in a very controlled environment, so we do | |
| 629 | * not need to worry about BH races etc. | |
| 630 | */ | |
| 631 | .globl xcall_sync_tick | |
| 632 | xcall_sync_tick: | |
| 45fec05f DM |
633 | |
| 634 | 661: rdpr %pstate, %g2 | |
| 1da177e4 | 635 | wrpr %g2, PSTATE_IG | PSTATE_AG, %pstate |
| df7d6aec | 636 | .section .sun4v_2insn_patch, "ax" |
| 45fec05f DM |
637 | .word 661b |
| 638 | nop | |
| 639 | nop | |
| 640 | .previous | |
| 641 | ||
| 1da177e4 | 642 | rdpr %pil, %g2 |
| b4f4372f | 643 | wrpr %g0, PIL_NORMAL_MAX, %pil |
| 1da177e4 LT |
644 | sethi %hi(109f), %g7 |
| 645 | b,pt %xcc, etrap_irq | |
| 646 | 109: or %g7, %lo(109b), %g7 | |
| 10e26723 DM |
647 | #ifdef CONFIG_TRACE_IRQFLAGS |
| 648 | call trace_hardirqs_off | |
| 649 | nop | |
| 650 | #endif | |
| 1da177e4 LT |
651 | call smp_synchronize_tick_client |
| 652 | nop | |
| 1da177e4 LT |
653 | b rtrap_xcall |
| 654 | ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1 | |
| 655 | ||
| 93dae5b7 DM |
656 | .globl xcall_fetch_glob_regs |
| 657 | xcall_fetch_glob_regs: | |
| 916ca14a DM |
658 | sethi %hi(global_cpu_snapshot), %g1 |
| 659 | or %g1, %lo(global_cpu_snapshot), %g1 | |
| 93dae5b7 DM |
660 | __GET_CPUID(%g2) |
| 661 | sllx %g2, 6, %g3 | |
| 662 | add %g1, %g3, %g1 | |
| 663 | rdpr %tstate, %g7 | |
| 664 | stx %g7, [%g1 + GR_SNAP_TSTATE] | |
| 665 | rdpr %tpc, %g7 | |
| 666 | stx %g7, [%g1 + GR_SNAP_TPC] | |
| 667 | rdpr %tnpc, %g7 | |
| 668 | stx %g7, [%g1 + GR_SNAP_TNPC] | |
| 669 | stx %o7, [%g1 + GR_SNAP_O7] | |
| 670 | stx %i7, [%g1 + GR_SNAP_I7] | |
| 5afe2738 | 671 | /* Don't try this at home kids... */ |
| a5a737e0 DM |
672 | rdpr %cwp, %g3 |
| 673 | sub %g3, 1, %g7 | |
| 5afe2738 DM |
674 | wrpr %g7, %cwp |
| 675 | mov %i7, %g7 | |
| a5a737e0 | 676 | wrpr %g3, %cwp |
| 5afe2738 | 677 | stx %g7, [%g1 + GR_SNAP_RPC] |
| 93dae5b7 DM |
678 | sethi %hi(trap_block), %g7 |
| 679 | or %g7, %lo(trap_block), %g7 | |
| 680 | sllx %g2, TRAP_BLOCK_SZ_SHIFT, %g2 | |
| 681 | add %g7, %g2, %g7 | |
| 682 | ldx [%g7 + TRAP_PER_CPU_THREAD], %g3 | |
| 93dae5b7 DM |
683 | stx %g3, [%g1 + GR_SNAP_THREAD] |
| 684 | retry | |
| 93dae5b7 | 685 | |
| 916ca14a DM |
686 | .globl xcall_fetch_glob_pmu |
| 687 | xcall_fetch_glob_pmu: | |
| 688 | sethi %hi(global_cpu_snapshot), %g1 | |
| 689 | or %g1, %lo(global_cpu_snapshot), %g1 | |
| 690 | __GET_CPUID(%g2) | |
| 691 | sllx %g2, 6, %g3 | |
| 692 | add %g1, %g3, %g1 | |
| 693 | rd %pic, %g7 | |
| 694 | stx %g7, [%g1 + (4 * 8)] | |
| 695 | rd %pcr, %g7 | |
| 696 | stx %g7, [%g1 + (0 * 8)] | |
| 697 | retry | |
| 698 | ||
| 699 | .globl xcall_fetch_glob_pmu_n4 | |
| 700 | xcall_fetch_glob_pmu_n4: | |
| 701 | sethi %hi(global_cpu_snapshot), %g1 | |
| 702 | or %g1, %lo(global_cpu_snapshot), %g1 | |
| 703 | __GET_CPUID(%g2) | |
| 704 | sllx %g2, 6, %g3 | |
| 705 | add %g1, %g3, %g1 | |
| 706 | ||
| 707 | ldxa [%g0] ASI_PIC, %g7 | |
| 708 | stx %g7, [%g1 + (4 * 8)] | |
| 709 | mov 0x08, %g3 | |
| 710 | ldxa [%g3] ASI_PIC, %g7 | |
| 711 | stx %g7, [%g1 + (5 * 8)] | |
| 712 | mov 0x10, %g3 | |
| 713 | ldxa [%g3] ASI_PIC, %g7 | |
| 714 | stx %g7, [%g1 + (6 * 8)] | |
| 715 | mov 0x18, %g3 | |
| 716 | ldxa [%g3] ASI_PIC, %g7 | |
| 717 | stx %g7, [%g1 + (7 * 8)] | |
| 718 | ||
| 719 | mov %o0, %g2 | |
| 720 | mov %o1, %g3 | |
| 721 | mov %o5, %g7 | |
| 722 | ||
| 723 | mov HV_FAST_VT_GET_PERFREG, %o5 | |
| 724 | mov 3, %o0 | |
| 725 | ta HV_FAST_TRAP | |
| 726 | stx %o1, [%g1 + (3 * 8)] | |
| 727 | mov HV_FAST_VT_GET_PERFREG, %o5 | |
| 728 | mov 2, %o0 | |
| 729 | ta HV_FAST_TRAP | |
| 730 | stx %o1, [%g1 + (2 * 8)] | |
| 731 | mov HV_FAST_VT_GET_PERFREG, %o5 | |
| 732 | mov 1, %o0 | |
| 733 | ta HV_FAST_TRAP | |
| 734 | stx %o1, [%g1 + (1 * 8)] | |
| 735 | mov HV_FAST_VT_GET_PERFREG, %o5 | |
| 736 | mov 0, %o0 | |
| 737 | ta HV_FAST_TRAP | |
| 738 | stx %o1, [%g1 + (0 * 8)] | |
| 739 | ||
| 740 | mov %g2, %o0 | |
| 741 | mov %g3, %o1 | |
| 742 | mov %g7, %o5 | |
| 743 | ||
| 744 | retry | |
| 745 | ||
| a74ad5e6 DM |
746 | __cheetah_xcall_flush_tlb_kernel_range: /* 44 insns */ |
| 747 | sethi %hi(PAGE_SIZE - 1), %g2 | |
| 748 | or %g2, %lo(PAGE_SIZE - 1), %g2 | |
| 749 | andn %g1, %g2, %g1 | |
| 750 | andn %g7, %g2, %g7 | |
| 751 | sub %g7, %g1, %g3 | |
| 752 | srlx %g3, 18, %g2 | |
| 753 | brnz,pn %g2, 2f | |
| 754 | add %g2, 1, %g2 | |
| 755 | sub %g3, %g2, %g3 | |
| 756 | or %g1, 0x20, %g1 ! Nucleus | |
| 757 | 1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP | |
| 758 | stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP | |
| 759 | membar #Sync | |
| 760 | brnz,pt %g3, 1b | |
| 761 | sub %g3, %g2, %g3 | |
| 762 | retry | |
| 763 | 2: mov 0x80, %g2 | |
| 764 | stxa %g0, [%g2] ASI_DMMU_DEMAP | |
| 765 | membar #Sync | |
| 766 | stxa %g0, [%g2] ASI_IMMU_DEMAP | |
| 767 | membar #Sync | |
| 768 | retry | |
| 769 | nop | |
| 770 | nop | |
| 771 | nop | |
| 772 | nop | |
| 773 | nop | |
| 774 | nop | |
| 775 | nop | |
| 776 | nop | |
| 777 | nop | |
| 778 | nop | |
| 779 | nop | |
| 780 | nop | |
| 781 | nop | |
| 782 | nop | |
| 783 | nop | |
| 784 | nop | |
| 785 | nop | |
| 786 | nop | |
| 787 | nop | |
| 788 | nop | |
| 789 | nop | |
| 790 | nop | |
| 791 | ||
| 1da177e4 LT |
792 | #ifdef DCACHE_ALIASING_POSSIBLE |
| 793 | .align 32 | |
| 794 | .globl xcall_flush_dcache_page_cheetah | |
| 795 | xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */ | |
| 796 | sethi %hi(PAGE_SIZE), %g3 | |
| 797 | 1: subcc %g3, (1 << 5), %g3 | |
| 798 | stxa %g0, [%g1 + %g3] ASI_DCACHE_INVALIDATE | |
| 799 | membar #Sync | |
| 800 | bne,pt %icc, 1b | |
| 801 | nop | |
| 802 | retry | |
| 803 | nop | |
| 804 | #endif /* DCACHE_ALIASING_POSSIBLE */ | |
| 805 | ||
| 806 | .globl xcall_flush_dcache_page_spitfire | |
| 807 | xcall_flush_dcache_page_spitfire: /* %g1 == physical page address | |
| 808 | %g7 == kernel page virtual address | |
| 809 | %g5 == (page->mapping != NULL) */ | |
| 810 | #ifdef DCACHE_ALIASING_POSSIBLE | |
| 811 | srlx %g1, (13 - 2), %g1 ! Form tag comparitor | |
| 812 | sethi %hi(L1DCACHE_SIZE), %g3 ! D$ size == 16K | |
| 813 | sub %g3, (1 << 5), %g3 ! D$ linesize == 32 | |
| 814 | 1: ldxa [%g3] ASI_DCACHE_TAG, %g2 | |
| 815 | andcc %g2, 0x3, %g0 | |
| 816 | be,pn %xcc, 2f | |
| 817 | andn %g2, 0x3, %g2 | |
| 818 | cmp %g2, %g1 | |
| 819 | ||
| 820 | bne,pt %xcc, 2f | |
| 821 | nop | |
| 822 | stxa %g0, [%g3] ASI_DCACHE_TAG | |
| 823 | membar #Sync | |
| 824 | 2: cmp %g3, 0 | |
| 825 | bne,pt %xcc, 1b | |
| 826 | sub %g3, (1 << 5), %g3 | |
| 827 | ||
| 828 | brz,pn %g5, 2f | |
| 829 | #endif /* DCACHE_ALIASING_POSSIBLE */ | |
| 830 | sethi %hi(PAGE_SIZE), %g3 | |
| 831 | ||
| 832 | 1: flush %g7 | |
| 833 | subcc %g3, (1 << 5), %g3 | |
| 834 | bne,pt %icc, 1b | |
| 835 | add %g7, (1 << 5), %g7 | |
| 836 | ||
| 837 | 2: retry | |
| 838 | nop | |
| 839 | nop | |
| 840 | ||
| 2a3a5f5d DM |
841 | /* %g5: error |
| 842 | * %g6: tlb op | |
| 843 | */ | |
| 844 | __hypervisor_tlb_xcall_error: | |
| 845 | mov %g5, %g4 | |
| 846 | mov %g6, %g5 | |
| 847 | ba,pt %xcc, etrap | |
| 848 | rd %pc, %g7 | |
| 849 | mov %l4, %o0 | |
| 850 | call hypervisor_tlbop_error_xcall | |
| 851 | mov %l5, %o1 | |
| 7697daaa | 852 | ba,a,pt %xcc, rtrap |
| 2a3a5f5d | 853 | |
| 52bf082f | 854 | .globl __hypervisor_xcall_flush_tlb_mm |
| a236441b | 855 | __hypervisor_xcall_flush_tlb_mm: /* 24 insns */ |
| 52bf082f DM |
856 | /* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */ |
| 857 | mov %o0, %g2 | |
| 858 | mov %o1, %g3 | |
| 859 | mov %o2, %g4 | |
| 860 | mov %o3, %g1 | |
| 861 | mov %o5, %g7 | |
| 862 | clr %o0 /* ARG0: CPU lists unimplemented */ | |
| 863 | clr %o1 /* ARG1: CPU lists unimplemented */ | |
| 864 | mov %g5, %o2 /* ARG2: mmu context */ | |
| 865 | mov HV_MMU_ALL, %o3 /* ARG3: flags */ | |
| 866 | mov HV_FAST_MMU_DEMAP_CTX, %o5 | |
| 867 | ta HV_FAST_TRAP | |
| 2a3a5f5d | 868 | mov HV_FAST_MMU_DEMAP_CTX, %g6 |
| a236441b | 869 | brnz,pn %o0, 1f |
| 2a3a5f5d | 870 | mov %o0, %g5 |
| 52bf082f DM |
871 | mov %g2, %o0 |
| 872 | mov %g3, %o1 | |
| 873 | mov %g4, %o2 | |
| 874 | mov %g1, %o3 | |
| 875 | mov %g7, %o5 | |
| 876 | membar #Sync | |
| 877 | retry | |
| a236441b DM |
878 | 1: sethi %hi(__hypervisor_tlb_xcall_error), %g4 |
| 879 | jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 | |
| 880 | nop | |
| 52bf082f | 881 | |
| f36391d2 | 882 | .globl __hypervisor_xcall_flush_tlb_page |
| a236441b | 883 | __hypervisor_xcall_flush_tlb_page: /* 20 insns */ |
| f36391d2 | 884 | /* %g5=ctx, %g1=vaddr */ |
| 52bf082f DM |
885 | mov %o0, %g2 |
| 886 | mov %o1, %g3 | |
| 887 | mov %o2, %g4 | |
| f36391d2 | 888 | mov %g1, %o0 /* ARG0: virtual address */ |
| 52bf082f | 889 | mov %g5, %o1 /* ARG1: mmu context */ |
| 2a3a5f5d DM |
890 | mov HV_MMU_ALL, %o2 /* ARG2: flags */ |
| 891 | srlx %o0, PAGE_SHIFT, %o0 | |
| 892 | sllx %o0, PAGE_SHIFT, %o0 | |
| 52bf082f | 893 | ta HV_MMU_UNMAP_ADDR_TRAP |
| 2a3a5f5d | 894 | mov HV_MMU_UNMAP_ADDR_TRAP, %g6 |
| a236441b | 895 | brnz,a,pn %o0, 1f |
| 2a3a5f5d | 896 | mov %o0, %g5 |
| 52bf082f DM |
897 | mov %g2, %o0 |
| 898 | mov %g3, %o1 | |
| 899 | mov %g4, %o2 | |
| 900 | membar #Sync | |
| 901 | retry | |
| a236441b DM |
902 | 1: sethi %hi(__hypervisor_tlb_xcall_error), %g4 |
| 903 | jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 | |
| 904 | nop | |
| 52bf082f DM |
905 | |
| 906 | .globl __hypervisor_xcall_flush_tlb_kernel_range | |
| a74ad5e6 | 907 | __hypervisor_xcall_flush_tlb_kernel_range: /* 44 insns */ |
| 2a3a5f5d | 908 | /* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */ |
| 52bf082f DM |
909 | sethi %hi(PAGE_SIZE - 1), %g2 |
| 910 | or %g2, %lo(PAGE_SIZE - 1), %g2 | |
| 911 | andn %g1, %g2, %g1 | |
| 912 | andn %g7, %g2, %g7 | |
| 913 | sub %g7, %g1, %g3 | |
| a74ad5e6 | 914 | srlx %g3, 18, %g7 |
| 52bf082f DM |
915 | add %g2, 1, %g2 |
| 916 | sub %g3, %g2, %g3 | |
| 917 | mov %o0, %g2 | |
| 918 | mov %o1, %g4 | |
| a74ad5e6 DM |
919 | brnz,pn %g7, 2f |
| 920 | mov %o2, %g7 | |
| 52bf082f DM |
921 | 1: add %g1, %g3, %o0 /* ARG0: virtual address */ |
| 922 | mov 0, %o1 /* ARG1: mmu context */ | |
| 923 | mov HV_MMU_ALL, %o2 /* ARG2: flags */ | |
| 924 | ta HV_MMU_UNMAP_ADDR_TRAP | |
| 2a3a5f5d | 925 | mov HV_MMU_UNMAP_ADDR_TRAP, %g6 |
| a236441b | 926 | brnz,pn %o0, 1f |
| 2a3a5f5d | 927 | mov %o0, %g5 |
| 52bf082f DM |
928 | sethi %hi(PAGE_SIZE), %o2 |
| 929 | brnz,pt %g3, 1b | |
| 930 | sub %g3, %o2, %g3 | |
| a74ad5e6 | 931 | 5: mov %g2, %o0 |
| 52bf082f | 932 | mov %g4, %o1 |
| 2a3a5f5d | 933 | mov %g7, %o2 |
| 52bf082f DM |
934 | membar #Sync |
| 935 | retry | |
| a236441b DM |
936 | 1: sethi %hi(__hypervisor_tlb_xcall_error), %g4 |
| 937 | jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 | |
| 938 | nop | |
| a74ad5e6 DM |
939 | 2: mov %o3, %g1 |
| 940 | mov %o5, %g3 | |
| 941 | mov 0, %o0 /* ARG0: CPU lists unimplemented */ | |
| 942 | mov 0, %o1 /* ARG1: CPU lists unimplemented */ | |
| 943 | mov 0, %o2 /* ARG2: mmu context == nucleus */ | |
| 944 | mov HV_MMU_ALL, %o3 /* ARG3: flags */ | |
| 945 | mov HV_FAST_MMU_DEMAP_CTX, %o5 | |
| 946 | ta HV_FAST_TRAP | |
| 947 | mov %g1, %o3 | |
| 948 | brz,pt %o0, 5b | |
| 949 | mov %g3, %o5 | |
| 950 | mov HV_FAST_MMU_DEMAP_CTX, %g6 | |
| 951 | ba,pt %xcc, 1b | |
| 952 | clr %g5 | |
| 52bf082f | 953 | |
| 1da177e4 LT |
954 | /* These just get rescheduled to PIL vectors. */ |
| 955 | .globl xcall_call_function | |
| 956 | xcall_call_function: | |
| 957 | wr %g0, (1 << PIL_SMP_CALL_FUNC), %set_softint | |
| 958 | retry | |
| 959 | ||
| d172ad18 DM |
960 | .globl xcall_call_function_single |
| 961 | xcall_call_function_single: | |
| 962 | wr %g0, (1 << PIL_SMP_CALL_FUNC_SNGL), %set_softint | |
| 963 | retry | |
| 964 | ||
| 1da177e4 LT |
965 | .globl xcall_receive_signal |
| 966 | xcall_receive_signal: | |
| 967 | wr %g0, (1 << PIL_SMP_RECEIVE_SIGNAL), %set_softint | |
| 968 | retry | |
| 969 | ||
| 970 | .globl xcall_capture | |
| 971 | xcall_capture: | |
| 972 | wr %g0, (1 << PIL_SMP_CAPTURE), %set_softint | |
| 973 | retry | |
| 974 | ||
| e2fdd7fd DM |
975 | #ifdef CONFIG_KGDB |
| 976 | .globl xcall_kgdb_capture | |
| 977 | xcall_kgdb_capture: | |
| 42cc77c8 DM |
978 | wr %g0, (1 << PIL_KGDB_CAPTURE), %set_softint |
| 979 | retry | |
| e2fdd7fd DM |
980 | #endif |
| 981 | ||
| 1da177e4 | 982 | #endif /* CONFIG_SMP */ |
| 52bf082f | 983 | |
| a74ad5e6 DM |
984 | .globl cheetah_patch_cachetlbops |
| 985 | cheetah_patch_cachetlbops: | |
| 986 | save %sp, -128, %sp | |
| 987 | ||
| 988 | sethi %hi(__flush_tlb_mm), %o0 | |
| 989 | or %o0, %lo(__flush_tlb_mm), %o0 | |
| 990 | sethi %hi(__cheetah_flush_tlb_mm), %o1 | |
| 991 | or %o1, %lo(__cheetah_flush_tlb_mm), %o1 | |
| 992 | call tlb_patch_one | |
| 993 | mov 19, %o2 | |
| 994 | ||
| 995 | sethi %hi(__flush_tlb_page), %o0 | |
| 996 | or %o0, %lo(__flush_tlb_page), %o0 | |
| 997 | sethi %hi(__cheetah_flush_tlb_page), %o1 | |
| 998 | or %o1, %lo(__cheetah_flush_tlb_page), %o1 | |
| 999 | call tlb_patch_one | |
| 1000 | mov 22, %o2 | |
| 1001 | ||
| 1002 | sethi %hi(__flush_tlb_pending), %o0 | |
| 1003 | or %o0, %lo(__flush_tlb_pending), %o0 | |
| 1004 | sethi %hi(__cheetah_flush_tlb_pending), %o1 | |
| 1005 | or %o1, %lo(__cheetah_flush_tlb_pending), %o1 | |
| 1006 | call tlb_patch_one | |
| 1007 | mov 27, %o2 | |
| 1008 | ||
| 1009 | sethi %hi(__flush_tlb_kernel_range), %o0 | |
| 1010 | or %o0, %lo(__flush_tlb_kernel_range), %o0 | |
| 1011 | sethi %hi(__cheetah_flush_tlb_kernel_range), %o1 | |
| 1012 | or %o1, %lo(__cheetah_flush_tlb_kernel_range), %o1 | |
| 1013 | call tlb_patch_one | |
| 1014 | mov 31, %o2 | |
| 1015 | ||
| 1016 | #ifdef DCACHE_ALIASING_POSSIBLE | |
| 1017 | sethi %hi(__flush_dcache_page), %o0 | |
| 1018 | or %o0, %lo(__flush_dcache_page), %o0 | |
| 1019 | sethi %hi(__cheetah_flush_dcache_page), %o1 | |
| 1020 | or %o1, %lo(__cheetah_flush_dcache_page), %o1 | |
| 1021 | call tlb_patch_one | |
| 1022 | mov 11, %o2 | |
| 1023 | #endif /* DCACHE_ALIASING_POSSIBLE */ | |
| 1024 | ||
| 1025 | #ifdef CONFIG_SMP | |
| 1026 | sethi %hi(xcall_flush_tlb_kernel_range), %o0 | |
| 1027 | or %o0, %lo(xcall_flush_tlb_kernel_range), %o0 | |
| 1028 | sethi %hi(__cheetah_xcall_flush_tlb_kernel_range), %o1 | |
| 1029 | or %o1, %lo(__cheetah_xcall_flush_tlb_kernel_range), %o1 | |
| 1030 | call tlb_patch_one | |
| 1031 | mov 44, %o2 | |
| 1032 | #endif /* CONFIG_SMP */ | |
| 1033 | ||
| 1034 | ret | |
| 1035 | restore | |
| 52bf082f DM |
1036 | |
| 1037 | .globl hypervisor_patch_cachetlbops | |
| 1038 | hypervisor_patch_cachetlbops: | |
| 1039 | save %sp, -128, %sp | |
| 1040 | ||
| 1041 | sethi %hi(__flush_tlb_mm), %o0 | |
| 1042 | or %o0, %lo(__flush_tlb_mm), %o0 | |
| 1043 | sethi %hi(__hypervisor_flush_tlb_mm), %o1 | |
| 1044 | or %o1, %lo(__hypervisor_flush_tlb_mm), %o1 | |
| 1045 | call tlb_patch_one | |
| b429ae4d | 1046 | mov 19, %o2 |
| 52bf082f | 1047 | |
| f36391d2 DM |
1048 | sethi %hi(__flush_tlb_page), %o0 |
| 1049 | or %o0, %lo(__flush_tlb_page), %o0 | |
| 1050 | sethi %hi(__hypervisor_flush_tlb_page), %o1 | |
| 1051 | or %o1, %lo(__hypervisor_flush_tlb_page), %o1 | |
| 1052 | call tlb_patch_one | |
| b429ae4d | 1053 | mov 22, %o2 |
| f36391d2 | 1054 | |
| 52bf082f DM |
1055 | sethi %hi(__flush_tlb_pending), %o0 |
| 1056 | or %o0, %lo(__flush_tlb_pending), %o0 | |
| 1057 | sethi %hi(__hypervisor_flush_tlb_pending), %o1 | |
| 1058 | or %o1, %lo(__hypervisor_flush_tlb_pending), %o1 | |
| 1059 | call tlb_patch_one | |
| b429ae4d | 1060 | mov 27, %o2 |
| 52bf082f DM |
1061 | |
| 1062 | sethi %hi(__flush_tlb_kernel_range), %o0 | |
| 1063 | or %o0, %lo(__flush_tlb_kernel_range), %o0 | |
| 1064 | sethi %hi(__hypervisor_flush_tlb_kernel_range), %o1 | |
| 1065 | or %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1 | |
| 1066 | call tlb_patch_one | |
| a74ad5e6 | 1067 | mov 31, %o2 |
| 52bf082f DM |
1068 | |
| 1069 | #ifdef DCACHE_ALIASING_POSSIBLE | |
| 1070 | sethi %hi(__flush_dcache_page), %o0 | |
| 1071 | or %o0, %lo(__flush_dcache_page), %o0 | |
| 1072 | sethi %hi(__hypervisor_flush_dcache_page), %o1 | |
| 1073 | or %o1, %lo(__hypervisor_flush_dcache_page), %o1 | |
| 1074 | call tlb_patch_one | |
| 1075 | mov 2, %o2 | |
| 1076 | #endif /* DCACHE_ALIASING_POSSIBLE */ | |
| 1077 | ||
| 1078 | #ifdef CONFIG_SMP | |
| 1079 | sethi %hi(xcall_flush_tlb_mm), %o0 | |
| 1080 | or %o0, %lo(xcall_flush_tlb_mm), %o0 | |
| 1081 | sethi %hi(__hypervisor_xcall_flush_tlb_mm), %o1 | |
| 1082 | or %o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1 | |
| 1083 | call tlb_patch_one | |
| a236441b | 1084 | mov 24, %o2 |
| 52bf082f | 1085 | |
| f36391d2 DM |
1086 | sethi %hi(xcall_flush_tlb_page), %o0 |
| 1087 | or %o0, %lo(xcall_flush_tlb_page), %o0 | |
| 1088 | sethi %hi(__hypervisor_xcall_flush_tlb_page), %o1 | |
| 1089 | or %o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1 | |
| 52bf082f | 1090 | call tlb_patch_one |
| a236441b | 1091 | mov 20, %o2 |
| 52bf082f DM |
1092 | |
| 1093 | sethi %hi(xcall_flush_tlb_kernel_range), %o0 | |
| 1094 | or %o0, %lo(xcall_flush_tlb_kernel_range), %o0 | |
| 1095 | sethi %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1 | |
| 1096 | or %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1 | |
| 1097 | call tlb_patch_one | |
| a74ad5e6 | 1098 | mov 44, %o2 |
| 52bf082f DM |
1099 | #endif /* CONFIG_SMP */ |
| 1100 | ||
| 1101 | ret | |
| 1102 | restore |