arch/powerpc/kernel/vector.S

   1 #include <asm/processor.h>
   2 #include <asm/ppc_asm.h>
   3 #include <asm/reg.h>
   4 #include <asm/asm-offsets.h>
   5 #include <asm/cputable.h>
   6 #include <asm/thread_info.h>
   7 #include <asm/page.h>
   8 #include <asm/ptrace.h>
   9 #include <asm/export.h>
  10
  11 /*
  12  * Load state from memory into VMX registers including VSCR.
  13  * Assumes the caller has enabled VMX in the MSR.
  14  */
  15 _GLOBAL(load_vr_state)
  16         li      r4,VRSTATE_VSCR
  17         lvx     v0,r4,r3
  18         mtvscr  v0
  19         REST_32VRS(0,r4,r3)
  20         blr
  21 EXPORT_SYMBOL(load_vr_state)
  22
  23 /*
  24  * Store VMX state into memory, including VSCR.
  25  * Assumes the caller has enabled VMX in the MSR.
  26  */
  27 _GLOBAL(store_vr_state)
  28         SAVE_32VRS(0, r4, r3)
  29         mfvscr  v0
  30         li      r4, VRSTATE_VSCR
  31         stvx    v0, r4, r3
  32         blr
  33 EXPORT_SYMBOL(store_vr_state)
  34
  35 /*
  36  * Disable VMX for the task which had it previously,
  37  * and save its vector registers in its thread_struct.
  38  * Enables the VMX for use in the kernel on return.
  39  * On SMP we know the VMX is free, since we give it up every
  40  * switch (ie, no lazy save of the vector registers).
  41  *
  42  * Note that on 32-bit this can only use registers that will be
  43  * restored by fast_exception_return, i.e. r3 - r6, r10 and r11.
  44  */
  45 _GLOBAL(load_up_altivec)
  46         mfmsr   r5                      /* grab the current MSR */
  47         oris    r5,r5,MSR_VEC@h
  48         MTMSRD(r5)                      /* enable use of AltiVec now */
  49         isync
  50
  51         /*
  52          * While userspace in general ignores VRSAVE, glibc uses it as a boolean
  53          * to optimise userspace context save/restore. Whenever we take an
  54          * altivec unavailable exception we must set VRSAVE to something non
  55          * zero. Set it to all 1s. See also the programming note in the ISA.
  56          */
  57         mfspr   r4,SPRN_VRSAVE
  58         cmpwi   0,r4,0
  59         bne+    1f
  60         li      r4,-1
  61         mtspr   SPRN_VRSAVE,r4
  62 1:
  63         /* enable use of VMX after return */
  64 #ifdef CONFIG_PPC32
  65         mfspr   r5,SPRN_SPRG_THREAD             /* current task's THREAD (phys) */
  66         oris    r9,r9,MSR_VEC@h
  67 #else
  68         ld      r4,PACACURRENT(r13)
  69         addi    r5,r4,THREAD            /* Get THREAD */
  70         oris    r12,r12,MSR_VEC@h
  71         std     r12,_MSR(r1)
  72 #endif
  73         /* Don't care if r4 overflows, this is desired behaviour */
  74         lbz     r4,THREAD_LOAD_VEC(r5)
  75         addi    r4,r4,1
  76         stb     r4,THREAD_LOAD_VEC(r5)
  77         addi    r6,r5,THREAD_VRSTATE
  78         li      r4,1
  79         li      r10,VRSTATE_VSCR
  80         stw     r4,THREAD_USED_VR(r5)
  81         lvx     v0,r10,r6
  82         mtvscr  v0
  83         REST_32VRS(0,r4,r6)
  84         /* restore registers and return */
  85         blr
  86
  87 /*
  88  * save_altivec(tsk)
  89  * Save the vector registers to its thread_struct
  90  */
  91 _GLOBAL(save_altivec)
  92         addi    r3,r3,THREAD            /* want THREAD of task */
  93         PPC_LL  r7,THREAD_VRSAVEAREA(r3)
  94         PPC_LL  r5,PT_REGS(r3)
  95         PPC_LCMPI       0,r7,0
  96         bne     2f
  97         addi    r7,r3,THREAD_VRSTATE
  98 2:      SAVE_32VRS(0,r4,r7)
  99         mfvscr  v0
 100         li      r4,VRSTATE_VSCR
 101         stvx    v0,r4,r7
 102         blr
 103
 104 #ifdef CONFIG_VSX
 105
 106 #ifdef CONFIG_PPC32
 107 #error This asm code isn't ready for 32-bit kernels
 108 #endif
 109
 110 /*
 111  * load_up_vsx(unused, unused, tsk)
 112  * Disable VSX for the task which had it previously,
 113  * and save its vector registers in its thread_struct.
 114  * Reuse the fp and vsx saves, but first check to see if they have
 115  * been saved already.
 116  */
 117 _GLOBAL(load_up_vsx)
 118 /* Load FP and VSX registers if they haven't been done yet */
 119         andi.   r5,r12,MSR_FP
 120         beql+   load_up_fpu             /* skip if already loaded */
 121         andis.  r5,r12,MSR_VEC@h
 122         beql+   load_up_altivec         /* skip if already loaded */
 123
 124         ld      r4,PACACURRENT(r13)
 125         addi    r4,r4,THREAD            /* Get THREAD */
 126         li      r6,1
 127         stw     r6,THREAD_USED_VSR(r4) /* ... also set thread used vsr */
 128         /* enable use of VSX after return */
 129         oris    r12,r12,MSR_VSX@h
 130         std     r12,_MSR(r1)
 131         b       fast_exception_return
 132
 133 #endif /* CONFIG_VSX */
 134
 135
 136 /*
 137  * The routines below are in assembler so we can closely control the
 138  * usage of floating-point registers.  These routines must be called
 139  * with preempt disabled.
 140  */
 141 #ifdef CONFIG_PPC32
 142         .data
 143 fpzero:
 144         .long   0
 145 fpone:
 146         .long   0x3f800000      /* 1.0 in single-precision FP */
 147 fphalf:
 148         .long   0x3f000000      /* 0.5 in single-precision FP */
 149
 150 #define LDCONST(fr, name)       \
 151         lis     r11,name@ha;    \
 152         lfs     fr,name@l(r11)
 153 #else
 154
 155         .section ".toc","aw"
 156 fpzero:
 157         .tc     FD_0_0[TC],0
 158 fpone:
 159         .tc     FD_3ff00000_0[TC],0x3ff0000000000000    /* 1.0 */
 160 fphalf:
 161         .tc     FD_3fe00000_0[TC],0x3fe0000000000000    /* 0.5 */
 162
 163 #define LDCONST(fr, name)       \
 164         lfd     fr,name@toc(r2)
 165 #endif
 166
 167         .text
 168 /*
 169  * Internal routine to enable floating point and set FPSCR to 0.
 170  * Don't call it from C; it doesn't use the normal calling convention.
 171  */
 172 fpenable:
 173 #ifdef CONFIG_PPC32
 174         stwu    r1,-64(r1)
 175 #else
 176         stdu    r1,-64(r1)
 177 #endif
 178         mfmsr   r10
 179         ori     r11,r10,MSR_FP
 180         mtmsr   r11
 181         isync
 182         stfd    fr0,24(r1)
 183         stfd    fr1,16(r1)
 184         stfd    fr31,8(r1)
 185         LDCONST(fr1, fpzero)
 186         mffs    fr31
 187         MTFSF_L(fr1)
 188         blr
 189
 190 fpdisable:
 191         mtlr    r12
 192         MTFSF_L(fr31)
 193         lfd     fr31,8(r1)
 194         lfd     fr1,16(r1)
 195         lfd     fr0,24(r1)
 196         mtmsr   r10
 197         isync
 198         addi    r1,r1,64
 199         blr
 200
 201 /*
 202  * Vector add, floating point.
 203  */
 204 _GLOBAL(vaddfp)
 205         mflr    r12
 206         bl      fpenable
 207         li      r0,4
 208         mtctr   r0
 209         li      r6,0
 210 1:      lfsx    fr0,r4,r6
 211         lfsx    fr1,r5,r6
 212         fadds   fr0,fr0,fr1
 213         stfsx   fr0,r3,r6
 214         addi    r6,r6,4
 215         bdnz    1b
 216         b       fpdisable
 217
 218 /*
 219  * Vector subtract, floating point.
 220  */
 221 _GLOBAL(vsubfp)
 222         mflr    r12
 223         bl      fpenable
 224         li      r0,4
 225         mtctr   r0
 226         li      r6,0
 227 1:      lfsx    fr0,r4,r6
 228         lfsx    fr1,r5,r6
 229         fsubs   fr0,fr0,fr1
 230         stfsx   fr0,r3,r6
 231         addi    r6,r6,4
 232         bdnz    1b
 233         b       fpdisable
 234
 235 /*
 236  * Vector multiply and add, floating point.
 237  */
 238 _GLOBAL(vmaddfp)
 239         mflr    r12
 240         bl      fpenable
 241         stfd    fr2,32(r1)
 242         li      r0,4
 243         mtctr   r0
 244         li      r7,0
 245 1:      lfsx    fr0,r4,r7
 246         lfsx    fr1,r5,r7
 247         lfsx    fr2,r6,r7
 248         fmadds  fr0,fr0,fr2,fr1
 249         stfsx   fr0,r3,r7
 250         addi    r7,r7,4
 251         bdnz    1b
 252         lfd     fr2,32(r1)
 253         b       fpdisable
 254
 255 /*
 256  * Vector negative multiply and subtract, floating point.
 257  */
 258 _GLOBAL(vnmsubfp)
 259         mflr    r12
 260         bl      fpenable
 261         stfd    fr2,32(r1)
 262         li      r0,4
 263         mtctr   r0
 264         li      r7,0
 265 1:      lfsx    fr0,r4,r7
 266         lfsx    fr1,r5,r7
 267         lfsx    fr2,r6,r7
 268         fnmsubs fr0,fr0,fr2,fr1
 269         stfsx   fr0,r3,r7
 270         addi    r7,r7,4
 271         bdnz    1b
 272         lfd     fr2,32(r1)
 273         b       fpdisable
 274
 275 /*
 276  * Vector reciprocal estimate.  We just compute 1.0/x.
 277  * r3 -> destination, r4 -> source.
 278  */
 279 _GLOBAL(vrefp)
 280         mflr    r12
 281         bl      fpenable
 282         li      r0,4
 283         LDCONST(fr1, fpone)
 284         mtctr   r0
 285         li      r6,0
 286 1:      lfsx    fr0,r4,r6
 287         fdivs   fr0,fr1,fr0
 288         stfsx   fr0,r3,r6
 289         addi    r6,r6,4
 290         bdnz    1b
 291         b       fpdisable
 292
 293 /*
 294  * Vector reciprocal square-root estimate, floating point.
 295  * We use the frsqrte instruction for the initial estimate followed
 296  * by 2 iterations of Newton-Raphson to get sufficient accuracy.
 297  * r3 -> destination, r4 -> source.
 298  */
 299 _GLOBAL(vrsqrtefp)
 300         mflr    r12
 301         bl      fpenable
 302         stfd    fr2,32(r1)
 303         stfd    fr3,40(r1)
 304         stfd    fr4,48(r1)
 305         stfd    fr5,56(r1)
 306         li      r0,4
 307         LDCONST(fr4, fpone)
 308         LDCONST(fr5, fphalf)
 309         mtctr   r0
 310         li      r6,0
 311 1:      lfsx    fr0,r4,r6
 312         frsqrte fr1,fr0         /* r = frsqrte(s) */
 313         fmuls   fr3,fr1,fr0     /* r * s */
 314         fmuls   fr2,fr1,fr5     /* r * 0.5 */
 315         fnmsubs fr3,fr1,fr3,fr4 /* 1 - s * r * r */
 316         fmadds  fr1,fr2,fr3,fr1 /* r = r + 0.5 * r * (1 - s * r * r) */
 317         fmuls   fr3,fr1,fr0     /* r * s */
 318         fmuls   fr2,fr1,fr5     /* r * 0.5 */
 319         fnmsubs fr3,fr1,fr3,fr4 /* 1 - s * r * r */
 320         fmadds  fr1,fr2,fr3,fr1 /* r = r + 0.5 * r * (1 - s * r * r) */
 321         stfsx   fr1,r3,r6
 322         addi    r6,r6,4
 323         bdnz    1b
 324         lfd     fr5,56(r1)
 325         lfd     fr4,48(r1)
 326         lfd     fr3,40(r1)
 327         lfd     fr2,32(r1)
 328         b       fpdisable