arch/arm/lib/copy_template.S

   1 /*
   2  *  linux/arch/arm/lib/copy_template.s
   3  *
   4  *  Code template for optimized memory copy functions
   5  *
   6  *  Author:     Nicolas Pitre
   7  *  Created:    Sep 28, 2005
   8  *  Copyright:  MontaVista Software, Inc.
   9  *
  10  *  This program is free software; you can redistribute it and/or modify
  11  *  it under the terms of the GNU General Public License version 2 as
  12  *  published by the Free Software Foundation.
  13  */
  14
  15 /*
  16  * Theory of operation
  17  * -------------------
  18  *
  19  * This file provides the core code for a forward memory copy used in
  20  * the implementation of memcopy(), copy_to_user() and copy_from_user().
  21  *
  22  * The including file must define the following accessor macros
  23  * according to the need of the given function:
  24  *
  25  * ldr1w ptr reg abort
  26  *
  27  *      This loads one word from 'ptr', stores it in 'reg' and increments
  28  *      'ptr' to the next word. The 'abort' argument is used for fixup tables.
  29  *
  30  * ldr4w ptr reg1 reg2 reg3 reg4 abort
  31  * ldr8w ptr, reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
  32  *
  33  *      This loads four or eight words starting from 'ptr', stores them
  34  *      in provided registers and increments 'ptr' past those words.
  35  *      The'abort' argument is used for fixup tables.
  36  *
  37  * ldr1b ptr reg cond abort
  38  *
  39  *      Similar to ldr1w, but it loads a byte and increments 'ptr' one byte.
  40  *      It also must apply the condition code if provided, otherwise the
  41  *      "al" condition is assumed by default.
  42  *
  43  * str1w ptr reg abort
  44  * str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
  45  * str1b ptr reg cond abort
  46  *
  47  *      Same as their ldr* counterparts, but data is stored to 'ptr' location
  48  *      rather than being loaded.
  49  *
  50  * enter reg1 reg2
  51  *
  52  *      Preserve the provided registers on the stack plus any additional
  53  *      data as needed by the implementation including this code. Called
  54  *      upon code entry.
  55  *
  56  * exit reg1 reg2
  57  *
  58  *      Restore registers with the values previously saved with the
  59  *      'preserv' macro. Called upon code termination.
  60  *
  61  * LDR1W_SHIFT
  62  * STR1W_SHIFT
  63  *
  64  *      Correction to be applied to the "ip" register when branching into
  65  *      the ldr1w or str1w instructions (some of these macros may expand to
  66  *      than one 32bit instruction in Thumb-2)
  67  */
  68
  69
  70                 enter   r4, lr
  71
  72                 subs    r2, r2, #4
  73                 blt     8f
  74                 ands    ip, r0, #3
  75         PLD(    pld     [r1, #0]                )
  76                 bne     9f
  77                 ands    ip, r1, #3
  78                 bne     10f
  79
  80 1:              subs    r2, r2, #(28)
  81                 stmfd   sp!, {r5 - r8}
  82                 blt     5f
  83
  84         CALGN(  ands    ip, r0, #31             )
  85         CALGN(  rsb     r3, ip, #32             )
  86         CALGN(  sbcnes  r4, r3, r2              )  @ C is always set here
  87         CALGN(  bcs     2f                      )
  88         CALGN(  adr     r4, 6f                  )
  89         CALGN(  subs    r2, r2, r3              )  @ C gets set
  90         CALGN(  add     pc, r4, ip              )
  91
  92         PLD(    pld     [r1, #0]                )
  93 2:      PLD(    subs    r2, r2, #96             )
  94         PLD(    pld     [r1, #28]               )
  95         PLD(    blt     4f                      )
  96         PLD(    pld     [r1, #60]               )
  97         PLD(    pld     [r1, #92]               )
  98
  99 3:      PLD(    pld     [r1, #124]              )
 100 4:              ldr8w   r1, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
 101                 subs    r2, r2, #32
 102                 str8w   r0, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
 103                 bge     3b
 104         PLD(    cmn     r2, #96                 )
 105         PLD(    bge     4b                      )
 106
 107 5:              ands    ip, r2, #28
 108                 rsb     ip, ip, #32
 109 #if LDR1W_SHIFT > 0
 110                 lsl     ip, ip, #LDR1W_SHIFT
 111 #endif
 112                 addne   pc, pc, ip              @ C is always clear here
 113                 b       7f
 114 6:
 115                 .rept   (1 << LDR1W_SHIFT)
 116                 W(nop)
 117                 .endr
 118                 ldr1w   r1, r3, abort=20f
 119                 ldr1w   r1, r4, abort=20f
 120                 ldr1w   r1, r5, abort=20f
 121                 ldr1w   r1, r6, abort=20f
 122                 ldr1w   r1, r7, abort=20f
 123                 ldr1w   r1, r8, abort=20f
 124                 ldr1w   r1, lr, abort=20f
 125
 126 #if LDR1W_SHIFT < STR1W_SHIFT
 127                 lsl     ip, ip, #STR1W_SHIFT - LDR1W_SHIFT
 128 #elif LDR1W_SHIFT > STR1W_SHIFT
 129                 lsr     ip, ip, #LDR1W_SHIFT - STR1W_SHIFT
 130 #endif
 131                 add     pc, pc, ip
 132                 nop
 133                 .rept   (1 << STR1W_SHIFT)
 134                 W(nop)
 135                 .endr
 136                 str1w   r0, r3, abort=20f
 137                 str1w   r0, r4, abort=20f
 138                 str1w   r0, r5, abort=20f
 139                 str1w   r0, r6, abort=20f
 140                 str1w   r0, r7, abort=20f
 141                 str1w   r0, r8, abort=20f
 142                 str1w   r0, lr, abort=20f
 143
 144         CALGN(  bcs     2b                      )
 145
 146 7:              ldmfd   sp!, {r5 - r8}
 147
 148 8:              movs    r2, r2, lsl #31
 149                 ldr1b   r1, r3, ne, abort=21f
 150                 ldr1b   r1, r4, cs, abort=21f
 151                 ldr1b   r1, ip, cs, abort=21f
 152                 str1b   r0, r3, ne, abort=21f
 153                 str1b   r0, r4, cs, abort=21f
 154                 str1b   r0, ip, cs, abort=21f
 155
 156                 exit    r4, pc
 157
 158 9:              rsb     ip, ip, #4
 159                 cmp     ip, #2
 160                 ldr1b   r1, r3, gt, abort=21f
 161                 ldr1b   r1, r4, ge, abort=21f
 162                 ldr1b   r1, lr, abort=21f
 163                 str1b   r0, r3, gt, abort=21f
 164                 str1b   r0, r4, ge, abort=21f
 165                 subs    r2, r2, ip
 166                 str1b   r0, lr, abort=21f
 167                 blt     8b
 168                 ands    ip, r1, #3
 169                 beq     1b
 170
 171 10:             bic     r1, r1, #3
 172                 cmp     ip, #2
 173                 ldr1w   r1, lr, abort=21f
 174                 beq     17f
 175                 bgt     18f
 176
 177
 178                 .macro  forward_copy_shift pull push
 179
 180                 subs    r2, r2, #28
 181                 blt     14f
 182
 183         CALGN(  ands    ip, r0, #31             )
 184         CALGN(  rsb     ip, ip, #32             )
 185         CALGN(  sbcnes  r4, ip, r2              )  @ C is always set here
 186         CALGN(  subcc   r2, r2, ip              )
 187         CALGN(  bcc     15f                     )
 188
 189 11:             stmfd   sp!, {r5 - r9}
 190
 191         PLD(    pld     [r1, #0]                )
 192         PLD(    subs    r2, r2, #96             )
 193         PLD(    pld     [r1, #28]               )
 194         PLD(    blt     13f                     )
 195         PLD(    pld     [r1, #60]               )
 196         PLD(    pld     [r1, #92]               )
 197
 198 12:     PLD(    pld     [r1, #124]              )
 199 13:             ldr4w   r1, r4, r5, r6, r7, abort=19f
 200                 mov     r3, lr, pull #\pull
 201                 subs    r2, r2, #32
 202                 ldr4w   r1, r8, r9, ip, lr, abort=19f
 203                 orr     r3, r3, r4, push #\push
 204                 mov     r4, r4, pull #\pull
 205                 orr     r4, r4, r5, push #\push
 206                 mov     r5, r5, pull #\pull
 207                 orr     r5, r5, r6, push #\push
 208                 mov     r6, r6, pull #\pull
 209                 orr     r6, r6, r7, push #\push
 210                 mov     r7, r7, pull #\pull
 211                 orr     r7, r7, r8, push #\push
 212                 mov     r8, r8, pull #\pull
 213                 orr     r8, r8, r9, push #\push
 214                 mov     r9, r9, pull #\pull
 215                 orr     r9, r9, ip, push #\push
 216                 mov     ip, ip, pull #\pull
 217                 orr     ip, ip, lr, push #\push
 218                 str8w   r0, r3, r4, r5, r6, r7, r8, r9, ip, , abort=19f
 219                 bge     12b
 220         PLD(    cmn     r2, #96                 )
 221         PLD(    bge     13b                     )
 222
 223                 ldmfd   sp!, {r5 - r9}
 224
 225 14:             ands    ip, r2, #28
 226                 beq     16f
 227
 228 15:             mov     r3, lr, pull #\pull
 229                 ldr1w   r1, lr, abort=21f
 230                 subs    ip, ip, #4
 231                 orr     r3, r3, lr, push #\push
 232                 str1w   r0, r3, abort=21f
 233                 bgt     15b
 234         CALGN(  cmp     r2, #0                  )
 235         CALGN(  bge     11b                     )
 236
 237 16:             sub     r1, r1, #(\push / 8)
 238                 b       8b
 239
 240                 .endm
 241
 242
 243                 forward_copy_shift      pull=8  push=24
 244
 245 17:             forward_copy_shift      pull=16 push=16
 246
 247 18:             forward_copy_shift      pull=24 push=8
 248
 249
 250 /*
 251  * Abort preamble and completion macros.
 252  * If a fixup handler is required then those macros must surround it.
 253  * It is assumed that the fixup code will handle the private part of
 254  * the exit macro.
 255  */
 256
 257         .macro  copy_abort_preamble
 258 19:     ldmfd   sp!, {r5 - r9}
 259         b       21f
 260 20:     ldmfd   sp!, {r5 - r8}
 261 21:
 262         .endm
 263
 264         .macro  copy_abort_end
 265         ldmfd   sp!, {r4, pc}
 266         .endm
 267