arch/arm/lib/copy_template.S

   1 /*
   2  *  linux/arch/arm/lib/copy_template.s
   3  *
   4  *  Code template for optimized memory copy functions
   5  *
   6  *  Author:     Nicolas Pitre
   7  *  Created:    Sep 28, 2005
   8  *  Copyright:  MontaVista Software, Inc.
   9  *
  10  *  This program is free software; you can redistribute it and/or modify
  11  *  it under the terms of the GNU General Public License version 2 as
  12  *  published by the Free Software Foundation.
  13  */
  14
  15 /*
  16  * Theory of operation
  17  * -------------------
  18  *
  19  * This file provides the core code for a forward memory copy used in
  20  * the implementation of memcopy(), copy_to_user() and copy_from_user().
  21  *
  22  * The including file must define the following accessor macros
  23  * according to the need of the given function:
  24  *
  25  * ldr1w ptr reg abort
  26  *
  27  *      This loads one word from 'ptr', stores it in 'reg' and increments
  28  *      'ptr' to the next word. The 'abort' argument is used for fixup tables.
  29  *
  30  * ldr4w ptr reg1 reg2 reg3 reg4 abort
  31  * ldr8w ptr, reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
  32  *
  33  *      This loads four or eight words starting from 'ptr', stores them
  34  *      in provided registers and increments 'ptr' past those words.
  35  *      The'abort' argument is used for fixup tables.
  36  *
  37  * ldr1b ptr reg cond abort
  38  *
  39  *      Similar to ldr1w, but it loads a byte and increments 'ptr' one byte.
  40  *      It also must apply the condition code if provided, otherwise the
  41  *      "al" condition is assumed by default.
  42  *
  43  * str1w ptr reg abort
  44  * str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
  45  * str1b ptr reg cond abort
  46  *
  47  *      Same as their ldr* counterparts, but data is stored to 'ptr' location
  48  *      rather than being loaded.
  49  *
  50  * enter reg1 reg2
  51  *
  52  *      Preserve the provided registers on the stack plus any additional
  53  *      data as needed by the implementation including this code. Called
  54  *      upon code entry.
  55  *
  56  * exit reg1 reg2
  57  *
  58  *      Restore registers with the values previously saved with the
  59  *      'preserv' macro. Called upon code termination.
  60  */
  61
  62
  63                 enter   r4, lr
  64
  65                 subs    r2, r2, #4
  66                 blt     8f
  67                 ands    ip, r0, #3
  68         PLD(    pld     [r1, #0]                )
  69                 bne     9f
  70                 ands    ip, r1, #3
  71                 bne     10f
  72
  73 1:              subs    r2, r2, #(28)
  74                 stmfd   sp!, {r5 - r8}
  75                 blt     5f
  76
  77         CALGN(  ands    ip, r0, #31             )
  78         CALGN(  rsb     r3, ip, #32             )
  79         CALGN(  sbcnes  r4, r3, r2              )  @ C is always set here
  80         CALGN(  bcs     2f                      )
  81         CALGN(  adr     r4, 6f                  )
  82         CALGN(  subs    r2, r2, r3              )  @ C gets set
  83         CALGN(  add     pc, r4, ip              )
  84
  85         PLD(    pld     [r1, #0]                )
  86 2:      PLD(    subs    r2, r2, #96             )
  87         PLD(    pld     [r1, #28]               )
  88         PLD(    blt     4f                      )
  89         PLD(    pld     [r1, #60]               )
  90         PLD(    pld     [r1, #92]               )
  91
  92 3:      PLD(    pld     [r1, #124]              )
  93 4:              ldr8w   r1, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
  94                 subs    r2, r2, #32
  95                 str8w   r0, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
  96                 bge     3b
  97         PLD(    cmn     r2, #96                 )
  98         PLD(    bge     4b                      )
  99
 100 5:              ands    ip, r2, #28
 101                 rsb     ip, ip, #32
 102                 addne   pc, pc, ip              @ C is always clear here
 103                 b       7f
 104 6:              nop
 105                 ldr1w   r1, r3, abort=20f
 106                 ldr1w   r1, r4, abort=20f
 107                 ldr1w   r1, r5, abort=20f
 108                 ldr1w   r1, r6, abort=20f
 109                 ldr1w   r1, r7, abort=20f
 110                 ldr1w   r1, r8, abort=20f
 111                 ldr1w   r1, lr, abort=20f
 112
 113                 add     pc, pc, ip
 114                 nop
 115                 nop
 116                 str1w   r0, r3, abort=20f
 117                 str1w   r0, r4, abort=20f
 118                 str1w   r0, r5, abort=20f
 119                 str1w   r0, r6, abort=20f
 120                 str1w   r0, r7, abort=20f
 121                 str1w   r0, r8, abort=20f
 122                 str1w   r0, lr, abort=20f
 123
 124         CALGN(  bcs     2b                      )
 125
 126 7:              ldmfd   sp!, {r5 - r8}
 127
 128 8:              movs    r2, r2, lsl #31
 129                 ldr1b   r1, r3, ne, abort=21f
 130                 ldr1b   r1, r4, cs, abort=21f
 131                 ldr1b   r1, ip, cs, abort=21f
 132                 str1b   r0, r3, ne, abort=21f
 133                 str1b   r0, r4, cs, abort=21f
 134                 str1b   r0, ip, cs, abort=21f
 135
 136                 exit    r4, pc
 137
 138 9:              rsb     ip, ip, #4
 139                 cmp     ip, #2
 140                 ldr1b   r1, r3, gt, abort=21f
 141                 ldr1b   r1, r4, ge, abort=21f
 142                 ldr1b   r1, lr, abort=21f
 143                 str1b   r0, r3, gt, abort=21f
 144                 str1b   r0, r4, ge, abort=21f
 145                 subs    r2, r2, ip
 146                 str1b   r0, lr, abort=21f
 147                 blt     8b
 148                 ands    ip, r1, #3
 149                 beq     1b
 150
 151 10:             bic     r1, r1, #3
 152                 cmp     ip, #2
 153                 ldr1w   r1, lr, abort=21f
 154                 beq     17f
 155                 bgt     18f
 156
 157
 158                 .macro  forward_copy_shift pull push
 159
 160                 subs    r2, r2, #28
 161                 blt     14f
 162
 163         CALGN(  ands    ip, r0, #31             )
 164         CALGN(  rsb     ip, ip, #32             )
 165         CALGN(  sbcnes  r4, ip, r2              )  @ C is always set here
 166         CALGN(  subcc   r2, r2, ip              )
 167         CALGN(  bcc     15f                     )
 168
 169 11:             stmfd   sp!, {r5 - r9}
 170
 171         PLD(    pld     [r1, #0]                )
 172         PLD(    subs    r2, r2, #96             )
 173         PLD(    pld     [r1, #28]               )
 174         PLD(    blt     13f                     )
 175         PLD(    pld     [r1, #60]               )
 176         PLD(    pld     [r1, #92]               )
 177
 178 12:     PLD(    pld     [r1, #124]              )
 179 13:             ldr4w   r1, r4, r5, r6, r7, abort=19f
 180                 mov     r3, lr, pull #\pull
 181                 subs    r2, r2, #32
 182                 ldr4w   r1, r8, r9, ip, lr, abort=19f
 183                 orr     r3, r3, r4, push #\push
 184                 mov     r4, r4, pull #\pull
 185                 orr     r4, r4, r5, push #\push
 186                 mov     r5, r5, pull #\pull
 187                 orr     r5, r5, r6, push #\push
 188                 mov     r6, r6, pull #\pull
 189                 orr     r6, r6, r7, push #\push
 190                 mov     r7, r7, pull #\pull
 191                 orr     r7, r7, r8, push #\push
 192                 mov     r8, r8, pull #\pull
 193                 orr     r8, r8, r9, push #\push
 194                 mov     r9, r9, pull #\pull
 195                 orr     r9, r9, ip, push #\push
 196                 mov     ip, ip, pull #\pull
 197                 orr     ip, ip, lr, push #\push
 198                 str8w   r0, r3, r4, r5, r6, r7, r8, r9, ip, , abort=19f
 199                 bge     12b
 200         PLD(    cmn     r2, #96                 )
 201         PLD(    bge     13b                     )
 202
 203                 ldmfd   sp!, {r5 - r9}
 204
 205 14:             ands    ip, r2, #28
 206                 beq     16f
 207
 208 15:             mov     r3, lr, pull #\pull
 209                 ldr1w   r1, lr, abort=21f
 210                 subs    ip, ip, #4
 211                 orr     r3, r3, lr, push #\push
 212                 str1w   r0, r3, abort=21f
 213                 bgt     15b
 214         CALGN(  cmp     r2, #0                  )
 215         CALGN(  bge     11b                     )
 216
 217 16:             sub     r1, r1, #(\push / 8)
 218                 b       8b
 219
 220                 .endm
 221
 222
 223                 forward_copy_shift      pull=8  push=24
 224
 225 17:             forward_copy_shift      pull=16 push=16
 226
 227 18:             forward_copy_shift      pull=24 push=8
 228
 229
 230 /*
 231  * Abort preamble and completion macros.
 232  * If a fixup handler is required then those macros must surround it.
 233  * It is assumed that the fixup code will handle the private part of
 234  * the exit macro.
 235  */
 236
 237         .macro  copy_abort_preamble
 238 19:     ldmfd   sp!, {r5 - r9}
 239         b       21f
 240 20:     ldmfd   sp!, {r5 - r8}
 241 21:
 242         .endm
 243
 244         .macro  copy_abort_end
 245         ldmfd   sp!, {r4, pc}
 246         .endm
 247