arch/alpha/lib/ev6-memcpy.S

   1 /*
   2  * arch/alpha/lib/ev6-memcpy.S
   3  * 21264 version by Rick Gorton <rick.gorton@alpha-processor.com>
   4  *
   5  * Reasonably optimized memcpy() routine for the Alpha 21264
   6  *
   7  *      - memory accessed as aligned quadwords only
   8  *      - uses bcmpge to compare 8 bytes in parallel
   9  *
  10  * Much of the information about 21264 scheduling/coding comes from:
  11  *      Compiler Writer's Guide for the Alpha 21264
  12  *      abbreviated as 'CWG' in other comments here
  13  *      ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html
  14  * Scheduling notation:
  15  *      E       - either cluster
  16  *      U       - upper subcluster; U0 - subcluster U0; U1 - subcluster U1
  17  *      L       - lower subcluster; L0 - subcluster L0; L1 - subcluster L1
  18  *
  19  * Temp usage notes:
  20  *      $1,$2,          - scratch
  21  */
  22
  23         .set noreorder
  24         .set noat
  25
  26         .align  4
  27         .globl memcpy
  28         .ent memcpy
  29 memcpy:
  30         .frame $30,0,$26,0
  31         .prologue 0
  32
  33         mov     $16, $0                 # E : copy dest to return
  34         ble     $18, $nomoredata        # U : done with the copy?
  35         xor     $16, $17, $1            # E : are source and dest alignments the same?
  36         and     $1, 7, $1               # E : are they the same mod 8?
  37
  38         bne     $1, $misaligned         # U : Nope - gotta do this the slow way
  39         /* source and dest are same mod 8 address */
  40         and     $16, 7, $1              # E : Are both 0mod8?
  41         beq     $1, $both_0mod8         # U : Yes
  42         nop                             # E :
  43
  44         /*
  45          * source and dest are same misalignment.  move a byte at a time
  46          * until a 0mod8 alignment for both is reached.
  47          * At least one byte more to move
  48          */
  49
  50 $head_align:
  51         ldbu    $1, 0($17)              # L : grab a byte
  52         subq    $18, 1, $18             # E : count--
  53         addq    $17, 1, $17             # E : src++
  54         stb     $1, 0($16)              # L :
  55         addq    $16, 1, $16             # E : dest++
  56         and     $16, 7, $1              # E : Are we at 0mod8 yet?
  57         ble     $18, $nomoredata        # U : done with the copy?
  58         bne     $1, $head_align         # U :
  59
  60 $both_0mod8:
  61         cmple   $18, 127, $1            # E : Can we unroll the loop?
  62         bne     $1, $no_unroll          # U :
  63         and     $16, 63, $1             # E : get mod64 alignment
  64         beq     $1, $do_unroll          # U : no single quads to fiddle
  65
  66 $single_head_quad:
  67         ldq     $1, 0($17)              # L : get 8 bytes
  68         subq    $18, 8, $18             # E : count -= 8
  69         addq    $17, 8, $17             # E : src += 8
  70         nop                             # E :
  71
  72         stq     $1, 0($16)              # L : store
  73         addq    $16, 8, $16             # E : dest += 8
  74         and     $16, 63, $1             # E : get mod64 alignment
  75         bne     $1, $single_head_quad   # U : still not fully aligned
  76
  77 $do_unroll:
  78         addq    $16, 64, $7             # E : Initial (+1 trip) wh64 address
  79         cmple   $18, 127, $1            # E : Can we go through the unrolled loop?
  80         bne     $1, $tail_quads         # U : Nope
  81         nop                             # E :
  82
  83 $unroll_body:
  84         wh64    ($7)                    # L1 : memory subsystem hint: 64 bytes at
  85                                         # ($7) are about to be over-written
  86         ldq     $6, 0($17)              # L0 : bytes 0..7
  87         nop                             # E :
  88         nop                             # E :
  89
  90         ldq     $4, 8($17)              # L : bytes 8..15
  91         ldq     $5, 16($17)             # L : bytes 16..23
  92         addq    $7, 64, $7              # E : Update next wh64 address
  93         nop                             # E :
  94
  95         ldq     $3, 24($17)             # L : bytes 24..31
  96         addq    $16, 64, $1             # E : fallback value for wh64
  97         nop                             # E :
  98         nop                             # E :
  99
 100         addq    $17, 32, $17            # E : src += 32 bytes
 101         stq     $6, 0($16)              # L : bytes 0..7
 102         nop                             # E :
 103         nop                             # E :
 104
 105         stq     $4, 8($16)              # L : bytes 8..15
 106         stq     $5, 16($16)             # L : bytes 16..23
 107         subq    $18, 192, $2            # E : At least two more trips to go?
 108         nop                             # E :
 109
 110         stq     $3, 24($16)             # L : bytes 24..31
 111         addq    $16, 32, $16            # E : dest += 32 bytes
 112         nop                             # E :
 113         nop                             # E :
 114
 115         ldq     $6, 0($17)              # L : bytes 0..7
 116         ldq     $4, 8($17)              # L : bytes 8..15
 117         cmovlt  $2, $1, $7              # E : Latency 2, extra map slot - Use
 118                                         # fallback wh64 address if < 2 more trips
 119         nop                             # E :
 120
 121         ldq     $5, 16($17)             # L : bytes 16..23
 122         ldq     $3, 24($17)             # L : bytes 24..31
 123         addq    $16, 32, $16            # E : dest += 32
 124         subq    $18, 64, $18            # E : count -= 64
 125
 126         addq    $17, 32, $17            # E : src += 32
 127         stq     $6, -32($16)            # L : bytes 0..7
 128         stq     $4, -24($16)            # L : bytes 8..15
 129         cmple   $18, 63, $1             # E : At least one more trip?
 130
 131         stq     $5, -16($16)            # L : bytes 16..23
 132         stq     $3, -8($16)             # L : bytes 24..31
 133         nop                             # E :
 134         beq     $1, $unroll_body
 135
 136 $tail_quads:
 137 $no_unroll:
 138         .align 4
 139         subq    $18, 8, $18             # E : At least a quad left?
 140         blt     $18, $less_than_8       # U : Nope
 141         nop                             # E :
 142         nop                             # E :
 143
 144 $move_a_quad:
 145         ldq     $1, 0($17)              # L : fetch 8
 146         subq    $18, 8, $18             # E : count -= 8
 147         addq    $17, 8, $17             # E : src += 8
 148         nop                             # E :
 149
 150         stq     $1, 0($16)              # L : store 8
 151         addq    $16, 8, $16             # E : dest += 8
 152         bge     $18, $move_a_quad       # U :
 153         nop                             # E :
 154
 155 $less_than_8:
 156         .align 4
 157         addq    $18, 8, $18             # E : add back for trailing bytes
 158         ble     $18, $nomoredata        # U : All-done
 159         nop                             # E :
 160         nop                             # E :
 161
 162         /* Trailing bytes */
 163 $tail_bytes:
 164         subq    $18, 1, $18             # E : count--
 165         ldbu    $1, 0($17)              # L : fetch a byte
 166         addq    $17, 1, $17             # E : src++
 167         nop                             # E :
 168
 169         stb     $1, 0($16)              # L : store a byte
 170         addq    $16, 1, $16             # E : dest++
 171         bgt     $18, $tail_bytes        # U : more to be done?
 172         nop                             # E :
 173
 174         /* branching to exit takes 3 extra cycles, so replicate exit here */
 175         ret     $31, ($26), 1           # L0 :
 176         nop                             # E :
 177         nop                             # E :
 178         nop                             # E :
 179
 180 $misaligned:
 181         mov     $0, $4                  # E : dest temp
 182         and     $0, 7, $1               # E : dest alignment mod8
 183         beq     $1, $dest_0mod8         # U : life doesnt totally suck
 184         nop
 185
 186 $aligndest:
 187         ble     $18, $nomoredata        # U :
 188         ldbu    $1, 0($17)              # L : fetch a byte
 189         subq    $18, 1, $18             # E : count--
 190         addq    $17, 1, $17             # E : src++
 191
 192         stb     $1, 0($4)               # L : store it
 193         addq    $4, 1, $4               # E : dest++
 194         and     $4, 7, $1               # E : dest 0mod8 yet?
 195         bne     $1, $aligndest          # U : go until we are aligned.
 196
 197         /* Source has unknown alignment, but dest is known to be 0mod8 */
 198 $dest_0mod8:
 199         subq    $18, 8, $18             # E : At least a quad left?
 200         blt     $18, $misalign_tail     # U : Nope
 201         ldq_u   $3, 0($17)              # L : seed (rotating load) of 8 bytes
 202         nop                             # E :
 203
 204 $mis_quad:
 205         ldq_u   $16, 8($17)             # L : Fetch next 8
 206         extql   $3, $17, $3             # U : masking
 207         extqh   $16, $17, $1            # U : masking
 208         bis     $3, $1, $1              # E : merged bytes to store
 209
 210         subq    $18, 8, $18             # E : count -= 8
 211         addq    $17, 8, $17             # E : src += 8
 212         stq     $1, 0($4)               # L : store 8 (aligned)
 213         mov     $16, $3                 # E : "rotate" source data
 214
 215         addq    $4, 8, $4               # E : dest += 8
 216         bge     $18, $mis_quad          # U : More quads to move
 217         nop
 218         nop
 219
 220 $misalign_tail:
 221         addq    $18, 8, $18             # E : account for tail stuff
 222         ble     $18, $nomoredata        # U :
 223         nop
 224         nop
 225
 226 $misalign_byte:
 227         ldbu    $1, 0($17)              # L : fetch 1
 228         subq    $18, 1, $18             # E : count--
 229         addq    $17, 1, $17             # E : src++
 230         nop                             # E :
 231
 232         stb     $1, 0($4)               # L : store
 233         addq    $4, 1, $4               # E : dest++
 234         bgt     $18, $misalign_byte     # U : more to go?
 235         nop
 236
 237
 238 $nomoredata:
 239         ret     $31, ($26), 1           # L0 :
 240         nop                             # E :
 241         nop                             # E :
 242         nop                             # E :
 243
 244         .end memcpy
 245
 246 /* For backwards module compatibility.  */
 247 __memcpy = memcpy
 248 .globl __memcpy