arch/x86/lib/atomic64_cx8_32.S

   1 /*
   2  * atomic64_t for 586+
   3  *
   4  * Copyright © 2010  Luca Barbieri
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  */
  11
  12 #include <linux/linkage.h>
  13 #include <asm/alternative-asm.h>
  14 #include <asm/dwarf2.h>
  15
  16 .macro SAVE reg
  17         pushl_cfi %\reg
  18         CFI_REL_OFFSET \reg, 0
  19 .endm
  20
  21 .macro RESTORE reg
  22         popl_cfi %\reg
  23         CFI_RESTORE \reg
  24 .endm
  25
  26 .macro read64 reg
  27         movl %ebx, %eax
  28         movl %ecx, %edx
  29 /* we need LOCK_PREFIX since otherwise cmpxchg8b always does the write */
  30         LOCK_PREFIX
  31         cmpxchg8b (\reg)
  32 .endm
  33
  34 ENTRY(atomic64_read_cx8)
  35         CFI_STARTPROC
  36
  37         read64 %ecx
  38         ret
  39         CFI_ENDPROC
  40 ENDPROC(atomic64_read_cx8)
  41
  42 ENTRY(atomic64_set_cx8)
  43         CFI_STARTPROC
  44
  45 1:
  46 /* we don't need LOCK_PREFIX since aligned 64-bit writes
  47  * are atomic on 586 and newer */
  48         cmpxchg8b (%esi)
  49         jne 1b
  50
  51         ret
  52         CFI_ENDPROC
  53 ENDPROC(atomic64_set_cx8)
  54
  55 ENTRY(atomic64_xchg_cx8)
  56         CFI_STARTPROC
  57
  58         movl %ebx, %eax
  59         movl %ecx, %edx
  60 1:
  61         LOCK_PREFIX
  62         cmpxchg8b (%esi)
  63         jne 1b
  64
  65         ret
  66         CFI_ENDPROC
  67 ENDPROC(atomic64_xchg_cx8)
  68
  69 .macro addsub_return func ins insc
  70 ENTRY(atomic64_\func\()_return_cx8)
  71         CFI_STARTPROC
  72         SAVE ebp
  73         SAVE ebx
  74         SAVE esi
  75         SAVE edi
  76
  77         movl %eax, %esi
  78         movl %edx, %edi
  79         movl %ecx, %ebp
  80
  81         read64 %ebp
  82 1:
  83         movl %eax, %ebx
  84         movl %edx, %ecx
  85         \ins\()l %esi, %ebx
  86         \insc\()l %edi, %ecx
  87         LOCK_PREFIX
  88         cmpxchg8b (%ebp)
  89         jne 1b
  90
  91 10:
  92         movl %ebx, %eax
  93         movl %ecx, %edx
  94         RESTORE edi
  95         RESTORE esi
  96         RESTORE ebx
  97         RESTORE ebp
  98         ret
  99         CFI_ENDPROC
 100 ENDPROC(atomic64_\func\()_return_cx8)
 101 .endm
 102
 103 addsub_return add add adc
 104 addsub_return sub sub sbb
 105
 106 .macro incdec_return func ins insc
 107 ENTRY(atomic64_\func\()_return_cx8)
 108         CFI_STARTPROC
 109         SAVE ebx
 110
 111         read64 %esi
 112 1:
 113         movl %eax, %ebx
 114         movl %edx, %ecx
 115         \ins\()l $1, %ebx
 116         \insc\()l $0, %ecx
 117         LOCK_PREFIX
 118         cmpxchg8b (%esi)
 119         jne 1b
 120
 121 10:
 122         movl %ebx, %eax
 123         movl %ecx, %edx
 124         RESTORE ebx
 125         ret
 126         CFI_ENDPROC
 127 ENDPROC(atomic64_\func\()_return_cx8)
 128 .endm
 129
 130 incdec_return inc add adc
 131 incdec_return dec sub sbb
 132
 133 ENTRY(atomic64_dec_if_positive_cx8)
 134         CFI_STARTPROC
 135         SAVE ebx
 136
 137         read64 %esi
 138 1:
 139         movl %eax, %ebx
 140         movl %edx, %ecx
 141         subl $1, %ebx
 142         sbb $0, %ecx
 143         js 2f
 144         LOCK_PREFIX
 145         cmpxchg8b (%esi)
 146         jne 1b
 147
 148 2:
 149         movl %ebx, %eax
 150         movl %ecx, %edx
 151         RESTORE ebx
 152         ret
 153         CFI_ENDPROC
 154 ENDPROC(atomic64_dec_if_positive_cx8)
 155
 156 ENTRY(atomic64_add_unless_cx8)
 157         CFI_STARTPROC
 158         SAVE ebp
 159         SAVE ebx
 160 /* these just push these two parameters on the stack */
 161         SAVE edi
 162         SAVE esi
 163
 164         movl %ecx, %ebp
 165         movl %eax, %esi
 166         movl %edx, %edi
 167
 168         read64 %ebp
 169 1:
 170         cmpl %eax, 0(%esp)
 171         je 4f
 172 2:
 173         movl %eax, %ebx
 174         movl %edx, %ecx
 175         addl %esi, %ebx
 176         adcl %edi, %ecx
 177         LOCK_PREFIX
 178         cmpxchg8b (%ebp)
 179         jne 1b
 180
 181         movl $1, %eax
 182 3:
 183         addl $8, %esp
 184         CFI_ADJUST_CFA_OFFSET -8
 185         RESTORE ebx
 186         RESTORE ebp
 187         ret
 188 4:
 189         cmpl %edx, 4(%esp)
 190         jne 2b
 191         xorl %eax, %eax
 192         jmp 3b
 193         CFI_ENDPROC
 194 ENDPROC(atomic64_add_unless_cx8)
 195
 196 ENTRY(atomic64_inc_not_zero_cx8)
 197         CFI_STARTPROC
 198         SAVE ebx
 199
 200         read64 %esi
 201 1:
 202         testl %eax, %eax
 203         je 4f
 204 2:
 205         movl %eax, %ebx
 206         movl %edx, %ecx
 207         addl $1, %ebx
 208         adcl $0, %ecx
 209         LOCK_PREFIX
 210         cmpxchg8b (%esi)
 211         jne 1b
 212
 213         movl $1, %eax
 214 3:
 215         RESTORE ebx
 216         ret
 217 4:
 218         testl %edx, %edx
 219         jne 2b
 220         jmp 3b
 221         CFI_ENDPROC
 222 ENDPROC(atomic64_inc_not_zero_cx8)