zfs/lib/libsolcompat/amd64/atomic.S

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25
  26         .ident  "%Z%%M% %I%     %E% SMI"
  27
  28         .file   "%M%"
  29
  30 #define _ASM
  31 #include <sys/asm_linkage.h>
  32
  33 #if defined(_KERNEL)
  34         /*
  35          * Legacy kernel interfaces; they will go away (eventually).
  36          */
  37         ANSI_PRAGMA_WEAK2(cas8,atomic_cas_8,function)
  38         ANSI_PRAGMA_WEAK2(cas32,atomic_cas_32,function)
  39         ANSI_PRAGMA_WEAK2(cas64,atomic_cas_64,function)
  40         ANSI_PRAGMA_WEAK2(caslong,atomic_cas_ulong,function)
  41         ANSI_PRAGMA_WEAK2(casptr,atomic_cas_ptr,function)
  42         ANSI_PRAGMA_WEAK2(atomic_and_long,atomic_and_ulong,function)
  43         ANSI_PRAGMA_WEAK2(atomic_or_long,atomic_or_ulong,function)
  44 #else
  45         /*
  46          * Include the definitions for the libc weak aliases.
  47          */
  48 #include "../atomic_asm_weak.h"
  49 #endif
  50
  51         ENTRY(atomic_inc_8)
  52         ALTENTRY(atomic_inc_uchar)
  53         lock
  54         incb    (%rdi)
  55         ret
  56         SET_SIZE(atomic_inc_uchar)
  57         SET_SIZE(atomic_inc_8)
  58
  59         ENTRY(atomic_inc_16)
  60         ALTENTRY(atomic_inc_ushort)
  61         lock
  62         incw    (%rdi)
  63         ret
  64         SET_SIZE(atomic_inc_ushort)
  65         SET_SIZE(atomic_inc_16)
  66
  67         ENTRY(atomic_inc_32)
  68         ALTENTRY(atomic_inc_uint)
  69         lock
  70         incl    (%rdi)
  71         ret
  72         SET_SIZE(atomic_inc_uint)
  73         SET_SIZE(atomic_inc_32)
  74
  75         ENTRY(atomic_inc_64)
  76         ALTENTRY(atomic_inc_ulong)
  77         lock
  78         incq    (%rdi)
  79         ret
  80         SET_SIZE(atomic_inc_ulong)
  81         SET_SIZE(atomic_inc_64)
  82
  83         ENTRY(atomic_inc_8_nv)
  84         ALTENTRY(atomic_inc_uchar_nv)
  85         movb    (%rdi), %al
  86 1:
  87         leaq    1(%rax), %rcx
  88         lock
  89         cmpxchgb %cl, (%rdi)
  90         jne     1b
  91         movzbl  %cl, %eax
  92         ret
  93         SET_SIZE(atomic_inc_uchar_nv)
  94         SET_SIZE(atomic_inc_8_nv)
  95
  96         ENTRY(atomic_inc_16_nv)
  97         ALTENTRY(atomic_inc_ushort_nv)
  98         movw    (%rdi), %ax
  99 1:
 100         leaq    1(%rax), %rcx
 101         lock
 102         cmpxchgw %cx, (%rdi)
 103         jne     1b
 104         movzwl  %cx, %eax
 105         ret
 106         SET_SIZE(atomic_inc_ushort_nv)
 107         SET_SIZE(atomic_inc_16_nv)
 108
 109         ENTRY(atomic_inc_32_nv)
 110         ALTENTRY(atomic_inc_uint_nv)
 111         movl    (%rdi), %eax
 112 1:
 113         leaq    1(%rax), %rcx
 114         lock
 115         cmpxchgl %ecx, (%rdi)
 116         jne     1b
 117         movl    %ecx, %eax
 118         ret
 119         SET_SIZE(atomic_inc_uint_nv)
 120         SET_SIZE(atomic_inc_32_nv)
 121
 122         ENTRY(atomic_inc_64_nv)
 123         ALTENTRY(atomic_inc_ulong_nv)
 124         movq    (%rdi), %rax
 125 1:
 126         leaq    1(%rax), %rcx
 127         lock
 128         cmpxchgq %rcx, (%rdi)
 129         jne     1b
 130         movq    %rcx, %rax
 131         ret
 132         SET_SIZE(atomic_inc_ulong_nv)
 133         SET_SIZE(atomic_inc_64_nv)
 134
 135         ENTRY(atomic_dec_8)
 136         ALTENTRY(atomic_dec_uchar)
 137         lock
 138         decb    (%rdi)
 139         ret
 140         SET_SIZE(atomic_dec_uchar)
 141         SET_SIZE(atomic_dec_8)
 142
 143         ENTRY(atomic_dec_16)
 144         ALTENTRY(atomic_dec_ushort)
 145         lock
 146         decw    (%rdi)
 147         ret
 148         SET_SIZE(atomic_dec_ushort)
 149         SET_SIZE(atomic_dec_16)
 150
 151         ENTRY(atomic_dec_32)
 152         ALTENTRY(atomic_dec_uint)
 153         lock
 154         decl    (%rdi)
 155         ret
 156         SET_SIZE(atomic_dec_uint)
 157         SET_SIZE(atomic_dec_32)
 158
 159         ENTRY(atomic_dec_64)
 160         ALTENTRY(atomic_dec_ulong)
 161         lock
 162         decq    (%rdi)
 163         ret
 164         SET_SIZE(atomic_dec_ulong)
 165         SET_SIZE(atomic_dec_64)
 166
 167         ENTRY(atomic_dec_8_nv)
 168         ALTENTRY(atomic_dec_uchar_nv)
 169         movb    (%rdi), %al
 170 1:
 171         leaq    -1(%rax), %rcx
 172         lock
 173         cmpxchgb %cl, (%rdi)
 174         jne     1b
 175         movzbl  %cl, %eax
 176         ret
 177         SET_SIZE(atomic_dec_uchar_nv)
 178         SET_SIZE(atomic_dec_8_nv)
 179
 180         ENTRY(atomic_dec_16_nv)
 181         ALTENTRY(atomic_dec_ushort_nv)
 182         movw    (%rdi), %ax
 183 1:
 184         leaq    -1(%rax), %rcx
 185         lock
 186         cmpxchgw %cx, (%rdi)
 187         jne     1b
 188         movzwl  %cx, %eax
 189         ret
 190         SET_SIZE(atomic_dec_ushort_nv)
 191         SET_SIZE(atomic_dec_16_nv)
 192
 193         ENTRY(atomic_dec_32_nv)
 194         ALTENTRY(atomic_dec_uint_nv)
 195         movl    (%rdi), %eax
 196 1:
 197         leaq    -1(%rax), %rcx
 198         lock
 199         cmpxchgl %ecx, (%rdi)
 200         jne     1b
 201         movl    %ecx, %eax
 202         ret
 203         SET_SIZE(atomic_dec_uint_nv)
 204         SET_SIZE(atomic_dec_32_nv)
 205
 206         ENTRY(atomic_dec_64_nv)
 207         ALTENTRY(atomic_dec_ulong_nv)
 208         movq    (%rdi), %rax
 209 1:
 210         leaq    -1(%rax), %rcx
 211         lock
 212         cmpxchgq %rcx, (%rdi)
 213         jne     1b
 214         movq    %rcx, %rax
 215         ret
 216         SET_SIZE(atomic_dec_ulong_nv)
 217         SET_SIZE(atomic_dec_64_nv)
 218
 219         ENTRY(atomic_add_8)
 220         ALTENTRY(atomic_add_char)
 221         lock
 222         addb    %sil, (%rdi)
 223         ret
 224         SET_SIZE(atomic_add_char)
 225         SET_SIZE(atomic_add_8)
 226
 227         ENTRY(atomic_add_16)
 228         ALTENTRY(atomic_add_short)
 229         lock
 230         addw    %si, (%rdi)
 231         ret
 232         SET_SIZE(atomic_add_short)
 233         SET_SIZE(atomic_add_16)
 234
 235         ENTRY(atomic_add_32)
 236         ALTENTRY(atomic_add_int)
 237         lock
 238         addl    %esi, (%rdi)
 239         ret
 240         SET_SIZE(atomic_add_int)
 241         SET_SIZE(atomic_add_32)
 242
 243         ENTRY(atomic_add_64)
 244         ALTENTRY(atomic_add_ptr)
 245         ALTENTRY(atomic_add_long)
 246         lock
 247         addq    %rsi, (%rdi)
 248         ret
 249         SET_SIZE(atomic_add_long)
 250         SET_SIZE(atomic_add_ptr)
 251         SET_SIZE(atomic_add_64)
 252
 253         ENTRY(atomic_or_8)
 254         ALTENTRY(atomic_or_uchar)
 255         lock
 256         orb     %sil, (%rdi)
 257         ret
 258         SET_SIZE(atomic_or_uchar)
 259         SET_SIZE(atomic_or_8)
 260
 261         ENTRY(atomic_or_16)
 262         ALTENTRY(atomic_or_ushort)
 263         lock
 264         orw     %si, (%rdi)
 265         ret
 266         SET_SIZE(atomic_or_ushort)
 267         SET_SIZE(atomic_or_16)
 268
 269         ENTRY(atomic_or_32)
 270         ALTENTRY(atomic_or_uint)
 271         lock
 272         orl     %esi, (%rdi)
 273         ret
 274         SET_SIZE(atomic_or_uint)
 275         SET_SIZE(atomic_or_32)
 276
 277         ENTRY(atomic_or_64)
 278         ALTENTRY(atomic_or_ulong)
 279         lock
 280         orq     %rsi, (%rdi)
 281         ret
 282         SET_SIZE(atomic_or_ulong)
 283         SET_SIZE(atomic_or_64)
 284
 285         ENTRY(atomic_and_8)
 286         ALTENTRY(atomic_and_uchar)
 287         lock
 288         andb    %sil, (%rdi)
 289         ret
 290         SET_SIZE(atomic_and_uchar)
 291         SET_SIZE(atomic_and_8)
 292
 293         ENTRY(atomic_and_16)
 294         ALTENTRY(atomic_and_ushort)
 295         lock
 296         andw    %si, (%rdi)
 297         ret
 298         SET_SIZE(atomic_and_ushort)
 299         SET_SIZE(atomic_and_16)
 300
 301         ENTRY(atomic_and_32)
 302         ALTENTRY(atomic_and_uint)
 303         lock
 304         andl    %esi, (%rdi)
 305         ret
 306         SET_SIZE(atomic_and_uint)
 307         SET_SIZE(atomic_and_32)
 308
 309         ENTRY(atomic_and_64)
 310         ALTENTRY(atomic_and_ulong)
 311         lock
 312         andq    %rsi, (%rdi)
 313         ret
 314         SET_SIZE(atomic_and_ulong)
 315         SET_SIZE(atomic_and_64)
 316
 317         ENTRY(atomic_add_8_nv)
 318         ALTENTRY(atomic_add_char_nv)
 319         movb    (%rdi), %al
 320 1:
 321         movb    %sil, %cl
 322         addb    %al, %cl
 323         lock
 324         cmpxchgb %cl, (%rdi)
 325         jne     1b
 326         movzbl  %cl, %eax
 327         ret
 328         SET_SIZE(atomic_add_char_nv)
 329         SET_SIZE(atomic_add_8_nv)
 330
 331         ENTRY(atomic_add_16_nv)
 332         ALTENTRY(atomic_add_short_nv)
 333         movw    (%rdi), %ax
 334 1:
 335         movw    %si, %cx
 336         addw    %ax, %cx
 337         lock
 338         cmpxchgw %cx, (%rdi)
 339         jne     1b
 340         movzwl  %cx, %eax
 341         ret
 342         SET_SIZE(atomic_add_short_nv)
 343         SET_SIZE(atomic_add_16_nv)
 344
 345         ENTRY(atomic_add_32_nv)
 346         ALTENTRY(atomic_add_int_nv)
 347         movl    (%rdi), %eax
 348 1:
 349         movl    %esi, %ecx
 350         addl    %eax, %ecx
 351         lock
 352         cmpxchgl %ecx, (%rdi)
 353         jne     1b
 354         movl    %ecx, %eax
 355         ret
 356         SET_SIZE(atomic_add_int_nv)
 357         SET_SIZE(atomic_add_32_nv)
 358
 359         ENTRY(atomic_add_64_nv)
 360         ALTENTRY(atomic_add_ptr_nv)
 361         ALTENTRY(atomic_add_long_nv)
 362         movq    (%rdi), %rax
 363 1:
 364         movq    %rsi, %rcx
 365         addq    %rax, %rcx
 366         lock
 367         cmpxchgq %rcx, (%rdi)
 368         jne     1b
 369         movq    %rcx, %rax
 370         ret
 371         SET_SIZE(atomic_add_long_nv)
 372         SET_SIZE(atomic_add_ptr_nv)
 373         SET_SIZE(atomic_add_64_nv)
 374
 375         ENTRY(atomic_and_8_nv)
 376         ALTENTRY(atomic_and_uchar_nv)
 377         movb    (%rdi), %al
 378 1:
 379         movb    %sil, %cl
 380         andb    %al, %cl
 381         lock
 382         cmpxchgb %cl, (%rdi)
 383         jne     1b
 384         movzbl  %cl, %eax
 385         ret
 386         SET_SIZE(atomic_and_uchar_nv)
 387         SET_SIZE(atomic_and_8_nv)
 388
 389         ENTRY(atomic_and_16_nv)
 390         ALTENTRY(atomic_and_ushort_nv)
 391         movw    (%rdi), %ax
 392 1:
 393         movw    %si, %cx
 394         andw    %ax, %cx
 395         lock
 396         cmpxchgw %cx, (%rdi)
 397         jne     1b
 398         movzwl  %cx, %eax
 399         ret
 400         SET_SIZE(atomic_and_ushort_nv)
 401         SET_SIZE(atomic_and_16_nv)
 402
 403         ENTRY(atomic_and_32_nv)
 404         ALTENTRY(atomic_and_uint_nv)
 405         movl    (%rdi), %eax
 406 1:
 407         movl    %esi, %ecx
 408         andl    %eax, %ecx
 409         lock
 410         cmpxchgl %ecx, (%rdi)
 411         jne     1b
 412         movl    %ecx, %eax
 413         ret
 414         SET_SIZE(atomic_and_uint_nv)
 415         SET_SIZE(atomic_and_32_nv)
 416
 417         ENTRY(atomic_and_64_nv)
 418         ALTENTRY(atomic_and_ulong_nv)
 419         movq    (%rdi), %rax
 420 1:
 421         movq    %rsi, %rcx
 422         andq    %rax, %rcx
 423         lock
 424         cmpxchgq %rcx, (%rdi)
 425         jne     1b
 426         movq    %rcx, %rax
 427         ret
 428         SET_SIZE(atomic_and_ulong_nv)
 429         SET_SIZE(atomic_and_64_nv)
 430
 431         ENTRY(atomic_or_8_nv)
 432         ALTENTRY(atomic_or_uchar_nv)
 433         movb    (%rdi), %al
 434 1:
 435         movb    %sil, %cl
 436         orb     %al, %cl
 437         lock
 438         cmpxchgb %cl, (%rdi)
 439         jne     1b
 440         movzbl  %cl, %eax
 441         ret
 442         SET_SIZE(atomic_and_uchar_nv)
 443         SET_SIZE(atomic_and_8_nv)
 444
 445         ENTRY(atomic_or_16_nv)
 446         ALTENTRY(atomic_or_ushort_nv)
 447         movw    (%rdi), %ax
 448 1:
 449         movw    %si, %cx
 450         orw     %ax, %cx
 451         lock
 452         cmpxchgw %cx, (%rdi)
 453         jne     1b
 454         movzwl  %cx, %eax
 455         ret
 456         SET_SIZE(atomic_or_ushort_nv)
 457         SET_SIZE(atomic_or_16_nv)
 458
 459         ENTRY(atomic_or_32_nv)
 460         ALTENTRY(atomic_or_uint_nv)
 461         movl    (%rdi), %eax
 462 1:
 463         movl    %esi, %ecx
 464         orl     %eax, %ecx
 465         lock
 466         cmpxchgl %ecx, (%rdi)
 467         jne     1b
 468         movl    %ecx, %eax
 469         ret
 470         SET_SIZE(atomic_or_uint_nv)
 471         SET_SIZE(atomic_or_32_nv)
 472
 473         ENTRY(atomic_or_64_nv)
 474         ALTENTRY(atomic_or_ulong_nv)
 475         movq    (%rdi), %rax
 476 1:
 477         movq    %rsi, %rcx
 478         orq     %rax, %rcx
 479         lock
 480         cmpxchgq %rcx, (%rdi)
 481         jne     1b
 482         movq    %rcx, %rax
 483         ret
 484         SET_SIZE(atomic_or_ulong_nv)
 485         SET_SIZE(atomic_or_64_nv)
 486
 487         ENTRY(atomic_cas_8)
 488         ALTENTRY(atomic_cas_uchar)
 489         movzbl  %sil, %eax
 490         lock
 491         cmpxchgb %dl, (%rdi)
 492         ret
 493         SET_SIZE(atomic_cas_uchar)
 494         SET_SIZE(atomic_cas_8)
 495
 496         ENTRY(atomic_cas_16)
 497         ALTENTRY(atomic_cas_ushort)
 498         movzwl  %si, %eax
 499         lock
 500         cmpxchgw %dx, (%rdi)
 501         ret
 502         SET_SIZE(atomic_cas_ushort)
 503         SET_SIZE(atomic_cas_16)
 504
 505         ENTRY(atomic_cas_32)
 506         ALTENTRY(atomic_cas_uint)
 507         movl    %esi, %eax
 508         lock
 509         cmpxchgl %edx, (%rdi)
 510         ret
 511         SET_SIZE(atomic_cas_uint)
 512         SET_SIZE(atomic_cas_32)
 513
 514         ENTRY(atomic_cas_64)
 515         ALTENTRY(atomic_cas_ulong)
 516         ALTENTRY(atomic_cas_ptr)
 517         movq    %rsi, %rax
 518         lock
 519         cmpxchgq %rdx, (%rdi)
 520         ret
 521         SET_SIZE(atomic_cas_ptr)
 522         SET_SIZE(atomic_cas_ulong)
 523         SET_SIZE(atomic_cas_64)
 524
 525         ENTRY(atomic_swap_8)
 526         ALTENTRY(atomic_swap_uchar)
 527         movzbl  %sil, %eax
 528         lock
 529         xchgb %al, (%rdi)
 530         ret
 531         SET_SIZE(atomic_swap_uchar)
 532         SET_SIZE(atomic_swap_8)
 533
 534         ENTRY(atomic_swap_16)
 535         ALTENTRY(atomic_swap_ushort)
 536         movzwl  %si, %eax
 537         lock
 538         xchgw %ax, (%rdi)
 539         ret
 540         SET_SIZE(atomic_swap_ushort)
 541         SET_SIZE(atomic_swap_16)
 542
 543         ENTRY(atomic_swap_32)
 544         ALTENTRY(atomic_swap_uint)
 545         movl    %esi, %eax
 546         lock
 547         xchgl %eax, (%rdi)
 548         ret
 549         SET_SIZE(atomic_swap_uint)
 550         SET_SIZE(atomic_swap_32)
 551
 552         ENTRY(atomic_swap_64)
 553         ALTENTRY(atomic_swap_ulong)
 554         ALTENTRY(atomic_swap_ptr)
 555         movq    %rsi, %rax
 556         lock
 557         xchgq %rax, (%rdi)
 558         ret
 559         SET_SIZE(atomic_swap_ptr)
 560         SET_SIZE(atomic_swap_ulong)
 561         SET_SIZE(atomic_swap_64)
 562
 563         ENTRY(atomic_set_long_excl)
 564         xorl    %eax, %eax
 565         lock
 566         btsq    %rsi, (%rdi)
 567         jnc     1f
 568         decl    %eax
 569 1:
 570         ret
 571         SET_SIZE(atomic_set_long_excl)
 572
 573         ENTRY(atomic_clear_long_excl)
 574         xorl    %eax, %eax
 575         lock
 576         btrq    %rsi, (%rdi)
 577         jc      1f
 578         decl    %eax
 579 1:
 580         ret
 581         SET_SIZE(atomic_clear_long_excl)
 582
 583 #if !defined(_KERNEL)
 584
 585         /*
 586          * NOTE: membar_enter, and membar_exit are identical routines.
 587          * We define them separately, instead of using an ALTENTRY
 588          * definitions to alias them together, so that DTrace and
 589          * debuggers will see a unique address for them, allowing
 590          * more accurate tracing.
 591         */
 592
 593         ENTRY(membar_enter)
 594         mfence
 595         ret
 596         SET_SIZE(membar_enter)
 597
 598         ENTRY(membar_exit)
 599         mfence
 600         ret
 601         SET_SIZE(membar_exit)
 602
 603         ENTRY(membar_producer)
 604         sfence
 605         ret
 606         SET_SIZE(membar_producer)
 607
 608         ENTRY(membar_consumer)
 609         lfence
 610         ret
 611         SET_SIZE(membar_consumer)
 612
 613 #endif  /* !_KERNEL */
 614
 615 #ifdef __ELF__
 616 .section .note.GNU-stack,"",%progbits
 617 #endif