arch/x86/lib/hweight.S

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #include <linux/linkage.h>
   3 #include <asm/export.h>
   4
   5 #include <asm/asm.h>
   6
   7 /*
   8  * unsigned int __sw_hweight32(unsigned int w)
   9  * %rdi: w
  10  */
  11 ENTRY(__sw_hweight32)
  12
  13 #ifdef CONFIG_X86_64
  14         movl %edi, %eax                         # w
  15 #endif
  16         __ASM_SIZE(push,) %__ASM_REG(dx)
  17         movl %eax, %edx                         # w -> t
  18         shrl %edx                               # t >>= 1
  19         andl $0x55555555, %edx                  # t &= 0x55555555
  20         subl %edx, %eax                         # w -= t
  21
  22         movl %eax, %edx                         # w -> t
  23         shrl $2, %eax                           # w_tmp >>= 2
  24         andl $0x33333333, %edx                  # t     &= 0x33333333
  25         andl $0x33333333, %eax                  # w_tmp &= 0x33333333
  26         addl %edx, %eax                         # w = w_tmp + t
  27
  28         movl %eax, %edx                         # w -> t
  29         shrl $4, %edx                           # t >>= 4
  30         addl %edx, %eax                         # w_tmp += t
  31         andl  $0x0f0f0f0f, %eax                 # w_tmp &= 0x0f0f0f0f
  32         imull $0x01010101, %eax, %eax           # w_tmp *= 0x01010101
  33         shrl $24, %eax                          # w = w_tmp >> 24
  34         __ASM_SIZE(pop,) %__ASM_REG(dx)
  35         ret
  36 ENDPROC(__sw_hweight32)
  37 EXPORT_SYMBOL(__sw_hweight32)
  38
  39 ENTRY(__sw_hweight64)
  40 #ifdef CONFIG_X86_64
  41         pushq   %rdi
  42         pushq   %rdx
  43
  44         movq    %rdi, %rdx                      # w -> t
  45         movabsq $0x5555555555555555, %rax
  46         shrq    %rdx                            # t >>= 1
  47         andq    %rdx, %rax                      # t &= 0x5555555555555555
  48         movabsq $0x3333333333333333, %rdx
  49         subq    %rax, %rdi                      # w -= t
  50
  51         movq    %rdi, %rax                      # w -> t
  52         shrq    $2, %rdi                        # w_tmp >>= 2
  53         andq    %rdx, %rax                      # t     &= 0x3333333333333333
  54         andq    %rdi, %rdx                      # w_tmp &= 0x3333333333333333
  55         addq    %rdx, %rax                      # w = w_tmp + t
  56
  57         movq    %rax, %rdx                      # w -> t
  58         shrq    $4, %rdx                        # t >>= 4
  59         addq    %rdx, %rax                      # w_tmp += t
  60         movabsq $0x0f0f0f0f0f0f0f0f, %rdx
  61         andq    %rdx, %rax                      # w_tmp &= 0x0f0f0f0f0f0f0f0f
  62         movabsq $0x0101010101010101, %rdx
  63         imulq   %rdx, %rax                      # w_tmp *= 0x0101010101010101
  64         shrq    $56, %rax                       # w = w_tmp >> 56
  65
  66         popq    %rdx
  67         popq    %rdi
  68         ret
  69 #else /* CONFIG_X86_32 */
  70         /* We're getting an u64 arg in (%eax,%edx): unsigned long hweight64(__u64 w) */
  71         pushl   %ecx
  72
  73         call    __sw_hweight32
  74         movl    %eax, %ecx                      # stash away result
  75         movl    %edx, %eax                      # second part of input
  76         call    __sw_hweight32
  77         addl    %ecx, %eax                      # result
  78
  79         popl    %ecx
  80         ret
  81 #endif
  82 ENDPROC(__sw_hweight64)
  83 EXPORT_SYMBOL(__sw_hweight64)