util/bufferiszero.c

   1 /*
   2  * Simple C functions to supplement the C library
   3  *
   4  * Copyright (c) 2006 Fabrice Bellard
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a copy
   7  * of this software and associated documentation files (the "Software"), to deal
   8  * in the Software without restriction, including without limitation the rights
   9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10  * copies of the Software, and to permit persons to whom the Software is
  11  * furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22  * THE SOFTWARE.
  23  */
  24 #include "qemu/osdep.h"
  25 #include "qemu-common.h"
  26 #include "qemu/cutils.h"
  27 #include "qemu/bswap.h"
  28
  29
  30 /* vector definitions */
  31
  32 extern void link_error(void);
  33
  34 #define ACCEL_BUFFER_ZERO(NAME, SIZE, VECTYPE, NONZERO)         \
  35 static bool NAME(const void *buf, size_t len)                   \
  36 {                                                               \
  37     const void *end = buf + len;                                \
  38     do {                                                        \
  39         const VECTYPE *p = buf;                                 \
  40         VECTYPE t;                                              \
  41         if (SIZE == sizeof(VECTYPE) * 4) {                      \
  42             t = (p[0] | p[1]) | (p[2] | p[3]);                  \
  43         } else if (SIZE == sizeof(VECTYPE) * 8) {               \
  44             t  = p[0] | p[1];                                   \
  45             t |= p[2] | p[3];                                   \
  46             t |= p[4] | p[5];                                   \
  47             t |= p[6] | p[7];                                   \
  48         } else {                                                \
  49             link_error();                                       \
  50         }                                                       \
  51         if (unlikely(NONZERO(t))) {                             \
  52             return false;                                       \
  53         }                                                       \
  54         buf += SIZE;                                            \
  55     } while (buf < end);                                        \
  56     return true;                                                \
  57 }
  58
  59 static bool
  60 buffer_zero_int(const void *buf, size_t len)
  61 {
  62     if (unlikely(len < 8)) {
  63         /* For a very small buffer, simply accumulate all the bytes.  */
  64         const unsigned char *p = buf;
  65         const unsigned char *e = buf + len;
  66         unsigned char t = 0;
  67
  68         do {
  69             t |= *p++;
  70         } while (p < e);
  71
  72         return t == 0;
  73     } else {
  74         /* Otherwise, use the unaligned memory access functions to
  75            handle the beginning and end of the buffer, with a couple
  76            of loops handling the middle aligned section.  */
  77         uint64_t t = ldq_he_p(buf);
  78         const uint64_t *p = (uint64_t *)(((uintptr_t)buf + 8) & -8);
  79         const uint64_t *e = (uint64_t *)(((uintptr_t)buf + len) & -8);
  80
  81         for (; p + 8 <= e; p += 8) {
  82             __builtin_prefetch(p + 8);
  83             if (t) {
  84                 return false;
  85             }
  86             t = p[0] | p[1] | p[2] | p[3] | p[4] | p[5] | p[6] | p[7];
  87         }
  88         while (p < e) {
  89             t |= *p++;
  90         }
  91         t |= ldq_he_p(buf + len - 8);
  92
  93         return t == 0;
  94     }
  95 }
  96
  97 #if defined(__ALTIVEC__)
  98 #include <altivec.h>
  99 /* The altivec.h header says we're allowed to undef these for
 100  * C++ compatibility.  Here we don't care about C++, but we
 101  * undef them anyway to avoid namespace pollution.
 102  * altivec.h may redefine the bool macro as vector type.
 103  * Reset it to POSIX semantics.
 104  */
 105 #undef vector
 106 #undef pixel
 107 #undef bool
 108 #define bool _Bool
 109 #define DO_NONZERO(X)  vec_any_ne(X, (__vector unsigned char){ 0 })
 110 ACCEL_BUFFER_ZERO(buffer_zero_ppc, 128, __vector unsigned char, DO_NONZERO)
 111
 112 static bool select_accel_fn(const void *buf, size_t len)
 113 {
 114     uintptr_t ibuf = (uintptr_t)buf;
 115     if (len % 128 == 0 && ibuf % sizeof(__vector unsigned char) == 0) {
 116         return buffer_zero_ppc(buf, len);
 117     }
 118     return buffer_zero_int(buf, len);
 119 }
 120
 121 #elif defined(CONFIG_AVX2_OPT) || (defined(CONFIG_CPUID_H) && defined(__SSE2__))
 122 #include <cpuid.h>
 123
 124 /* Do not use push_options pragmas unnecessarily, because clang
 125  * does not support them.
 126  */
 127 #ifndef __SSE2__
 128 #pragma GCC push_options
 129 #pragma GCC target("sse2")
 130 #endif
 131 #include <emmintrin.h>
 132 #define SSE2_NONZERO(X) \
 133     (_mm_movemask_epi8(_mm_cmpeq_epi8((X), _mm_setzero_si128())) != 0xFFFF)
 134 ACCEL_BUFFER_ZERO(buffer_zero_sse2, 64, __m128i, SSE2_NONZERO)
 135 #ifndef __SSE2__
 136 #pragma GCC pop_options
 137 #endif
 138
 139 #ifdef CONFIG_AVX2_OPT
 140 #pragma GCC push_options
 141 #pragma GCC target("avx2")
 142 #include <immintrin.h>
 143 #define AVX2_NONZERO(X)  !_mm256_testz_si256((X), (X))
 144 ACCEL_BUFFER_ZERO(buffer_zero_avx2, 128, __m256i, AVX2_NONZERO)
 145 #pragma GCC pop_options
 146 #endif
 147
 148 #define CACHE_AVX2    2
 149 #define CACHE_AVX1    4
 150 #define CACHE_SSE4    8
 151 #define CACHE_SSE2    16
 152
 153 static unsigned cpuid_cache;
 154
 155 static void __attribute__((constructor)) init_cpuid_cache(void)
 156 {
 157     int max = __get_cpuid_max(0, NULL);
 158     int a, b, c, d;
 159     unsigned cache = 0;
 160
 161     if (max >= 1) {
 162         __cpuid(1, a, b, c, d);
 163         if (d & bit_SSE2) {
 164             cache |= CACHE_SSE2;
 165         }
 166 #ifdef CONFIG_AVX2_OPT
 167         if (c & bit_SSE4_1) {
 168             cache |= CACHE_SSE4;
 169         }
 170
 171         /* We must check that AVX is not just available, but usable.  */
 172         if ((c & bit_OSXSAVE) && (c & bit_AVX)) {
 173             __asm("xgetbv" : "=a"(a), "=d"(d) : "c"(0));
 174             if ((a & 6) == 6) {
 175                 cache |= CACHE_AVX1;
 176                 if (max >= 7) {
 177                     __cpuid_count(7, 0, a, b, c, d);
 178                     if (b & bit_AVX2) {
 179                         cache |= CACHE_AVX2;
 180                     }
 181                 }
 182             }
 183         }
 184 #endif
 185     }
 186     cpuid_cache = cache;
 187 }
 188
 189 static bool select_accel_fn(const void *buf, size_t len)
 190 {
 191     uintptr_t ibuf = (uintptr_t)buf;
 192 #ifdef CONFIG_AVX2_OPT
 193     if (len % 128 == 0 && ibuf % 32 == 0 && (cpuid_cache & CACHE_AVX2)) {
 194         return buffer_zero_avx2(buf, len);
 195     }
 196 #endif
 197     if (len % 64 == 0 && ibuf % 16 == 0 && (cpuid_cache & CACHE_SSE2)) {
 198         return buffer_zero_sse2(buf, len);
 199     }
 200     return buffer_zero_int(buf, len);
 201 }
 202
 203 #elif defined(__aarch64__)
 204 #include "arm_neon.h"
 205
 206 #define DO_NONZERO(X)  (vgetq_lane_u64((X), 0) | vgetq_lane_u64((X), 1))
 207 ACCEL_BUFFER_ZERO(buffer_zero_neon, 128, uint64x2_t, DO_NONZERO)
 208
 209 static bool select_accel_fn(const void *buf, size_t len)
 210 {
 211     uintptr_t ibuf = (uintptr_t)buf;
 212     if (len % 128 == 0 && ibuf % sizeof(uint64x2_t) == 0) {
 213         return buffer_zero_neon(buf, len);
 214     }
 215     return buffer_zero_int(buf, len);
 216 }
 217
 218 #else
 219 #define select_accel_fn  buffer_zero_int
 220 #endif
 221
 222 /*
 223  * Checks if a buffer is all zeroes
 224  */
 225 bool buffer_is_zero(const void *buf, size_t len)
 226 {
 227     if (unlikely(len == 0)) {
 228         return true;
 229     }
 230
 231     /* Use an optimized zero check if possible.  Note that this also
 232        includes a check for an unrolled loop over 64-bit integers.  */
 233     return select_accel_fn(buf, len);
 234 }