ceph/src/boost/libs/log/src/dump_avx2.cpp

   1 /*
   2  *          Copyright Andrey Semashev 2007 - 2015.
   3  * Distributed under the Boost Software License, Version 1.0.
   4  *    (See accompanying file LICENSE_1_0.txt or copy at
   5  *          http://www.boost.org/LICENSE_1_0.txt)
   6  */
   7 /*!
   8  * \file   dump_avx2.cpp
   9  * \author Andrey Semashev
  10  * \date   05.05.2013
  11  *
  12  * \brief  This header is the Boost.Log library implementation, see the library documentation
  13  *         at http://www.boost.org/doc/libs/release/libs/log/doc/html/index.html.
  14  */
  15
  16 // NOTE: You should generally avoid including headers as much as possible here, because this file
  17 //       is compiled with special compiler options, and any included header may result in generation of
  18 //       unintended code with these options and violation of ODR.
  19 #include <boost/log/detail/config.hpp>
  20 #include <ostream>
  21 #include <immintrin.h>
  22 #include <boost/cstdint.hpp>
  23 #include <boost/log/detail/header.hpp>
  24
  25 #if defined(__x86_64) || defined(__x86_64__) || \
  26     defined(__amd64__) || defined(__amd64) || \
  27     defined(_M_X64)
  28 #define BOOST_LOG_AUX_X86_64
  29 #endif
  30
  31 namespace boost {
  32
  33 BOOST_LOG_OPEN_NAMESPACE
  34
  35 namespace aux {
  36
  37 extern const char g_hex_char_table[2][16];
  38
  39 template< typename CharT >
  40 extern void dump_data_generic(const void* data, std::size_t size, std::basic_ostream< CharT >& strm);
  41
  42 BOOST_LOG_ANONYMOUS_NAMESPACE {
  43
  44 enum
  45 {
  46     packs_per_stride = 32,
  47     stride = packs_per_stride * 32
  48 };
  49
  50 union ymm_constant
  51 {
  52     uint8_t as_bytes[32];
  53     __m256i as_mm;
  54
  55     BOOST_FORCEINLINE operator __m256i () const { return as_mm; }
  56 };
  57
  58 static const ymm_constant mm_shuffle_pattern1 = {{ 0x80, 0, 1, 0x80, 2, 3, 0x80, 4, 5, 0x80, 6, 7, 0x80, 8, 9, 0x80,       0x80, 0, 1, 0x80, 2, 3, 0x80, 4, 5, 0x80, 6, 7, 0x80, 8, 9, 0x80 }};
  59 static const ymm_constant mm_shuffle_pattern2 = {{ 0, 1, 0x80, 2, 3, 0x80, 4, 5, 0x80, 6, 7, 0x80, 8, 9, 0x80, 10,         0, 1, 0x80, 2, 3, 0x80, 4, 5, 0x80, 6, 7, 0x80, 8, 9, 0x80, 10 }};
  60 static const ymm_constant mm_shuffle_pattern3 = {{ 5, 0x80, 6, 7, 0x80, 8, 9, 0x80, 10, 11, 0x80, 12, 13, 0x80, 14, 15,    5, 0x80, 6, 7, 0x80, 8, 9, 0x80, 10, 11, 0x80, 12, 13, 0x80, 14, 15 }};
  61 static const ymm_constant mm_shuffle_pattern13 = {{ 0x80, 0, 1, 0x80, 2, 3, 0x80, 4, 5, 0x80, 6, 7, 0x80, 8, 9, 0x80,      5, 0x80, 6, 7, 0x80, 8, 9, 0x80, 10, 11, 0x80, 12, 13, 0x80, 14, 15 }};
  62
  63 #if defined(BOOST_LOG_AUX_X86_64)
  64
  65 // x86-64 architecture has more registers which we can utilize to pass constants
  66 #define BOOST_LOG_AUX_MM_CONSTANT_ARGS_DECL __m256i mm_15, __m256i mm_9, __m256i mm_char_0, __m256i mm_char_space,
  67 #define BOOST_LOG_AUX_MM_CONSTANT_ARGS mm_15, mm_9, mm_char_0, mm_char_space,
  68 #define BOOST_LOG_AUX_MM_CONSTANTS \
  69     const __m256i mm_15 = _mm256_set1_epi32(0x0F0F0F0F);\
  70     const __m256i mm_9 = _mm256_set1_epi32(0x09090909);\
  71     const __m256i mm_char_0 = _mm256_set1_epi32(0x30303030);\
  72     const __m256i mm_char_space = _mm256_set1_epi32(0x20202020);
  73
  74 #else
  75
  76 // MSVC in 32-bit mode is not able to pass all constants to dump_pack, and is also not able to align them on the stack, so we have to fetch them from global constants
  77 static const ymm_constant mm_15 = {{ 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F,   0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F }};
  78 static const ymm_constant mm_9 = {{ 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09,   0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09 }};
  79 static const ymm_constant mm_char_0 = {{ 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,   0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 }};
  80 static const ymm_constant mm_char_space = {{ 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,   0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20 }};
  81 #define BOOST_LOG_AUX_MM_CONSTANT_ARGS_DECL
  82 #define BOOST_LOG_AUX_MM_CONSTANT_ARGS
  83 #define BOOST_LOG_AUX_MM_CONSTANTS
  84
  85 #endif
  86
  87 /*!
  88  * \brief Dumps a pack of input data into a string of 8 bit ASCII characters.
  89  *
  90  * The composed string is placed as follows (in Intel notation): mm_output1[127:0], mm_output2[127:0], mm_output3[127:0], mm_output1[255:128], mm_output2[255:128], mm_output3[255:128].
  91  */
  92 static BOOST_FORCEINLINE void dump_pack
  93 (
  94     BOOST_LOG_AUX_MM_CONSTANT_ARGS_DECL
  95     __m256i mm_char_10_to_a, __m256i mm_input,
  96     __m256i& mm_output1, __m256i& mm_output2, __m256i& mm_output3
  97 )
  98 {
  99     // Split half-bytes
 100     __m256i mm_input_hi = _mm256_and_si256(_mm256_srli_epi16(mm_input, 4), mm_15);
 101     __m256i mm_input_lo = _mm256_and_si256(mm_input, mm_15);
 102
 103     // Stringize each of the halves
 104     __m256i mm_addend_hi = _mm256_cmpgt_epi8(mm_input_hi, mm_9);
 105     __m256i mm_addend_lo = _mm256_cmpgt_epi8(mm_input_lo, mm_9);
 106     mm_addend_hi = _mm256_and_si256(mm_char_10_to_a, mm_addend_hi);
 107     mm_addend_lo = _mm256_and_si256(mm_char_10_to_a, mm_addend_lo);
 108
 109     mm_input_hi = _mm256_add_epi8(mm_input_hi, mm_char_0);
 110     mm_input_lo = _mm256_add_epi8(mm_input_lo, mm_char_0);
 111
 112     mm_input_hi = _mm256_add_epi8(mm_input_hi, mm_addend_hi);
 113     mm_input_lo = _mm256_add_epi8(mm_input_lo, mm_addend_lo);
 114
 115     // Join them back together
 116     __m256i mm_1 = _mm256_unpacklo_epi8(mm_input_hi, mm_input_lo);
 117     __m256i mm_2 = _mm256_unpackhi_epi8(mm_input_hi, mm_input_lo);
 118
 119     // Insert spaces between stringized bytes:
 120     // |0123456789abcdef|0123456789abcdef|
 121     // | 01 23 45 67 89 |ab cd ef 01 23 4|5 67 89 ab cd ef|
 122     __m256i mm_out1 = _mm256_shuffle_epi8(mm_1, mm_shuffle_pattern1.as_mm);
 123     __m256i mm_out3 = _mm256_shuffle_epi8(mm_2, mm_shuffle_pattern3.as_mm);
 124     __m256i mm_out2 = _mm256_shuffle_epi8(_mm256_alignr_epi8(mm_2, mm_1, 10), mm_shuffle_pattern2.as_mm);
 125
 126     mm_output1 = _mm256_max_epu8(mm_out1, mm_char_space);
 127     mm_output2 = _mm256_max_epu8(mm_out2, mm_char_space);
 128     mm_output3 = _mm256_max_epu8(mm_out3, mm_char_space);
 129 }
 130
 131 //! Dumps a pack of input data into a string of 8 bit ASCII characters
 132 static BOOST_FORCEINLINE void dump_pack
 133 (
 134     BOOST_LOG_AUX_MM_CONSTANT_ARGS_DECL
 135     __m256i mm_char_10_to_a, __m128i mm_input,
 136     __m128i& mm_output1, __m128i& mm_output2, __m128i& mm_output3
 137 )
 138 {
 139     // Split half-bytes
 140     __m128i mm_input_hi = _mm_srli_epi16(mm_input, 4);
 141     __m256i mm = _mm256_inserti128_si256(_mm256_castsi128_si256(_mm_unpacklo_epi8(mm_input_hi, mm_input)), _mm_unpackhi_epi8(mm_input_hi, mm_input), 1);
 142     mm = _mm256_and_si256(mm, mm_15);
 143
 144     // Stringize the halves
 145     __m256i mm_addend = _mm256_cmpgt_epi8(mm, mm_9);
 146     mm_addend = _mm256_and_si256(mm_char_10_to_a, mm_addend);
 147
 148     mm = _mm256_add_epi8(mm, mm_char_0);
 149     mm = _mm256_add_epi8(mm, mm_addend);
 150
 151     // Insert spaces between stringized bytes:
 152     __m256i mm_out13 = _mm256_shuffle_epi8(mm, mm_shuffle_pattern13.as_mm);
 153     __m128i mm_out2 = _mm_shuffle_epi8(_mm_alignr_epi8(_mm256_extracti128_si256(mm, 1), _mm256_castsi256_si128(mm), 10), _mm256_castsi256_si128(mm_shuffle_pattern2.as_mm));
 154
 155     mm_out13 = _mm256_max_epu8(mm_out13, mm_char_space);
 156     mm_output2 = _mm_max_epu8(mm_out2, _mm256_castsi256_si128(mm_char_space));
 157     mm_output1 = _mm256_castsi256_si128(mm_out13);
 158     mm_output3 = _mm256_extracti128_si256(mm_out13, 1);
 159 }
 160
 161 template< typename CharT >
 162 BOOST_FORCEINLINE void store_characters(__m128i mm_chars, CharT* buf)
 163 {
 164     switch (sizeof(CharT))
 165     {
 166     case 1:
 167         _mm_store_si128(reinterpret_cast< __m128i* >(buf), mm_chars);
 168         break;
 169
 170     case 2:
 171         _mm256_store_si256(reinterpret_cast< __m256i* >(buf), _mm256_cvtepu8_epi16(mm_chars));
 172         break;
 173
 174     case 4:
 175         {
 176             __m128i mm = _mm_unpackhi_epi64(mm_chars, mm_chars);
 177             _mm256_store_si256(reinterpret_cast< __m256i* >(buf), _mm256_cvtepu8_epi32(mm_chars));
 178             _mm256_store_si256(reinterpret_cast< __m256i* >(buf) + 1, _mm256_cvtepu8_epi32(mm));
 179         }
 180         break;
 181     }
 182 }
 183
 184 template< typename CharT >
 185 BOOST_FORCEINLINE void store_characters_x3(__m256i mm_chars1, __m256i mm_chars2, __m256i mm_chars3, CharT* buf)
 186 {
 187     store_characters(_mm256_castsi256_si128(mm_chars1), buf);
 188     store_characters(_mm256_castsi256_si128(mm_chars2), buf + 16);
 189     store_characters(_mm256_castsi256_si128(mm_chars3), buf + 32);
 190     store_characters(_mm256_extracti128_si256(mm_chars1, 1), buf + 48);
 191     store_characters(_mm256_extracti128_si256(mm_chars2, 1), buf + 64);
 192     store_characters(_mm256_extracti128_si256(mm_chars3, 1), buf + 80);
 193 }
 194
 195 template< typename CharT >
 196 BOOST_FORCEINLINE void dump_data_avx2(const void* data, std::size_t size, std::basic_ostream< CharT >& strm)
 197 {
 198     typedef CharT char_type;
 199
 200     char_type buf_storage[stride * 3u + 32u];
 201     // Align the temporary buffer at 32 bytes
 202     char_type* const buf = reinterpret_cast< char_type* >((uint8_t*)buf_storage + (32u - (((uintptr_t)(char_type*)buf_storage) & 31u)));
 203     char_type* buf_begin = buf + 1u; // skip the first space of the first chunk
 204     char_type* buf_end = buf + stride * 3u;
 205
 206     __m256i mm_char_10_to_a;
 207     if (strm.flags() & std::ios_base::uppercase)
 208         mm_char_10_to_a = _mm256_set1_epi32(0x07070707); // '9' is 0x39 and 'A' is 0x41 in ASCII, so we have to add 0x07 to 0x3A to get uppercase letters
 209     else
 210         mm_char_10_to_a = _mm256_set1_epi32(0x27272727); // ...and 'a' is 0x61, which means we have to add 0x27 to 0x3A to get lowercase letters
 211
 212     // First, check the input alignment. Also, if we can dump the whole data in one go, do it right away. It turns out to be faster than splitting
 213     // the work between prealign and tail part. It is also a fairly common case since on most platforms memory is not aligned to 32 bytes (i.e. prealign is often needed).
 214     const uint8_t* p = static_cast< const uint8_t* >(data);
 215     const std::size_t prealign_size = size == 32u ? static_cast< std::size_t >(32u) : static_cast< std::size_t >((32u - ((uintptr_t)p & 31u)) & 31u);
 216     if (prealign_size)
 217     {
 218         __m256i mm_input = _mm256_lddqu_si256(reinterpret_cast< const __m256i* >(p));
 219         BOOST_LOG_AUX_MM_CONSTANTS
 220
 221         __m256i mm_output1, mm_output2, mm_output3;
 222         dump_pack(BOOST_LOG_AUX_MM_CONSTANT_ARGS mm_char_10_to_a, mm_input, mm_output1, mm_output2, mm_output3);
 223
 224         store_characters_x3(mm_output1, mm_output2, mm_output3, buf);
 225
 226         _mm256_zeroall(); // need to zero all ymm registers to avoid register spills/restores the compler generates around the function call
 227         strm.write(buf_begin, prealign_size * 3u - 1u);
 228
 229         buf_begin = buf;
 230         size -= prealign_size;
 231         p += prealign_size;
 232     }
 233
 234     const std::size_t stride_count = size / stride;
 235     std::size_t tail_size = size % stride;
 236     for (std::size_t i = 0; i < stride_count; ++i)
 237     {
 238         char_type* b = buf;
 239         BOOST_LOG_AUX_MM_CONSTANTS
 240
 241         for (unsigned int j = 0; j < packs_per_stride; ++j, b += 3u * 32u, p += 32u)
 242         {
 243             __m256i mm_input = _mm256_load_si256(reinterpret_cast< const __m256i* >(p));
 244             __m256i mm_output1, mm_output2, mm_output3;
 245             dump_pack(BOOST_LOG_AUX_MM_CONSTANT_ARGS mm_char_10_to_a, mm_input, mm_output1, mm_output2, mm_output3);
 246
 247             store_characters_x3(mm_output1, mm_output2, mm_output3, b);
 248         }
 249
 250         _mm256_zeroall(); // need to zero all ymm registers to avoid register spills/restores the compler generates around the function call
 251         strm.write(buf_begin, buf_end - buf_begin);
 252         buf_begin = buf;
 253     }
 254
 255     if (BOOST_UNLIKELY(tail_size > 0))
 256     {
 257         char_type* b = buf;
 258         while (tail_size >= 16u)
 259         {
 260             __m128i mm_input = _mm_load_si128(reinterpret_cast< const __m128i* >(p));
 261             BOOST_LOG_AUX_MM_CONSTANTS
 262
 263             __m128i mm_output1, mm_output2, mm_output3;
 264             dump_pack(BOOST_LOG_AUX_MM_CONSTANT_ARGS mm_char_10_to_a, mm_input, mm_output1, mm_output2, mm_output3);
 265
 266             store_characters(mm_output1, b);
 267             store_characters(mm_output2, b + 16u);
 268             store_characters(mm_output3, b + 32u);
 269
 270             b += 3u * 16u;
 271             p += 16u;
 272             tail_size -= 16u;
 273         }
 274
 275         _mm256_zeroall(); // need to zero all ymm registers to avoid register spills/restores the compler generates around the function call
 276         const char* const char_table = g_hex_char_table[(strm.flags() & std::ios_base::uppercase) != 0];
 277         for (unsigned int i = 0; i < tail_size; ++i, ++p, b += 3u)
 278         {
 279             uint32_t n = *p;
 280             b[0] = static_cast< char_type >(' ');
 281             b[1] = static_cast< char_type >(char_table[n >> 4]);
 282             b[2] = static_cast< char_type >(char_table[n & 0x0F]);
 283         }
 284
 285         strm.write(buf_begin, b - buf_begin);
 286     }
 287 }
 288
 289 } // namespace
 290
 291 void dump_data_char_avx2(const void* data, std::size_t size, std::basic_ostream< char >& strm)
 292 {
 293     if (size >= 32)
 294     {
 295         dump_data_avx2(data, size, strm);
 296     }
 297     else
 298     {
 299         dump_data_generic(data, size, strm);
 300     }
 301 }
 302
 303 void dump_data_wchar_avx2(const void* data, std::size_t size, std::basic_ostream< wchar_t >& strm)
 304 {
 305     if (size >= 32)
 306     {
 307         dump_data_avx2(data, size, strm);
 308     }
 309     else
 310     {
 311         dump_data_generic(data, size, strm);
 312     }
 313 }
 314
 315 #if !defined(BOOST_NO_CXX11_CHAR16_T)
 316 void dump_data_char16_avx2(const void* data, std::size_t size, std::basic_ostream< char16_t >& strm)
 317 {
 318     if (size >= 32)
 319     {
 320         dump_data_avx2(data, size, strm);
 321     }
 322     else
 323     {
 324         dump_data_generic(data, size, strm);
 325     }
 326 }
 327 #endif
 328
 329 #if !defined(BOOST_NO_CXX11_CHAR32_T)
 330 void dump_data_char32_avx2(const void* data, std::size_t size, std::basic_ostream< char32_t >& strm)
 331 {
 332     if (size >= 32)
 333     {
 334         dump_data_avx2(data, size, strm);
 335     }
 336     else
 337     {
 338         dump_data_generic(data, size, strm);
 339     }
 340 }
 341 #endif
 342
 343 } // namespace aux
 344
 345 BOOST_LOG_CLOSE_NAMESPACE // namespace log
 346
 347 } // namespace boost
 348
 349 #include <boost/log/detail/footer.hpp>