ceph/src/isa-l/erasure_code/ec_highlevel_func.c

   1 /**********************************************************************
   2   Copyright(c) 2011-2015 Intel Corporation All rights reserved.
   3
   4   Redistribution and use in source and binary forms, with or without
   5   modification, are permitted provided that the following conditions
   6   are met:
   7     * Redistributions of source code must retain the above copyright
   8       notice, this list of conditions and the following disclaimer.
   9     * Redistributions in binary form must reproduce the above copyright
  10       notice, this list of conditions and the following disclaimer in
  11       the documentation and/or other materials provided with the
  12       distribution.
  13     * Neither the name of Intel Corporation nor the names of its
  14       contributors may be used to endorse or promote products derived
  15       from this software without specific prior written permission.
  16
  17   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 **********************************************************************/
  29 #include <limits.h>
  30 #include "erasure_code.h"
  31 #include "types.h"
  32
  33 void ec_init_tables(int k, int rows, unsigned char *a, unsigned char *g_tbls)
  34 {
  35         int i, j;
  36
  37         for (i = 0; i < rows; i++) {
  38                 for (j = 0; j < k; j++) {
  39                         gf_vect_mul_init(*a++, g_tbls);
  40                         g_tbls += 32;
  41                 }
  42         }
  43 }
  44
  45 void ec_encode_data_sse(int len, int k, int rows, unsigned char *g_tbls, unsigned char **data,
  46                         unsigned char **coding)
  47 {
  48
  49         if (len < 16) {
  50                 ec_encode_data_base(len, k, rows, g_tbls, data, coding);
  51                 return;
  52         }
  53
  54         while (rows >= 4) {
  55                 gf_4vect_dot_prod_sse(len, k, g_tbls, data, coding);
  56                 g_tbls += 4 * k * 32;
  57                 coding += 4;
  58                 rows -= 4;
  59         }
  60         switch (rows) {
  61         case 3:
  62                 gf_3vect_dot_prod_sse(len, k, g_tbls, data, coding);
  63                 break;
  64         case 2:
  65                 gf_2vect_dot_prod_sse(len, k, g_tbls, data, coding);
  66                 break;
  67         case 1:
  68                 gf_vect_dot_prod_sse(len, k, g_tbls, data, *coding);
  69                 break;
  70         case 0:
  71                 break;
  72         }
  73
  74 }
  75
  76 void ec_encode_data_avx(int len, int k, int rows, unsigned char *g_tbls, unsigned char **data,
  77                         unsigned char **coding)
  78 {
  79         if (len < 16) {
  80                 ec_encode_data_base(len, k, rows, g_tbls, data, coding);
  81                 return;
  82         }
  83
  84         while (rows >= 4) {
  85                 gf_4vect_dot_prod_avx(len, k, g_tbls, data, coding);
  86                 g_tbls += 4 * k * 32;
  87                 coding += 4;
  88                 rows -= 4;
  89         }
  90         switch (rows) {
  91         case 3:
  92                 gf_3vect_dot_prod_avx(len, k, g_tbls, data, coding);
  93                 break;
  94         case 2:
  95                 gf_2vect_dot_prod_avx(len, k, g_tbls, data, coding);
  96                 break;
  97         case 1:
  98                 gf_vect_dot_prod_avx(len, k, g_tbls, data, *coding);
  99                 break;
 100         case 0:
 101                 break;
 102         }
 103
 104 }
 105
 106 void ec_encode_data_avx2(int len, int k, int rows, unsigned char *g_tbls, unsigned char **data,
 107                          unsigned char **coding)
 108 {
 109
 110         if (len < 32) {
 111                 ec_encode_data_base(len, k, rows, g_tbls, data, coding);
 112                 return;
 113         }
 114
 115         while (rows >= 4) {
 116                 gf_4vect_dot_prod_avx2(len, k, g_tbls, data, coding);
 117                 g_tbls += 4 * k * 32;
 118                 coding += 4;
 119                 rows -= 4;
 120         }
 121         switch (rows) {
 122         case 3:
 123                 gf_3vect_dot_prod_avx2(len, k, g_tbls, data, coding);
 124                 break;
 125         case 2:
 126                 gf_2vect_dot_prod_avx2(len, k, g_tbls, data, coding);
 127                 break;
 128         case 1:
 129                 gf_vect_dot_prod_avx2(len, k, g_tbls, data, *coding);
 130                 break;
 131         case 0:
 132                 break;
 133         }
 134
 135 }
 136
 137 #ifdef HAVE_AS_KNOWS_AVX512
 138
 139 extern int gf_vect_dot_prod_avx512(int len, int k, unsigned char *g_tbls, unsigned char **data,
 140                                    unsigned char *dest);
 141 extern int gf_2vect_dot_prod_avx512(int len, int k, unsigned char *g_tbls,
 142                                     unsigned char **data, unsigned char **coding);
 143 extern int gf_3vect_dot_prod_avx512(int len, int k, unsigned char *g_tbls,
 144                                     unsigned char **data, unsigned char **coding);
 145 extern int gf_4vect_dot_prod_avx512(int len, int k, unsigned char *g_tbls,
 146                                     unsigned char **data, unsigned char **coding);
 147 extern void gf_vect_mad_avx512(int len, int vec, int vec_i, unsigned char *gftbls,
 148                                unsigned char *src, unsigned char *dest);
 149 extern void gf_2vect_mad_avx512(int len, int vec, int vec_i, unsigned char *gftbls,
 150                                 unsigned char *src, unsigned char **dest);
 151 extern void gf_3vect_mad_avx512(int len, int vec, int vec_i, unsigned char *gftbls,
 152                                 unsigned char *src, unsigned char **dest);
 153 extern void gf_4vect_mad_avx512(int len, int vec, int vec_i, unsigned char *gftbls,
 154                                 unsigned char *src, unsigned char **dest);
 155
 156 void ec_encode_data_avx512(int len, int k, int rows, unsigned char *g_tbls,
 157                            unsigned char **data, unsigned char **coding)
 158 {
 159
 160         if (len < 64) {
 161                 ec_encode_data_base(len, k, rows, g_tbls, data, coding);
 162                 return;
 163         }
 164
 165         while (rows >= 4) {
 166                 gf_4vect_dot_prod_avx512(len, k, g_tbls, data, coding);
 167                 g_tbls += 4 * k * 32;
 168                 coding += 4;
 169                 rows -= 4;
 170         }
 171         switch (rows) {
 172         case 3:
 173                 gf_3vect_dot_prod_avx512(len, k, g_tbls, data, coding);
 174                 break;
 175         case 2:
 176                 gf_2vect_dot_prod_avx512(len, k, g_tbls, data, coding);
 177                 break;
 178         case 1:
 179                 gf_vect_dot_prod_avx512(len, k, g_tbls, data, *coding);
 180                 break;
 181         case 0:
 182                 break;
 183         }
 184 }
 185
 186 void ec_encode_data_update_avx512(int len, int k, int rows, int vec_i, unsigned char *g_tbls,
 187                                   unsigned char *data, unsigned char **coding)
 188 {
 189         if (len < 64) {
 190                 ec_encode_data_update_base(len, k, rows, vec_i, g_tbls, data, coding);
 191                 return;
 192         }
 193
 194         while (rows >= 4) {
 195                 gf_4vect_mad_avx512(len, k, vec_i, g_tbls, data, coding);
 196                 g_tbls += 4 * k * 32;
 197                 coding += 4;
 198                 rows -= 4;
 199         }
 200         switch (rows) {
 201         case 3:
 202                 gf_3vect_mad_avx512(len, k, vec_i, g_tbls, data, coding);
 203                 break;
 204         case 2:
 205                 gf_2vect_mad_avx512(len, k, vec_i, g_tbls, data, coding);
 206                 break;
 207         case 1:
 208                 gf_vect_mad_avx512(len, k, vec_i, g_tbls, data, *coding);
 209                 break;
 210         case 0:
 211                 break;
 212         }
 213 }
 214
 215 #endif // HAVE_AS_KNOWS_AVX512
 216
 217 #if __WORDSIZE == 64 || _WIN64 || __x86_64__
 218
 219 void ec_encode_data_update_sse(int len, int k, int rows, int vec_i, unsigned char *g_tbls,
 220                                unsigned char *data, unsigned char **coding)
 221 {
 222         if (len < 16) {
 223                 ec_encode_data_update_base(len, k, rows, vec_i, g_tbls, data, coding);
 224                 return;
 225         }
 226
 227         while (rows > 6) {
 228                 gf_6vect_mad_sse(len, k, vec_i, g_tbls, data, coding);
 229                 g_tbls += 6 * k * 32;
 230                 coding += 6;
 231                 rows -= 6;
 232         }
 233         switch (rows) {
 234         case 6:
 235                 gf_6vect_mad_sse(len, k, vec_i, g_tbls, data, coding);
 236                 break;
 237         case 5:
 238                 gf_5vect_mad_sse(len, k, vec_i, g_tbls, data, coding);
 239                 break;
 240         case 4:
 241                 gf_4vect_mad_sse(len, k, vec_i, g_tbls, data, coding);
 242                 break;
 243         case 3:
 244                 gf_3vect_mad_sse(len, k, vec_i, g_tbls, data, coding);
 245                 break;
 246         case 2:
 247                 gf_2vect_mad_sse(len, k, vec_i, g_tbls, data, coding);
 248                 break;
 249         case 1:
 250                 gf_vect_mad_sse(len, k, vec_i, g_tbls, data, *coding);
 251                 break;
 252         case 0:
 253                 break;
 254         }
 255
 256 }
 257
 258 void ec_encode_data_update_avx(int len, int k, int rows, int vec_i, unsigned char *g_tbls,
 259                                unsigned char *data, unsigned char **coding)
 260 {
 261         if (len < 16) {
 262                 ec_encode_data_update_base(len, k, rows, vec_i, g_tbls, data, coding);
 263                 return;
 264         }
 265         while (rows > 6) {
 266                 gf_6vect_mad_avx(len, k, vec_i, g_tbls, data, coding);
 267                 g_tbls += 6 * k * 32;
 268                 coding += 6;
 269                 rows -= 6;
 270         }
 271         switch (rows) {
 272         case 6:
 273                 gf_6vect_mad_avx(len, k, vec_i, g_tbls, data, coding);
 274                 break;
 275         case 5:
 276                 gf_5vect_mad_avx(len, k, vec_i, g_tbls, data, coding);
 277                 break;
 278         case 4:
 279                 gf_4vect_mad_avx(len, k, vec_i, g_tbls, data, coding);
 280                 break;
 281         case 3:
 282                 gf_3vect_mad_avx(len, k, vec_i, g_tbls, data, coding);
 283                 break;
 284         case 2:
 285                 gf_2vect_mad_avx(len, k, vec_i, g_tbls, data, coding);
 286                 break;
 287         case 1:
 288                 gf_vect_mad_avx(len, k, vec_i, g_tbls, data, *coding);
 289                 break;
 290         case 0:
 291                 break;
 292         }
 293
 294 }
 295
 296 void ec_encode_data_update_avx2(int len, int k, int rows, int vec_i, unsigned char *g_tbls,
 297                                 unsigned char *data, unsigned char **coding)
 298 {
 299         if (len < 32) {
 300                 ec_encode_data_update_base(len, k, rows, vec_i, g_tbls, data, coding);
 301                 return;
 302         }
 303         while (rows > 6) {
 304                 gf_6vect_mad_avx2(len, k, vec_i, g_tbls, data, coding);
 305                 g_tbls += 6 * k * 32;
 306                 coding += 6;
 307                 rows -= 6;
 308         }
 309         switch (rows) {
 310         case 6:
 311                 gf_6vect_mad_avx2(len, k, vec_i, g_tbls, data, coding);
 312                 break;
 313         case 5:
 314                 gf_5vect_mad_avx2(len, k, vec_i, g_tbls, data, coding);
 315                 break;
 316         case 4:
 317                 gf_4vect_mad_avx2(len, k, vec_i, g_tbls, data, coding);
 318                 break;
 319         case 3:
 320                 gf_3vect_mad_avx2(len, k, vec_i, g_tbls, data, coding);
 321                 break;
 322         case 2:
 323                 gf_2vect_mad_avx2(len, k, vec_i, g_tbls, data, coding);
 324                 break;
 325         case 1:
 326                 gf_vect_mad_avx2(len, k, vec_i, g_tbls, data, *coding);
 327                 break;
 328         case 0:
 329                 break;
 330         }
 331
 332 }
 333
 334 #endif //__WORDSIZE == 64 || _WIN64 || __x86_64__
 335
 336 struct slver {
 337         UINT16 snum;
 338         UINT8 ver;
 339         UINT8 core;
 340 };
 341
 342 // Version info
 343 struct slver ec_init_tables_slver_00010068;
 344 struct slver ec_init_tables_slver = { 0x0068, 0x01, 0x00 };
 345
 346 struct slver ec_encode_data_sse_slver_00020069;
 347 struct slver ec_encode_data_sse_slver = { 0x0069, 0x02, 0x00 };