lib/raid6/algos.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /* -*- linux-c -*- ------------------------------------------------------- *
   3  *
   4  *   Copyright 2002 H. Peter Anvin - All Rights Reserved
   5  *
   6  * ----------------------------------------------------------------------- */
   7
   8 /*
   9  * raid6/algos.c
  10  *
  11  * Algorithm list and algorithm selection for RAID-6
  12  */
  13
  14 #include <linux/raid/pq.h>
  15 #ifndef __KERNEL__
  16 #include <sys/mman.h>
  17 #include <stdio.h>
  18 #else
  19 #include <linux/module.h>
  20 #include <linux/gfp.h>
  21 #if !RAID6_USE_EMPTY_ZERO_PAGE
  22 /* In .bss so it's zeroed */
  23 const char raid6_empty_zero_page[PAGE_SIZE] __attribute__((aligned(256)));
  24 EXPORT_SYMBOL(raid6_empty_zero_page);
  25 #endif
  26 #endif
  27
  28 struct raid6_calls raid6_call;
  29 EXPORT_SYMBOL_GPL(raid6_call);
  30
  31 const struct raid6_calls * const raid6_algos[] = {
  32 #if defined(__i386__) && !defined(__arch_um__)
  33 #ifdef CONFIG_AS_AVX512
  34         &raid6_avx512x2,
  35         &raid6_avx512x1,
  36 #endif
  37 #ifdef CONFIG_AS_AVX2
  38         &raid6_avx2x2,
  39         &raid6_avx2x1,
  40 #endif
  41         &raid6_sse2x2,
  42         &raid6_sse2x1,
  43         &raid6_sse1x2,
  44         &raid6_sse1x1,
  45         &raid6_mmxx2,
  46         &raid6_mmxx1,
  47 #endif
  48 #if defined(__x86_64__) && !defined(__arch_um__)
  49 #ifdef CONFIG_AS_AVX512
  50         &raid6_avx512x4,
  51         &raid6_avx512x2,
  52         &raid6_avx512x1,
  53 #endif
  54 #ifdef CONFIG_AS_AVX2
  55         &raid6_avx2x4,
  56         &raid6_avx2x2,
  57         &raid6_avx2x1,
  58 #endif
  59         &raid6_sse2x4,
  60         &raid6_sse2x2,
  61         &raid6_sse2x1,
  62 #endif
  63 #ifdef CONFIG_ALTIVEC
  64         &raid6_vpermxor8,
  65         &raid6_vpermxor4,
  66         &raid6_vpermxor2,
  67         &raid6_vpermxor1,
  68         &raid6_altivec8,
  69         &raid6_altivec4,
  70         &raid6_altivec2,
  71         &raid6_altivec1,
  72 #endif
  73 #if defined(CONFIG_S390)
  74         &raid6_s390vx8,
  75 #endif
  76 #ifdef CONFIG_KERNEL_MODE_NEON
  77         &raid6_neonx8,
  78         &raid6_neonx4,
  79         &raid6_neonx2,
  80         &raid6_neonx1,
  81 #endif
  82 #if defined(__ia64__)
  83         &raid6_intx32,
  84         &raid6_intx16,
  85 #endif
  86         &raid6_intx8,
  87         &raid6_intx4,
  88         &raid6_intx2,
  89         &raid6_intx1,
  90         NULL
  91 };
  92
  93 void (*raid6_2data_recov)(int, size_t, int, int, void **);
  94 EXPORT_SYMBOL_GPL(raid6_2data_recov);
  95
  96 void (*raid6_datap_recov)(int, size_t, int, void **);
  97 EXPORT_SYMBOL_GPL(raid6_datap_recov);
  98
  99 const struct raid6_recov_calls *const raid6_recov_algos[] = {
 100 #ifdef CONFIG_AS_AVX512
 101         &raid6_recov_avx512,
 102 #endif
 103 #ifdef CONFIG_AS_AVX2
 104         &raid6_recov_avx2,
 105 #endif
 106 #ifdef CONFIG_AS_SSSE3
 107         &raid6_recov_ssse3,
 108 #endif
 109 #ifdef CONFIG_S390
 110         &raid6_recov_s390xc,
 111 #endif
 112 #if defined(CONFIG_KERNEL_MODE_NEON)
 113         &raid6_recov_neon,
 114 #endif
 115         &raid6_recov_intx1,
 116         NULL
 117 };
 118
 119 #ifdef __KERNEL__
 120 #define RAID6_TIME_JIFFIES_LG2  4
 121 #else
 122 /* Need more time to be stable in userspace */
 123 #define RAID6_TIME_JIFFIES_LG2  9
 124 #define time_before(x, y) ((x) < (y))
 125 #endif
 126
 127 static inline const struct raid6_recov_calls *raid6_choose_recov(void)
 128 {
 129         const struct raid6_recov_calls *const *algo;
 130         const struct raid6_recov_calls *best;
 131
 132         for (best = NULL, algo = raid6_recov_algos; *algo; algo++)
 133                 if (!best || (*algo)->priority > best->priority)
 134                         if (!(*algo)->valid || (*algo)->valid())
 135                                 best = *algo;
 136
 137         if (best) {
 138                 raid6_2data_recov = best->data2;
 139                 raid6_datap_recov = best->datap;
 140
 141                 pr_info("raid6: using %s recovery algorithm\n", best->name);
 142         } else
 143                 pr_err("raid6: Yikes! No recovery algorithm found!\n");
 144
 145         return best;
 146 }
 147
 148 static inline const struct raid6_calls *raid6_choose_gen(
 149         void *(*const dptrs)[(65536/PAGE_SIZE)+2], const int disks)
 150 {
 151         unsigned long perf, bestgenperf, bestxorperf, j0, j1;
 152         int start = (disks>>1)-1, stop = disks-3;       /* work on the second half of the disks */
 153         const struct raid6_calls *const *algo;
 154         const struct raid6_calls *best;
 155
 156         for (bestgenperf = 0, bestxorperf = 0, best = NULL, algo = raid6_algos; *algo; algo++) {
 157                 if (!best || (*algo)->prefer >= best->prefer) {
 158                         if ((*algo)->valid && !(*algo)->valid())
 159                                 continue;
 160
 161                         if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) {
 162                                 best = *algo;
 163                                 break;
 164                         }
 165
 166                         perf = 0;
 167
 168                         preempt_disable();
 169                         j0 = jiffies;
 170                         while ((j1 = jiffies) == j0)
 171                                 cpu_relax();
 172                         while (time_before(jiffies,
 173                                             j1 + (1<<RAID6_TIME_JIFFIES_LG2))) {
 174                                 (*algo)->gen_syndrome(disks, PAGE_SIZE, *dptrs);
 175                                 perf++;
 176                         }
 177                         preempt_enable();
 178
 179                         if (perf > bestgenperf) {
 180                                 bestgenperf = perf;
 181                                 best = *algo;
 182                         }
 183                         pr_info("raid6: %-8s gen() %5ld MB/s\n", (*algo)->name,
 184                                (perf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2));
 185
 186                         if (!(*algo)->xor_syndrome)
 187                                 continue;
 188
 189                         perf = 0;
 190
 191                         preempt_disable();
 192                         j0 = jiffies;
 193                         while ((j1 = jiffies) == j0)
 194                                 cpu_relax();
 195                         while (time_before(jiffies,
 196                                             j1 + (1<<RAID6_TIME_JIFFIES_LG2))) {
 197                                 (*algo)->xor_syndrome(disks, start, stop,
 198                                                       PAGE_SIZE, *dptrs);
 199                                 perf++;
 200                         }
 201                         preempt_enable();
 202
 203                         if (best == *algo)
 204                                 bestxorperf = perf;
 205
 206                         pr_info("raid6: %-8s xor() %5ld MB/s\n", (*algo)->name,
 207                                 (perf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2+1));
 208                 }
 209         }
 210
 211         if (best) {
 212                 pr_info("raid6: using algorithm %s gen() %ld MB/s\n",
 213                        best->name,
 214                        (bestgenperf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2));
 215                 if (best->xor_syndrome)
 216                         pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n",
 217                                (bestxorperf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2+1));
 218                 raid6_call = *best;
 219         } else
 220                 pr_err("raid6: Yikes!  No algorithm found!\n");
 221
 222         return best;
 223 }
 224
 225
 226 /* Try to pick the best algorithm */
 227 /* This code uses the gfmul table as convenient data set to abuse */
 228
 229 int __init raid6_select_algo(void)
 230 {
 231         const int disks = (65536/PAGE_SIZE)+2;
 232
 233         const struct raid6_calls *gen_best;
 234         const struct raid6_recov_calls *rec_best;
 235         char *syndromes;
 236         void *dptrs[(65536/PAGE_SIZE)+2];
 237         int i;
 238
 239         for (i = 0; i < disks-2; i++)
 240                 dptrs[i] = ((char *)raid6_gfmul) + PAGE_SIZE*i;
 241
 242         /* Normal code - use a 2-page allocation to avoid D$ conflict */
 243         syndromes = (void *) __get_free_pages(GFP_KERNEL, 1);
 244
 245         if (!syndromes) {
 246                 pr_err("raid6: Yikes!  No memory available.\n");
 247                 return -ENOMEM;
 248         }
 249
 250         dptrs[disks-2] = syndromes;
 251         dptrs[disks-1] = syndromes + PAGE_SIZE;
 252
 253         /* select raid gen_syndrome function */
 254         gen_best = raid6_choose_gen(&dptrs, disks);
 255
 256         /* select raid recover functions */
 257         rec_best = raid6_choose_recov();
 258
 259         free_pages((unsigned long)syndromes, 1);
 260
 261         return gen_best && rec_best ? 0 : -EINVAL;
 262 }
 263
 264 static void raid6_exit(void)
 265 {
 266         do { } while (0);
 267 }
 268
 269 subsys_initcall(raid6_select_algo);
 270 module_exit(raid6_exit);
 271 MODULE_LICENSE("GPL");
 272 MODULE_DESCRIPTION("RAID6 Q-syndrome calculations");