]>
Commit | Line | Data |
---|---|---|
ab9f4b0b GN |
1 | /* |
2 | * CDDL HEADER START | |
3 | * | |
4 | * The contents of this file are subject to the terms of the | |
5 | * Common Development and Distribution License (the "License"). | |
6 | * You may not use this file except in compliance with the License. | |
7 | * | |
8 | * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE | |
9 | * or http://www.opensolaris.org/os/licensing. | |
10 | * See the License for the specific language governing permissions | |
11 | * and limitations under the License. | |
12 | * | |
13 | * When distributing Covered Code, include this CDDL HEADER in each | |
14 | * file and include the License file at usr/src/OPENSOLARIS.LICENSE. | |
15 | * If applicable, add the following below this CDDL HEADER, with the | |
16 | * fields enclosed by brackets "[]" replaced with your own identifying | |
17 | * information: Portions Copyright [yyyy] [name of copyright owner] | |
18 | * | |
19 | * CDDL HEADER END | |
20 | */ | |
21 | /* | |
22 | * Copyright (C) 2016 Gvozden Nešković. All rights reserved. | |
23 | */ | |
24 | ||
25 | #ifndef _VDEV_RAIDZ_H | |
26 | #define _VDEV_RAIDZ_H | |
27 | ||
28 | #include <sys/types.h> | |
29 | #include <sys/debug.h> | |
30 | #include <sys/kstat.h> | |
31 | ||
32 | #ifdef __cplusplus | |
33 | extern "C" { | |
34 | #endif | |
35 | ||
36 | #define CODE_P (0U) | |
37 | #define CODE_Q (1U) | |
38 | #define CODE_R (2U) | |
39 | ||
40 | #define PARITY_P (1U) | |
41 | #define PARITY_PQ (2U) | |
42 | #define PARITY_PQR (3U) | |
43 | ||
44 | #define TARGET_X (0U) | |
45 | #define TARGET_Y (1U) | |
46 | #define TARGET_Z (2U) | |
47 | ||
48 | /* | |
49 | * Parity generation methods indexes | |
50 | */ | |
51 | enum raidz_math_gen_op { | |
52 | RAIDZ_GEN_P = 0, | |
53 | RAIDZ_GEN_PQ, | |
54 | RAIDZ_GEN_PQR, | |
55 | RAIDZ_GEN_NUM = 3 | |
56 | }; | |
57 | /* | |
58 | * Data reconstruction methods indexes | |
59 | */ | |
60 | enum raidz_rec_op { | |
61 | RAIDZ_REC_P = 0, | |
62 | RAIDZ_REC_Q, | |
63 | RAIDZ_REC_R, | |
64 | RAIDZ_REC_PQ, | |
65 | RAIDZ_REC_PR, | |
66 | RAIDZ_REC_QR, | |
67 | RAIDZ_REC_PQR, | |
68 | RAIDZ_REC_NUM = 7 | |
69 | }; | |
70 | ||
71 | extern const char *raidz_gen_name[RAIDZ_GEN_NUM]; | |
72 | extern const char *raidz_rec_name[RAIDZ_REC_NUM]; | |
73 | ||
74 | /* | |
75 | * Methods used to define raidz implementation | |
76 | * | |
77 | * @raidz_gen_f Parity generation function | |
78 | * @par1 pointer to raidz_map | |
79 | * @raidz_rec_f Data reconstruction function | |
80 | * @par1 pointer to raidz_map | |
81 | * @par2 array of reconstruction targets | |
82 | * @will_work_f Function returns TRUE if impl. is supported on the system | |
83 | * @init_impl_f Function is called once on init | |
84 | * @fini_impl_f Function is called once on fini | |
85 | */ | |
86 | typedef void (*raidz_gen_f)(void *); | |
87 | typedef int (*raidz_rec_f)(void *, const int *); | |
88 | typedef boolean_t (*will_work_f)(void); | |
89 | typedef void (*init_impl_f)(void); | |
90 | typedef void (*fini_impl_f)(void); | |
91 | ||
c9187d86 GN |
92 | #define RAIDZ_IMPL_NAME_MAX (16) |
93 | ||
ab9f4b0b GN |
94 | typedef struct raidz_impl_ops { |
95 | init_impl_f init; | |
96 | fini_impl_f fini; | |
97 | raidz_gen_f gen[RAIDZ_GEN_NUM]; /* Parity generate functions */ | |
98 | raidz_rec_f rec[RAIDZ_REC_NUM]; /* Data reconstruction functions */ | |
99 | will_work_f is_supported; /* Support check function */ | |
c9187d86 | 100 | char name[RAIDZ_IMPL_NAME_MAX]; /* Name of the implementation */ |
ab9f4b0b GN |
101 | } raidz_impl_ops_t; |
102 | ||
103 | typedef struct raidz_col { | |
104 | size_t rc_devidx; /* child device index for I/O */ | |
105 | size_t rc_offset; /* device offset */ | |
106 | size_t rc_size; /* I/O size */ | |
107 | void *rc_data; /* I/O data */ | |
108 | void *rc_gdata; /* used to store the "good" version */ | |
109 | int rc_error; /* I/O error for this device */ | |
110 | unsigned int rc_tried; /* Did we attempt this I/O column? */ | |
111 | unsigned int rc_skipped; /* Did we skip this I/O column? */ | |
112 | } raidz_col_t; | |
113 | ||
114 | typedef struct raidz_map { | |
115 | size_t rm_cols; /* Regular column count */ | |
116 | size_t rm_scols; /* Count including skipped columns */ | |
117 | size_t rm_bigcols; /* Number of oversized columns */ | |
118 | size_t rm_asize; /* Actual total I/O size */ | |
119 | size_t rm_missingdata; /* Count of missing data devices */ | |
120 | size_t rm_missingparity; /* Count of missing parity devices */ | |
121 | size_t rm_firstdatacol; /* First data column/parity count */ | |
122 | size_t rm_nskip; /* Skipped sectors for padding */ | |
123 | size_t rm_skipstart; /* Column index of padding start */ | |
124 | void *rm_datacopy; /* rm_asize-buffer of copied data */ | |
125 | size_t rm_reports; /* # of referencing checksum reports */ | |
126 | unsigned int rm_freed; /* map no longer has referencing ZIO */ | |
127 | unsigned int rm_ecksuminjected; /* checksum error was injected */ | |
128 | raidz_impl_ops_t *rm_ops; /* RAIDZ math operations */ | |
129 | raidz_col_t rm_col[1]; /* Flexible array of I/O columns */ | |
130 | } raidz_map_t; | |
131 | ||
c9187d86 GN |
132 | #define RAIDZ_ORIGINAL_IMPL (INT_MAX) |
133 | ||
ae25d222 GN |
134 | extern const raidz_impl_ops_t vdev_raidz_scalar_impl; |
135 | #if defined(__x86_64) && defined(HAVE_SSE2) /* only x86_64 for now */ | |
136 | extern const raidz_impl_ops_t vdev_raidz_sse2_impl; | |
137 | #endif | |
138 | #if defined(__x86_64) && defined(HAVE_SSSE3) /* only x86_64 for now */ | |
139 | extern const raidz_impl_ops_t vdev_raidz_ssse3_impl; | |
140 | #endif | |
141 | #if defined(__x86_64) && defined(HAVE_AVX2) /* only x86_64 for now */ | |
142 | extern const raidz_impl_ops_t vdev_raidz_avx2_impl; | |
143 | #endif | |
144 | ||
ab9f4b0b GN |
145 | /* |
146 | * Commonly used raidz_map helpers | |
147 | * | |
148 | * raidz_parity Returns parity of the RAIDZ block | |
149 | * raidz_ncols Returns number of columns the block spans | |
150 | * raidz_nbigcols Returns number of big columns columns | |
151 | * raidz_col_p Returns pointer to a column | |
152 | * raidz_col_size Returns size of a column | |
153 | * raidz_big_size Returns size of big columns | |
154 | * raidz_short_size Returns size of short columns | |
155 | */ | |
156 | #define raidz_parity(rm) ((rm)->rm_firstdatacol) | |
157 | #define raidz_ncols(rm) ((rm)->rm_cols) | |
158 | #define raidz_nbigcols(rm) ((rm)->rm_bigcols) | |
159 | #define raidz_col_p(rm, c) ((rm)->rm_col + (c)) | |
160 | #define raidz_col_size(rm, c) ((rm)->rm_col[c].rc_size) | |
161 | #define raidz_big_size(rm) (raidz_col_size(rm, CODE_P)) | |
162 | #define raidz_short_size(rm) (raidz_col_size(rm, raidz_ncols(rm)-1)) | |
163 | ||
164 | /* | |
165 | * Macro defines an RAIDZ parity generation method | |
166 | * | |
167 | * @code parity the function produce | |
168 | * @impl name of the implementation | |
169 | */ | |
170 | #define _RAIDZ_GEN_WRAP(code, impl) \ | |
171 | static void \ | |
172 | impl ## _gen_ ## code(void *rmp) \ | |
173 | { \ | |
174 | raidz_map_t *rm = (raidz_map_t *) rmp; \ | |
175 | raidz_generate_## code ## _impl(rm); \ | |
176 | } | |
177 | ||
178 | /* | |
179 | * Macro defines an RAIDZ data reconstruction method | |
180 | * | |
181 | * @code parity the function produce | |
182 | * @impl name of the implementation | |
183 | */ | |
184 | #define _RAIDZ_REC_WRAP(code, impl) \ | |
185 | static int \ | |
186 | impl ## _rec_ ## code(void *rmp, const int *tgtidx) \ | |
187 | { \ | |
188 | raidz_map_t *rm = (raidz_map_t *) rmp; \ | |
189 | return (raidz_reconstruct_## code ## _impl(rm, tgtidx)); \ | |
190 | } | |
191 | ||
192 | /* | |
193 | * Define all gen methods for an implementation | |
194 | * | |
195 | * @impl name of the implementation | |
196 | */ | |
197 | #define DEFINE_GEN_METHODS(impl) \ | |
198 | _RAIDZ_GEN_WRAP(p, impl); \ | |
199 | _RAIDZ_GEN_WRAP(pq, impl); \ | |
200 | _RAIDZ_GEN_WRAP(pqr, impl) | |
201 | ||
202 | /* | |
203 | * Define all rec functions for an implementation | |
204 | * | |
205 | * @impl name of the implementation | |
206 | */ | |
207 | #define DEFINE_REC_METHODS(impl) \ | |
208 | _RAIDZ_REC_WRAP(p, impl); \ | |
209 | _RAIDZ_REC_WRAP(q, impl); \ | |
210 | _RAIDZ_REC_WRAP(r, impl); \ | |
211 | _RAIDZ_REC_WRAP(pq, impl); \ | |
212 | _RAIDZ_REC_WRAP(pr, impl); \ | |
213 | _RAIDZ_REC_WRAP(qr, impl); \ | |
214 | _RAIDZ_REC_WRAP(pqr, impl) | |
215 | ||
216 | #define RAIDZ_GEN_METHODS(impl) \ | |
217 | { \ | |
218 | [RAIDZ_GEN_P] = & impl ## _gen_p, \ | |
219 | [RAIDZ_GEN_PQ] = & impl ## _gen_pq, \ | |
220 | [RAIDZ_GEN_PQR] = & impl ## _gen_pqr \ | |
221 | } | |
222 | ||
223 | #define RAIDZ_REC_METHODS(impl) \ | |
224 | { \ | |
225 | [RAIDZ_REC_P] = & impl ## _rec_p, \ | |
226 | [RAIDZ_REC_Q] = & impl ## _rec_q, \ | |
227 | [RAIDZ_REC_R] = & impl ## _rec_r, \ | |
228 | [RAIDZ_REC_PQ] = & impl ## _rec_pq, \ | |
229 | [RAIDZ_REC_PR] = & impl ## _rec_pr, \ | |
230 | [RAIDZ_REC_QR] = & impl ## _rec_qr, \ | |
231 | [RAIDZ_REC_PQR] = & impl ## _rec_pqr \ | |
232 | } | |
233 | ||
234 | ||
235 | typedef struct raidz_impl_kstat { | |
26a08b5c GN |
236 | uint64_t gen[RAIDZ_GEN_NUM]; /* gen method speed B/s */ |
237 | uint64_t rec[RAIDZ_REC_NUM]; /* rec method speed B/s */ | |
ab9f4b0b GN |
238 | } raidz_impl_kstat_t; |
239 | ||
240 | /* | |
241 | * Enumerate various multiplication constants | |
242 | * used in reconstruction methods | |
243 | */ | |
244 | typedef enum raidz_mul_info { | |
245 | /* Reconstruct Q */ | |
246 | MUL_Q_X = 0, | |
247 | /* Reconstruct R */ | |
248 | MUL_R_X = 0, | |
249 | /* Reconstruct PQ */ | |
250 | MUL_PQ_X = 0, | |
251 | MUL_PQ_Y = 1, | |
252 | /* Reconstruct PR */ | |
253 | MUL_PR_X = 0, | |
254 | MUL_PR_Y = 1, | |
255 | /* Reconstruct QR */ | |
256 | MUL_QR_XQ = 0, | |
257 | MUL_QR_X = 1, | |
258 | MUL_QR_YQ = 2, | |
259 | MUL_QR_Y = 3, | |
260 | /* Reconstruct PQR */ | |
261 | MUL_PQR_XP = 0, | |
262 | MUL_PQR_XQ = 1, | |
263 | MUL_PQR_XR = 2, | |
264 | MUL_PQR_YU = 3, | |
265 | MUL_PQR_YP = 4, | |
266 | MUL_PQR_YQ = 5, | |
267 | ||
268 | MUL_CNT = 6 | |
269 | } raidz_mul_info_t; | |
270 | ||
271 | /* | |
272 | * Powers of 2 in the Galois field. | |
273 | */ | |
274 | extern const uint8_t vdev_raidz_pow2[256] __attribute__((aligned(256))); | |
275 | /* Logs of 2 in the Galois field defined above. */ | |
276 | extern const uint8_t vdev_raidz_log2[256] __attribute__((aligned(256))); | |
277 | ||
278 | /* | |
279 | * Multiply a given number by 2 raised to the given power. | |
280 | */ | |
281 | static inline uint8_t | |
282 | vdev_raidz_exp2(const uint8_t a, const unsigned exp) | |
283 | { | |
284 | if (a == 0) | |
285 | return (0); | |
286 | ||
287 | return (vdev_raidz_pow2[(exp + (unsigned) vdev_raidz_log2[a]) % 255]); | |
288 | } | |
289 | ||
290 | /* | |
291 | * Galois Field operations. | |
292 | * | |
293 | * gf_exp2 - computes 2 raised to the given power | |
294 | * gf_exp2 - computes 4 raised to the given power | |
295 | * gf_mul - multiplication | |
296 | * gf_div - division | |
297 | * gf_inv - multiplicative inverse | |
298 | */ | |
299 | typedef unsigned gf_t; | |
300 | typedef unsigned gf_log_t; | |
301 | ||
302 | static inline gf_t | |
303 | gf_mul(const gf_t a, const gf_t b) | |
304 | { | |
305 | gf_log_t logsum; | |
306 | ||
307 | if (a == 0 || b == 0) | |
308 | return (0); | |
309 | ||
310 | logsum = (gf_log_t) vdev_raidz_log2[a] + (gf_log_t) vdev_raidz_log2[b]; | |
311 | ||
312 | return ((gf_t) vdev_raidz_pow2[logsum % 255]); | |
313 | } | |
314 | ||
315 | static inline gf_t | |
316 | gf_div(const gf_t a, const gf_t b) | |
317 | { | |
318 | gf_log_t logsum; | |
319 | ||
320 | ASSERT3U(b, >, 0); | |
321 | if (a == 0) | |
322 | return (0); | |
323 | ||
324 | logsum = (gf_log_t) 255 + (gf_log_t) vdev_raidz_log2[a] - | |
325 | (gf_log_t) vdev_raidz_log2[b]; | |
326 | ||
327 | return ((gf_t) vdev_raidz_pow2[logsum % 255]); | |
328 | } | |
329 | ||
330 | static inline gf_t | |
331 | gf_inv(const gf_t a) | |
332 | { | |
333 | gf_log_t logsum; | |
334 | ||
335 | ASSERT3U(a, >, 0); | |
336 | ||
337 | logsum = (gf_log_t) 255 - (gf_log_t) vdev_raidz_log2[a]; | |
338 | ||
339 | return ((gf_t) vdev_raidz_pow2[logsum]); | |
340 | } | |
341 | ||
342 | static inline gf_t | |
343 | gf_exp2(gf_log_t exp) | |
344 | { | |
345 | return (vdev_raidz_pow2[exp % 255]); | |
346 | } | |
347 | ||
348 | static inline gf_t | |
349 | gf_exp4(gf_log_t exp) | |
350 | { | |
351 | ASSERT3U(exp, <=, 255); | |
352 | return ((gf_t) vdev_raidz_pow2[(2 * exp) % 255]); | |
353 | } | |
354 | ||
355 | #ifdef __cplusplus | |
356 | } | |
357 | #endif | |
358 | ||
359 | #endif /* _VDEV_RAIDZ_H */ |