]>
Commit | Line | Data |
---|---|---|
a739ff3f ST |
1 | /* |
2 | * Copyright (C) 2015 Google, Inc. | |
3 | * | |
4 | * Author: Sami Tolvanen <samitolvanen@google.com> | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify it | |
7 | * under the terms of the GNU General Public License as published by the Free | |
8 | * Software Foundation; either version 2 of the License, or (at your option) | |
9 | * any later version. | |
10 | */ | |
11 | ||
12 | #include "dm-verity-fec.h" | |
13 | #include <linux/math64.h> | |
14 | ||
15 | #define DM_MSG_PREFIX "verity-fec" | |
16 | ||
17 | /* | |
18 | * If error correction has been configured, returns true. | |
19 | */ | |
20 | bool verity_fec_is_enabled(struct dm_verity *v) | |
21 | { | |
22 | return v->fec && v->fec->dev; | |
23 | } | |
24 | ||
25 | /* | |
26 | * Return a pointer to dm_verity_fec_io after dm_verity_io and its variable | |
27 | * length fields. | |
28 | */ | |
29 | static inline struct dm_verity_fec_io *fec_io(struct dm_verity_io *io) | |
30 | { | |
31 | return (struct dm_verity_fec_io *) verity_io_digest_end(io->v, io); | |
32 | } | |
33 | ||
34 | /* | |
35 | * Return an interleaved offset for a byte in RS block. | |
36 | */ | |
37 | static inline u64 fec_interleave(struct dm_verity *v, u64 offset) | |
38 | { | |
39 | u32 mod; | |
40 | ||
41 | mod = do_div(offset, v->fec->rsn); | |
42 | return offset + mod * (v->fec->rounds << v->data_dev_block_bits); | |
43 | } | |
44 | ||
45 | /* | |
46 | * Decode an RS block using Reed-Solomon. | |
47 | */ | |
48 | static int fec_decode_rs8(struct dm_verity *v, struct dm_verity_fec_io *fio, | |
49 | u8 *data, u8 *fec, int neras) | |
50 | { | |
51 | int i; | |
52 | uint16_t par[DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN]; | |
53 | ||
54 | for (i = 0; i < v->fec->roots; i++) | |
55 | par[i] = fec[i]; | |
56 | ||
57 | return decode_rs8(fio->rs, data, par, v->fec->rsn, NULL, neras, | |
58 | fio->erasures, 0, NULL); | |
59 | } | |
60 | ||
61 | /* | |
62 | * Read error-correcting codes for the requested RS block. Returns a pointer | |
63 | * to the data block. Caller is responsible for releasing buf. | |
64 | */ | |
65 | static u8 *fec_read_parity(struct dm_verity *v, u64 rsb, int index, | |
66 | unsigned *offset, struct dm_buffer **buf) | |
67 | { | |
68 | u64 position, block; | |
69 | u8 *res; | |
70 | ||
71 | position = (index + rsb) * v->fec->roots; | |
72 | block = position >> v->data_dev_block_bits; | |
73 | *offset = (unsigned)(position - (block << v->data_dev_block_bits)); | |
74 | ||
75 | res = dm_bufio_read(v->fec->bufio, v->fec->start + block, buf); | |
821b40da | 76 | if (IS_ERR(res)) { |
a739ff3f ST |
77 | DMERR("%s: FEC %llu: parity read failed (block %llu): %ld", |
78 | v->data_dev->name, (unsigned long long)rsb, | |
79 | (unsigned long long)(v->fec->start + block), | |
80 | PTR_ERR(res)); | |
81 | *buf = NULL; | |
82 | } | |
83 | ||
84 | return res; | |
85 | } | |
86 | ||
87 | /* Loop over each preallocated buffer slot. */ | |
88 | #define fec_for_each_prealloc_buffer(__i) \ | |
89 | for (__i = 0; __i < DM_VERITY_FEC_BUF_PREALLOC; __i++) | |
90 | ||
91 | /* Loop over each extra buffer slot. */ | |
92 | #define fec_for_each_extra_buffer(io, __i) \ | |
93 | for (__i = DM_VERITY_FEC_BUF_PREALLOC; __i < DM_VERITY_FEC_BUF_MAX; __i++) | |
94 | ||
95 | /* Loop over each allocated buffer. */ | |
96 | #define fec_for_each_buffer(io, __i) \ | |
97 | for (__i = 0; __i < (io)->nbufs; __i++) | |
98 | ||
99 | /* Loop over each RS block in each allocated buffer. */ | |
100 | #define fec_for_each_buffer_rs_block(io, __i, __j) \ | |
101 | fec_for_each_buffer(io, __i) \ | |
102 | for (__j = 0; __j < 1 << DM_VERITY_FEC_BUF_RS_BITS; __j++) | |
103 | ||
104 | /* | |
105 | * Return a pointer to the current RS block when called inside | |
106 | * fec_for_each_buffer_rs_block. | |
107 | */ | |
108 | static inline u8 *fec_buffer_rs_block(struct dm_verity *v, | |
109 | struct dm_verity_fec_io *fio, | |
110 | unsigned i, unsigned j) | |
111 | { | |
112 | return &fio->bufs[i][j * v->fec->rsn]; | |
113 | } | |
114 | ||
115 | /* | |
116 | * Return an index to the current RS block when called inside | |
117 | * fec_for_each_buffer_rs_block. | |
118 | */ | |
119 | static inline unsigned fec_buffer_rs_index(unsigned i, unsigned j) | |
120 | { | |
121 | return (i << DM_VERITY_FEC_BUF_RS_BITS) + j; | |
122 | } | |
123 | ||
124 | /* | |
125 | * Decode all RS blocks from buffers and copy corrected bytes into fio->output | |
126 | * starting from block_offset. | |
127 | */ | |
128 | static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio, | |
129 | u64 rsb, int byte_index, unsigned block_offset, | |
130 | int neras) | |
131 | { | |
132 | int r, corrected = 0, res; | |
133 | struct dm_buffer *buf; | |
134 | unsigned n, i, offset; | |
135 | u8 *par, *block; | |
136 | ||
137 | par = fec_read_parity(v, rsb, block_offset, &offset, &buf); | |
138 | if (IS_ERR(par)) | |
139 | return PTR_ERR(par); | |
140 | ||
141 | /* | |
142 | * Decode the RS blocks we have in bufs. Each RS block results in | |
143 | * one corrected target byte and consumes fec->roots parity bytes. | |
144 | */ | |
145 | fec_for_each_buffer_rs_block(fio, n, i) { | |
146 | block = fec_buffer_rs_block(v, fio, n, i); | |
147 | res = fec_decode_rs8(v, fio, block, &par[offset], neras); | |
148 | if (res < 0) { | |
a739ff3f ST |
149 | r = res; |
150 | goto error; | |
151 | } | |
152 | ||
153 | corrected += res; | |
154 | fio->output[block_offset] = block[byte_index]; | |
155 | ||
156 | block_offset++; | |
157 | if (block_offset >= 1 << v->data_dev_block_bits) | |
158 | goto done; | |
159 | ||
160 | /* read the next block when we run out of parity bytes */ | |
161 | offset += v->fec->roots; | |
162 | if (offset >= 1 << v->data_dev_block_bits) { | |
163 | dm_bufio_release(buf); | |
164 | ||
165 | par = fec_read_parity(v, rsb, block_offset, &offset, &buf); | |
821b40da | 166 | if (IS_ERR(par)) |
a739ff3f ST |
167 | return PTR_ERR(par); |
168 | } | |
169 | } | |
170 | done: | |
171 | r = corrected; | |
172 | error: | |
86e3e83b ST |
173 | dm_bufio_release(buf); |
174 | ||
a739ff3f ST |
175 | if (r < 0 && neras) |
176 | DMERR_LIMIT("%s: FEC %llu: failed to correct: %d", | |
177 | v->data_dev->name, (unsigned long long)rsb, r); | |
178 | else if (r > 0) | |
179 | DMWARN_LIMIT("%s: FEC %llu: corrected %d errors", | |
180 | v->data_dev->name, (unsigned long long)rsb, r); | |
181 | ||
182 | return r; | |
183 | } | |
184 | ||
185 | /* | |
186 | * Locate data block erasures using verity hashes. | |
187 | */ | |
188 | static int fec_is_erasure(struct dm_verity *v, struct dm_verity_io *io, | |
189 | u8 *want_digest, u8 *data) | |
190 | { | |
d1ac3ff0 | 191 | if (unlikely(verity_hash(v, verity_io_hash_req(v, io), |
a739ff3f ST |
192 | data, 1 << v->data_dev_block_bits, |
193 | verity_io_real_digest(v, io)))) | |
194 | return 0; | |
195 | ||
196 | return memcmp(verity_io_real_digest(v, io), want_digest, | |
197 | v->digest_size) != 0; | |
198 | } | |
199 | ||
200 | /* | |
201 | * Read data blocks that are part of the RS block and deinterleave as much as | |
202 | * fits into buffers. Check for erasure locations if @neras is non-NULL. | |
203 | */ | |
204 | static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io, | |
205 | u64 rsb, u64 target, unsigned block_offset, | |
206 | int *neras) | |
207 | { | |
0cc37c2d | 208 | bool is_zero; |
a739ff3f ST |
209 | int i, j, target_index = -1; |
210 | struct dm_buffer *buf; | |
211 | struct dm_bufio_client *bufio; | |
212 | struct dm_verity_fec_io *fio = fec_io(io); | |
213 | u64 block, ileaved; | |
214 | u8 *bbuf, *rs_block; | |
6d39a124 | 215 | u8 want_digest[HASH_MAX_DIGESTSIZE]; |
a739ff3f ST |
216 | unsigned n, k; |
217 | ||
218 | if (neras) | |
219 | *neras = 0; | |
220 | ||
6d39a124 KC |
221 | if (WARN_ON(v->digest_size > sizeof(want_digest))) |
222 | return -EINVAL; | |
223 | ||
a739ff3f ST |
224 | /* |
225 | * read each of the rsn data blocks that are part of the RS block, and | |
226 | * interleave contents to available bufs | |
227 | */ | |
228 | for (i = 0; i < v->fec->rsn; i++) { | |
229 | ileaved = fec_interleave(v, rsb * v->fec->rsn + i); | |
230 | ||
231 | /* | |
232 | * target is the data block we want to correct, target_index is | |
233 | * the index of this block within the rsn RS blocks | |
234 | */ | |
235 | if (ileaved == target) | |
236 | target_index = i; | |
237 | ||
238 | block = ileaved >> v->data_dev_block_bits; | |
239 | bufio = v->fec->data_bufio; | |
240 | ||
241 | if (block >= v->data_blocks) { | |
242 | block -= v->data_blocks; | |
243 | ||
244 | /* | |
245 | * blocks outside the area were assumed to contain | |
246 | * zeros when encoding data was generated | |
247 | */ | |
248 | if (unlikely(block >= v->fec->hash_blocks)) | |
249 | continue; | |
250 | ||
251 | block += v->hash_start; | |
252 | bufio = v->bufio; | |
253 | } | |
254 | ||
255 | bbuf = dm_bufio_read(bufio, block, &buf); | |
821b40da | 256 | if (IS_ERR(bbuf)) { |
a739ff3f ST |
257 | DMWARN_LIMIT("%s: FEC %llu: read failed (%llu): %ld", |
258 | v->data_dev->name, | |
259 | (unsigned long long)rsb, | |
260 | (unsigned long long)block, PTR_ERR(bbuf)); | |
261 | ||
262 | /* assume the block is corrupted */ | |
263 | if (neras && *neras <= v->fec->roots) | |
264 | fio->erasures[(*neras)++] = i; | |
265 | ||
266 | continue; | |
267 | } | |
268 | ||
269 | /* locate erasures if the block is on the data device */ | |
270 | if (bufio == v->fec->data_bufio && | |
0cc37c2d ST |
271 | verity_hash_for_block(v, io, block, want_digest, |
272 | &is_zero) == 0) { | |
273 | /* skip known zero blocks entirely */ | |
274 | if (is_zero) | |
86e3e83b | 275 | goto done; |
0cc37c2d | 276 | |
a739ff3f ST |
277 | /* |
278 | * skip if we have already found the theoretical | |
279 | * maximum number (i.e. fec->roots) of erasures | |
280 | */ | |
281 | if (neras && *neras <= v->fec->roots && | |
282 | fec_is_erasure(v, io, want_digest, bbuf)) | |
283 | fio->erasures[(*neras)++] = i; | |
284 | } | |
285 | ||
286 | /* | |
287 | * deinterleave and copy the bytes that fit into bufs, | |
288 | * starting from block_offset | |
289 | */ | |
290 | fec_for_each_buffer_rs_block(fio, n, j) { | |
291 | k = fec_buffer_rs_index(n, j) + block_offset; | |
292 | ||
293 | if (k >= 1 << v->data_dev_block_bits) | |
294 | goto done; | |
295 | ||
296 | rs_block = fec_buffer_rs_block(v, fio, n, j); | |
297 | rs_block[i] = bbuf[k]; | |
298 | } | |
299 | done: | |
300 | dm_bufio_release(buf); | |
301 | } | |
302 | ||
303 | return target_index; | |
304 | } | |
305 | ||
306 | /* | |
307 | * Allocate RS control structure and FEC buffers from preallocated mempools, | |
308 | * and attempt to allocate as many extra buffers as available. | |
309 | */ | |
310 | static int fec_alloc_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio) | |
311 | { | |
312 | unsigned n; | |
313 | ||
34c96507 | 314 | if (!fio->rs) |
6f1c819c | 315 | fio->rs = mempool_alloc(&v->fec->rs_pool, GFP_NOIO); |
a739ff3f ST |
316 | |
317 | fec_for_each_prealloc_buffer(n) { | |
318 | if (fio->bufs[n]) | |
319 | continue; | |
320 | ||
6f1c819c | 321 | fio->bufs[n] = mempool_alloc(&v->fec->prealloc_pool, GFP_NOWAIT); |
a739ff3f ST |
322 | if (unlikely(!fio->bufs[n])) { |
323 | DMERR("failed to allocate FEC buffer"); | |
324 | return -ENOMEM; | |
325 | } | |
326 | } | |
327 | ||
328 | /* try to allocate the maximum number of buffers */ | |
329 | fec_for_each_extra_buffer(fio, n) { | |
330 | if (fio->bufs[n]) | |
331 | continue; | |
332 | ||
6f1c819c | 333 | fio->bufs[n] = mempool_alloc(&v->fec->extra_pool, GFP_NOWAIT); |
a739ff3f ST |
334 | /* we can manage with even one buffer if necessary */ |
335 | if (unlikely(!fio->bufs[n])) | |
336 | break; | |
337 | } | |
338 | fio->nbufs = n; | |
339 | ||
34c96507 | 340 | if (!fio->output) |
6f1c819c | 341 | fio->output = mempool_alloc(&v->fec->output_pool, GFP_NOIO); |
a739ff3f | 342 | |
a739ff3f ST |
343 | return 0; |
344 | } | |
345 | ||
346 | /* | |
347 | * Initialize buffers and clear erasures. fec_read_bufs() assumes buffers are | |
348 | * zeroed before deinterleaving. | |
349 | */ | |
350 | static void fec_init_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio) | |
351 | { | |
352 | unsigned n; | |
353 | ||
354 | fec_for_each_buffer(fio, n) | |
355 | memset(fio->bufs[n], 0, v->fec->rsn << DM_VERITY_FEC_BUF_RS_BITS); | |
356 | ||
357 | memset(fio->erasures, 0, sizeof(fio->erasures)); | |
358 | } | |
359 | ||
360 | /* | |
361 | * Decode all RS blocks in a single data block and return the target block | |
362 | * (indicated by @offset) in fio->output. If @use_erasures is non-zero, uses | |
363 | * hashes to locate erasures. | |
364 | */ | |
365 | static int fec_decode_rsb(struct dm_verity *v, struct dm_verity_io *io, | |
366 | struct dm_verity_fec_io *fio, u64 rsb, u64 offset, | |
367 | bool use_erasures) | |
368 | { | |
369 | int r, neras = 0; | |
370 | unsigned pos; | |
371 | ||
372 | r = fec_alloc_bufs(v, fio); | |
373 | if (unlikely(r < 0)) | |
374 | return r; | |
375 | ||
376 | for (pos = 0; pos < 1 << v->data_dev_block_bits; ) { | |
377 | fec_init_bufs(v, fio); | |
378 | ||
379 | r = fec_read_bufs(v, io, rsb, offset, pos, | |
380 | use_erasures ? &neras : NULL); | |
381 | if (unlikely(r < 0)) | |
382 | return r; | |
383 | ||
384 | r = fec_decode_bufs(v, fio, rsb, r, pos, neras); | |
385 | if (r < 0) | |
386 | return r; | |
387 | ||
388 | pos += fio->nbufs << DM_VERITY_FEC_BUF_RS_BITS; | |
389 | } | |
390 | ||
391 | /* Always re-validate the corrected block against the expected hash */ | |
d1ac3ff0 | 392 | r = verity_hash(v, verity_io_hash_req(v, io), fio->output, |
a739ff3f ST |
393 | 1 << v->data_dev_block_bits, |
394 | verity_io_real_digest(v, io)); | |
395 | if (unlikely(r < 0)) | |
396 | return r; | |
397 | ||
398 | if (memcmp(verity_io_real_digest(v, io), verity_io_want_digest(v, io), | |
399 | v->digest_size)) { | |
400 | DMERR_LIMIT("%s: FEC %llu: failed to correct (%d erasures)", | |
401 | v->data_dev->name, (unsigned long long)rsb, neras); | |
402 | return -EILSEQ; | |
403 | } | |
404 | ||
405 | return 0; | |
406 | } | |
407 | ||
408 | static int fec_bv_copy(struct dm_verity *v, struct dm_verity_io *io, u8 *data, | |
409 | size_t len) | |
410 | { | |
411 | struct dm_verity_fec_io *fio = fec_io(io); | |
412 | ||
413 | memcpy(data, &fio->output[fio->output_pos], len); | |
414 | fio->output_pos += len; | |
415 | ||
416 | return 0; | |
417 | } | |
418 | ||
419 | /* | |
420 | * Correct errors in a block. Copies corrected block to dest if non-NULL, | |
421 | * otherwise to a bio_vec starting from iter. | |
422 | */ | |
423 | int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io, | |
424 | enum verity_block_type type, sector_t block, u8 *dest, | |
425 | struct bvec_iter *iter) | |
426 | { | |
427 | int r; | |
428 | struct dm_verity_fec_io *fio = fec_io(io); | |
429 | u64 offset, res, rsb; | |
430 | ||
431 | if (!verity_fec_is_enabled(v)) | |
432 | return -EOPNOTSUPP; | |
433 | ||
f1a880a9 ST |
434 | if (fio->level >= DM_VERITY_FEC_MAX_RECURSION) { |
435 | DMWARN_LIMIT("%s: FEC: recursion too deep", v->data_dev->name); | |
436 | return -EIO; | |
437 | } | |
438 | ||
439 | fio->level++; | |
440 | ||
a739ff3f ST |
441 | if (type == DM_VERITY_BLOCK_TYPE_METADATA) |
442 | block += v->data_blocks; | |
443 | ||
444 | /* | |
445 | * For RS(M, N), the continuous FEC data is divided into blocks of N | |
446 | * bytes. Since block size may not be divisible by N, the last block | |
447 | * is zero padded when decoding. | |
448 | * | |
449 | * Each byte of the block is covered by a different RS(M, N) code, | |
450 | * and each code is interleaved over N blocks to make it less likely | |
451 | * that bursty corruption will leave us in unrecoverable state. | |
452 | */ | |
453 | ||
454 | offset = block << v->data_dev_block_bits; | |
602d1657 | 455 | res = div64_u64(offset, v->fec->rounds << v->data_dev_block_bits); |
a739ff3f ST |
456 | |
457 | /* | |
458 | * The base RS block we can feed to the interleaver to find out all | |
459 | * blocks required for decoding. | |
460 | */ | |
461 | rsb = offset - res * (v->fec->rounds << v->data_dev_block_bits); | |
462 | ||
463 | /* | |
464 | * Locating erasures is slow, so attempt to recover the block without | |
465 | * them first. Do a second attempt with erasures if the corruption is | |
466 | * bad enough. | |
467 | */ | |
468 | r = fec_decode_rsb(v, io, fio, rsb, offset, false); | |
469 | if (r < 0) { | |
470 | r = fec_decode_rsb(v, io, fio, rsb, offset, true); | |
471 | if (r < 0) | |
f1a880a9 | 472 | goto done; |
a739ff3f ST |
473 | } |
474 | ||
475 | if (dest) | |
476 | memcpy(dest, fio->output, 1 << v->data_dev_block_bits); | |
477 | else if (iter) { | |
478 | fio->output_pos = 0; | |
479 | r = verity_for_bv_block(v, io, iter, fec_bv_copy); | |
480 | } | |
481 | ||
f1a880a9 ST |
482 | done: |
483 | fio->level--; | |
a739ff3f ST |
484 | return r; |
485 | } | |
486 | ||
487 | /* | |
488 | * Clean up per-bio data. | |
489 | */ | |
490 | void verity_fec_finish_io(struct dm_verity_io *io) | |
491 | { | |
492 | unsigned n; | |
493 | struct dm_verity_fec *f = io->v->fec; | |
494 | struct dm_verity_fec_io *fio = fec_io(io); | |
495 | ||
496 | if (!verity_fec_is_enabled(io->v)) | |
497 | return; | |
498 | ||
6f1c819c | 499 | mempool_free(fio->rs, &f->rs_pool); |
a739ff3f ST |
500 | |
501 | fec_for_each_prealloc_buffer(n) | |
6f1c819c | 502 | mempool_free(fio->bufs[n], &f->prealloc_pool); |
a739ff3f ST |
503 | |
504 | fec_for_each_extra_buffer(fio, n) | |
6f1c819c | 505 | mempool_free(fio->bufs[n], &f->extra_pool); |
a739ff3f | 506 | |
6f1c819c | 507 | mempool_free(fio->output, &f->output_pool); |
a739ff3f ST |
508 | } |
509 | ||
510 | /* | |
511 | * Initialize per-bio data. | |
512 | */ | |
513 | void verity_fec_init_io(struct dm_verity_io *io) | |
514 | { | |
515 | struct dm_verity_fec_io *fio = fec_io(io); | |
516 | ||
517 | if (!verity_fec_is_enabled(io->v)) | |
518 | return; | |
519 | ||
520 | fio->rs = NULL; | |
521 | memset(fio->bufs, 0, sizeof(fio->bufs)); | |
522 | fio->nbufs = 0; | |
523 | fio->output = NULL; | |
f1a880a9 | 524 | fio->level = 0; |
a739ff3f ST |
525 | } |
526 | ||
527 | /* | |
528 | * Append feature arguments and values to the status table. | |
529 | */ | |
530 | unsigned verity_fec_status_table(struct dm_verity *v, unsigned sz, | |
531 | char *result, unsigned maxlen) | |
532 | { | |
533 | if (!verity_fec_is_enabled(v)) | |
534 | return sz; | |
535 | ||
536 | DMEMIT(" " DM_VERITY_OPT_FEC_DEV " %s " | |
537 | DM_VERITY_OPT_FEC_BLOCKS " %llu " | |
538 | DM_VERITY_OPT_FEC_START " %llu " | |
539 | DM_VERITY_OPT_FEC_ROOTS " %d", | |
540 | v->fec->dev->name, | |
541 | (unsigned long long)v->fec->blocks, | |
542 | (unsigned long long)v->fec->start, | |
543 | v->fec->roots); | |
544 | ||
545 | return sz; | |
546 | } | |
547 | ||
548 | void verity_fec_dtr(struct dm_verity *v) | |
549 | { | |
550 | struct dm_verity_fec *f = v->fec; | |
551 | ||
552 | if (!verity_fec_is_enabled(v)) | |
553 | goto out; | |
554 | ||
6f1c819c KO |
555 | mempool_exit(&f->rs_pool); |
556 | mempool_exit(&f->prealloc_pool); | |
557 | mempool_exit(&f->extra_pool); | |
a739ff3f ST |
558 | kmem_cache_destroy(f->cache); |
559 | ||
560 | if (f->data_bufio) | |
561 | dm_bufio_client_destroy(f->data_bufio); | |
562 | if (f->bufio) | |
563 | dm_bufio_client_destroy(f->bufio); | |
564 | ||
565 | if (f->dev) | |
566 | dm_put_device(v->ti, f->dev); | |
567 | out: | |
568 | kfree(f); | |
569 | v->fec = NULL; | |
570 | } | |
571 | ||
572 | static void *fec_rs_alloc(gfp_t gfp_mask, void *pool_data) | |
573 | { | |
574 | struct dm_verity *v = (struct dm_verity *)pool_data; | |
575 | ||
eb366989 | 576 | return init_rs_gfp(8, 0x11d, 0, 1, v->fec->roots, gfp_mask); |
a739ff3f ST |
577 | } |
578 | ||
579 | static void fec_rs_free(void *element, void *pool_data) | |
580 | { | |
581 | struct rs_control *rs = (struct rs_control *)element; | |
582 | ||
583 | if (rs) | |
584 | free_rs(rs); | |
585 | } | |
586 | ||
587 | bool verity_is_fec_opt_arg(const char *arg_name) | |
588 | { | |
589 | return (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_DEV) || | |
590 | !strcasecmp(arg_name, DM_VERITY_OPT_FEC_BLOCKS) || | |
591 | !strcasecmp(arg_name, DM_VERITY_OPT_FEC_START) || | |
592 | !strcasecmp(arg_name, DM_VERITY_OPT_FEC_ROOTS)); | |
593 | } | |
594 | ||
595 | int verity_fec_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, | |
596 | unsigned *argc, const char *arg_name) | |
597 | { | |
598 | int r; | |
599 | struct dm_target *ti = v->ti; | |
600 | const char *arg_value; | |
601 | unsigned long long num_ll; | |
602 | unsigned char num_c; | |
603 | char dummy; | |
604 | ||
605 | if (!*argc) { | |
606 | ti->error = "FEC feature arguments require a value"; | |
607 | return -EINVAL; | |
608 | } | |
609 | ||
610 | arg_value = dm_shift_arg(as); | |
611 | (*argc)--; | |
612 | ||
613 | if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_DEV)) { | |
614 | r = dm_get_device(ti, arg_value, FMODE_READ, &v->fec->dev); | |
615 | if (r) { | |
616 | ti->error = "FEC device lookup failed"; | |
617 | return r; | |
618 | } | |
619 | ||
620 | } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_BLOCKS)) { | |
621 | if (sscanf(arg_value, "%llu%c", &num_ll, &dummy) != 1 || | |
622 | ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) | |
623 | >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) { | |
624 | ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS; | |
625 | return -EINVAL; | |
626 | } | |
627 | v->fec->blocks = num_ll; | |
628 | ||
629 | } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_START)) { | |
630 | if (sscanf(arg_value, "%llu%c", &num_ll, &dummy) != 1 || | |
631 | ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) >> | |
632 | (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) { | |
633 | ti->error = "Invalid " DM_VERITY_OPT_FEC_START; | |
634 | return -EINVAL; | |
635 | } | |
636 | v->fec->start = num_ll; | |
637 | ||
638 | } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_ROOTS)) { | |
639 | if (sscanf(arg_value, "%hhu%c", &num_c, &dummy) != 1 || !num_c || | |
640 | num_c < (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MAX_RSN) || | |
641 | num_c > (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN)) { | |
642 | ti->error = "Invalid " DM_VERITY_OPT_FEC_ROOTS; | |
643 | return -EINVAL; | |
644 | } | |
645 | v->fec->roots = num_c; | |
646 | ||
647 | } else { | |
648 | ti->error = "Unrecognized verity FEC feature request"; | |
649 | return -EINVAL; | |
650 | } | |
651 | ||
652 | return 0; | |
653 | } | |
654 | ||
655 | /* | |
656 | * Allocate dm_verity_fec for v->fec. Must be called before verity_fec_ctr. | |
657 | */ | |
658 | int verity_fec_ctr_alloc(struct dm_verity *v) | |
659 | { | |
660 | struct dm_verity_fec *f; | |
661 | ||
662 | f = kzalloc(sizeof(struct dm_verity_fec), GFP_KERNEL); | |
663 | if (!f) { | |
664 | v->ti->error = "Cannot allocate FEC structure"; | |
665 | return -ENOMEM; | |
666 | } | |
667 | v->fec = f; | |
668 | ||
669 | return 0; | |
670 | } | |
671 | ||
672 | /* | |
673 | * Validate arguments and preallocate memory. Must be called after arguments | |
674 | * have been parsed using verity_fec_parse_opt_args. | |
675 | */ | |
676 | int verity_fec_ctr(struct dm_verity *v) | |
677 | { | |
678 | struct dm_verity_fec *f = v->fec; | |
679 | struct dm_target *ti = v->ti; | |
680 | u64 hash_blocks; | |
6f1c819c | 681 | int ret; |
a739ff3f ST |
682 | |
683 | if (!verity_fec_is_enabled(v)) { | |
684 | verity_fec_dtr(v); | |
685 | return 0; | |
686 | } | |
687 | ||
688 | /* | |
689 | * FEC is computed over data blocks, possible metadata, and | |
690 | * hash blocks. In other words, FEC covers total of fec_blocks | |
691 | * blocks consisting of the following: | |
692 | * | |
693 | * data blocks | hash blocks | metadata (optional) | |
694 | * | |
695 | * We allow metadata after hash blocks to support a use case | |
696 | * where all data is stored on the same device and FEC covers | |
697 | * the entire area. | |
698 | * | |
699 | * If metadata is included, we require it to be available on the | |
700 | * hash device after the hash blocks. | |
701 | */ | |
702 | ||
703 | hash_blocks = v->hash_blocks - v->hash_start; | |
704 | ||
705 | /* | |
706 | * Require matching block sizes for data and hash devices for | |
707 | * simplicity. | |
708 | */ | |
709 | if (v->data_dev_block_bits != v->hash_dev_block_bits) { | |
710 | ti->error = "Block sizes must match to use FEC"; | |
711 | return -EINVAL; | |
712 | } | |
713 | ||
714 | if (!f->roots) { | |
715 | ti->error = "Missing " DM_VERITY_OPT_FEC_ROOTS; | |
716 | return -EINVAL; | |
717 | } | |
718 | f->rsn = DM_VERITY_FEC_RSM - f->roots; | |
719 | ||
720 | if (!f->blocks) { | |
721 | ti->error = "Missing " DM_VERITY_OPT_FEC_BLOCKS; | |
722 | return -EINVAL; | |
723 | } | |
724 | ||
725 | f->rounds = f->blocks; | |
726 | if (sector_div(f->rounds, f->rsn)) | |
727 | f->rounds++; | |
728 | ||
729 | /* | |
730 | * Due to optional metadata, f->blocks can be larger than | |
731 | * data_blocks and hash_blocks combined. | |
732 | */ | |
733 | if (f->blocks < v->data_blocks + hash_blocks || !f->rounds) { | |
734 | ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS; | |
735 | return -EINVAL; | |
736 | } | |
737 | ||
738 | /* | |
739 | * Metadata is accessed through the hash device, so we require | |
740 | * it to be large enough. | |
741 | */ | |
742 | f->hash_blocks = f->blocks - v->data_blocks; | |
743 | if (dm_bufio_get_device_size(v->bufio) < f->hash_blocks) { | |
744 | ti->error = "Hash device is too small for " | |
745 | DM_VERITY_OPT_FEC_BLOCKS; | |
746 | return -E2BIG; | |
747 | } | |
748 | ||
749 | f->bufio = dm_bufio_client_create(f->dev->bdev, | |
750 | 1 << v->data_dev_block_bits, | |
751 | 1, 0, NULL, NULL); | |
752 | if (IS_ERR(f->bufio)) { | |
753 | ti->error = "Cannot initialize FEC bufio client"; | |
754 | return PTR_ERR(f->bufio); | |
755 | } | |
756 | ||
757 | if (dm_bufio_get_device_size(f->bufio) < | |
758 | ((f->start + f->rounds * f->roots) >> v->data_dev_block_bits)) { | |
759 | ti->error = "FEC device is too small"; | |
760 | return -E2BIG; | |
761 | } | |
762 | ||
763 | f->data_bufio = dm_bufio_client_create(v->data_dev->bdev, | |
764 | 1 << v->data_dev_block_bits, | |
765 | 1, 0, NULL, NULL); | |
766 | if (IS_ERR(f->data_bufio)) { | |
767 | ti->error = "Cannot initialize FEC data bufio client"; | |
768 | return PTR_ERR(f->data_bufio); | |
769 | } | |
770 | ||
771 | if (dm_bufio_get_device_size(f->data_bufio) < v->data_blocks) { | |
772 | ti->error = "Data device is too small"; | |
773 | return -E2BIG; | |
774 | } | |
775 | ||
776 | /* Preallocate an rs_control structure for each worker thread */ | |
6f1c819c KO |
777 | ret = mempool_init(&f->rs_pool, num_online_cpus(), fec_rs_alloc, |
778 | fec_rs_free, (void *) v); | |
779 | if (ret) { | |
a739ff3f | 780 | ti->error = "Cannot allocate RS pool"; |
6f1c819c | 781 | return ret; |
a739ff3f ST |
782 | } |
783 | ||
784 | f->cache = kmem_cache_create("dm_verity_fec_buffers", | |
785 | f->rsn << DM_VERITY_FEC_BUF_RS_BITS, | |
786 | 0, 0, NULL); | |
787 | if (!f->cache) { | |
788 | ti->error = "Cannot create FEC buffer cache"; | |
789 | return -ENOMEM; | |
790 | } | |
791 | ||
792 | /* Preallocate DM_VERITY_FEC_BUF_PREALLOC buffers for each thread */ | |
6f1c819c KO |
793 | ret = mempool_init_slab_pool(&f->prealloc_pool, num_online_cpus() * |
794 | DM_VERITY_FEC_BUF_PREALLOC, | |
795 | f->cache); | |
796 | if (ret) { | |
a739ff3f | 797 | ti->error = "Cannot allocate FEC buffer prealloc pool"; |
6f1c819c | 798 | return ret; |
a739ff3f ST |
799 | } |
800 | ||
6f1c819c KO |
801 | ret = mempool_init_slab_pool(&f->extra_pool, 0, f->cache); |
802 | if (ret) { | |
a739ff3f | 803 | ti->error = "Cannot allocate FEC buffer extra pool"; |
6f1c819c | 804 | return ret; |
a739ff3f ST |
805 | } |
806 | ||
807 | /* Preallocate an output buffer for each thread */ | |
6f1c819c KO |
808 | ret = mempool_init_kmalloc_pool(&f->output_pool, num_online_cpus(), |
809 | 1 << v->data_dev_block_bits); | |
810 | if (ret) { | |
a739ff3f | 811 | ti->error = "Cannot allocate FEC output pool"; |
6f1c819c | 812 | return ret; |
a739ff3f ST |
813 | } |
814 | ||
815 | /* Reserve space for our per-bio data */ | |
30187e1d | 816 | ti->per_io_data_size += sizeof(struct dm_verity_fec_io); |
a739ff3f ST |
817 | |
818 | return 0; | |
819 | } |