]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blob - lib/iov_iter.c
Merge branch 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6
[mirror_ubuntu-artful-kernel.git] / lib / iov_iter.c
1 #include <linux/export.h>
2 #include <linux/uio.h>
3 #include <linux/pagemap.h>
4 #include <linux/slab.h>
5 #include <linux/vmalloc.h>
6 #include <net/checksum.h>
7
8 #define iterate_iovec(i, n, __v, __p, skip, STEP) { \
9 size_t left; \
10 size_t wanted = n; \
11 __p = i->iov; \
12 __v.iov_len = min(n, __p->iov_len - skip); \
13 if (likely(__v.iov_len)) { \
14 __v.iov_base = __p->iov_base + skip; \
15 left = (STEP); \
16 __v.iov_len -= left; \
17 skip += __v.iov_len; \
18 n -= __v.iov_len; \
19 } else { \
20 left = 0; \
21 } \
22 while (unlikely(!left && n)) { \
23 __p++; \
24 __v.iov_len = min(n, __p->iov_len); \
25 if (unlikely(!__v.iov_len)) \
26 continue; \
27 __v.iov_base = __p->iov_base; \
28 left = (STEP); \
29 __v.iov_len -= left; \
30 skip = __v.iov_len; \
31 n -= __v.iov_len; \
32 } \
33 n = wanted - n; \
34 }
35
36 #define iterate_kvec(i, n, __v, __p, skip, STEP) { \
37 size_t wanted = n; \
38 __p = i->kvec; \
39 __v.iov_len = min(n, __p->iov_len - skip); \
40 if (likely(__v.iov_len)) { \
41 __v.iov_base = __p->iov_base + skip; \
42 (void)(STEP); \
43 skip += __v.iov_len; \
44 n -= __v.iov_len; \
45 } \
46 while (unlikely(n)) { \
47 __p++; \
48 __v.iov_len = min(n, __p->iov_len); \
49 if (unlikely(!__v.iov_len)) \
50 continue; \
51 __v.iov_base = __p->iov_base; \
52 (void)(STEP); \
53 skip = __v.iov_len; \
54 n -= __v.iov_len; \
55 } \
56 n = wanted; \
57 }
58
59 #define iterate_bvec(i, n, __v, __p, skip, STEP) { \
60 size_t wanted = n; \
61 __p = i->bvec; \
62 __v.bv_len = min_t(size_t, n, __p->bv_len - skip); \
63 if (likely(__v.bv_len)) { \
64 __v.bv_page = __p->bv_page; \
65 __v.bv_offset = __p->bv_offset + skip; \
66 (void)(STEP); \
67 skip += __v.bv_len; \
68 n -= __v.bv_len; \
69 } \
70 while (unlikely(n)) { \
71 __p++; \
72 __v.bv_len = min_t(size_t, n, __p->bv_len); \
73 if (unlikely(!__v.bv_len)) \
74 continue; \
75 __v.bv_page = __p->bv_page; \
76 __v.bv_offset = __p->bv_offset; \
77 (void)(STEP); \
78 skip = __v.bv_len; \
79 n -= __v.bv_len; \
80 } \
81 n = wanted; \
82 }
83
84 #define iterate_all_kinds(i, n, v, I, B, K) { \
85 size_t skip = i->iov_offset; \
86 if (unlikely(i->type & ITER_BVEC)) { \
87 const struct bio_vec *bvec; \
88 struct bio_vec v; \
89 iterate_bvec(i, n, v, bvec, skip, (B)) \
90 } else if (unlikely(i->type & ITER_KVEC)) { \
91 const struct kvec *kvec; \
92 struct kvec v; \
93 iterate_kvec(i, n, v, kvec, skip, (K)) \
94 } else { \
95 const struct iovec *iov; \
96 struct iovec v; \
97 iterate_iovec(i, n, v, iov, skip, (I)) \
98 } \
99 }
100
101 #define iterate_and_advance(i, n, v, I, B, K) { \
102 if (unlikely(i->count < n)) \
103 n = i->count; \
104 if (n) { \
105 size_t skip = i->iov_offset; \
106 if (unlikely(i->type & ITER_BVEC)) { \
107 const struct bio_vec *bvec; \
108 struct bio_vec v; \
109 iterate_bvec(i, n, v, bvec, skip, (B)) \
110 if (skip == bvec->bv_len) { \
111 bvec++; \
112 skip = 0; \
113 } \
114 i->nr_segs -= bvec - i->bvec; \
115 i->bvec = bvec; \
116 } else if (unlikely(i->type & ITER_KVEC)) { \
117 const struct kvec *kvec; \
118 struct kvec v; \
119 iterate_kvec(i, n, v, kvec, skip, (K)) \
120 if (skip == kvec->iov_len) { \
121 kvec++; \
122 skip = 0; \
123 } \
124 i->nr_segs -= kvec - i->kvec; \
125 i->kvec = kvec; \
126 } else { \
127 const struct iovec *iov; \
128 struct iovec v; \
129 iterate_iovec(i, n, v, iov, skip, (I)) \
130 if (skip == iov->iov_len) { \
131 iov++; \
132 skip = 0; \
133 } \
134 i->nr_segs -= iov - i->iov; \
135 i->iov = iov; \
136 } \
137 i->count -= n; \
138 i->iov_offset = skip; \
139 } \
140 }
141
142 static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,
143 struct iov_iter *i)
144 {
145 size_t skip, copy, left, wanted;
146 const struct iovec *iov;
147 char __user *buf;
148 void *kaddr, *from;
149
150 if (unlikely(bytes > i->count))
151 bytes = i->count;
152
153 if (unlikely(!bytes))
154 return 0;
155
156 wanted = bytes;
157 iov = i->iov;
158 skip = i->iov_offset;
159 buf = iov->iov_base + skip;
160 copy = min(bytes, iov->iov_len - skip);
161
162 if (!fault_in_pages_writeable(buf, copy)) {
163 kaddr = kmap_atomic(page);
164 from = kaddr + offset;
165
166 /* first chunk, usually the only one */
167 left = __copy_to_user_inatomic(buf, from, copy);
168 copy -= left;
169 skip += copy;
170 from += copy;
171 bytes -= copy;
172
173 while (unlikely(!left && bytes)) {
174 iov++;
175 buf = iov->iov_base;
176 copy = min(bytes, iov->iov_len);
177 left = __copy_to_user_inatomic(buf, from, copy);
178 copy -= left;
179 skip = copy;
180 from += copy;
181 bytes -= copy;
182 }
183 if (likely(!bytes)) {
184 kunmap_atomic(kaddr);
185 goto done;
186 }
187 offset = from - kaddr;
188 buf += copy;
189 kunmap_atomic(kaddr);
190 copy = min(bytes, iov->iov_len - skip);
191 }
192 /* Too bad - revert to non-atomic kmap */
193 kaddr = kmap(page);
194 from = kaddr + offset;
195 left = __copy_to_user(buf, from, copy);
196 copy -= left;
197 skip += copy;
198 from += copy;
199 bytes -= copy;
200 while (unlikely(!left && bytes)) {
201 iov++;
202 buf = iov->iov_base;
203 copy = min(bytes, iov->iov_len);
204 left = __copy_to_user(buf, from, copy);
205 copy -= left;
206 skip = copy;
207 from += copy;
208 bytes -= copy;
209 }
210 kunmap(page);
211 done:
212 if (skip == iov->iov_len) {
213 iov++;
214 skip = 0;
215 }
216 i->count -= wanted - bytes;
217 i->nr_segs -= iov - i->iov;
218 i->iov = iov;
219 i->iov_offset = skip;
220 return wanted - bytes;
221 }
222
223 static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,
224 struct iov_iter *i)
225 {
226 size_t skip, copy, left, wanted;
227 const struct iovec *iov;
228 char __user *buf;
229 void *kaddr, *to;
230
231 if (unlikely(bytes > i->count))
232 bytes = i->count;
233
234 if (unlikely(!bytes))
235 return 0;
236
237 wanted = bytes;
238 iov = i->iov;
239 skip = i->iov_offset;
240 buf = iov->iov_base + skip;
241 copy = min(bytes, iov->iov_len - skip);
242
243 if (!fault_in_pages_readable(buf, copy)) {
244 kaddr = kmap_atomic(page);
245 to = kaddr + offset;
246
247 /* first chunk, usually the only one */
248 left = __copy_from_user_inatomic(to, buf, copy);
249 copy -= left;
250 skip += copy;
251 to += copy;
252 bytes -= copy;
253
254 while (unlikely(!left && bytes)) {
255 iov++;
256 buf = iov->iov_base;
257 copy = min(bytes, iov->iov_len);
258 left = __copy_from_user_inatomic(to, buf, copy);
259 copy -= left;
260 skip = copy;
261 to += copy;
262 bytes -= copy;
263 }
264 if (likely(!bytes)) {
265 kunmap_atomic(kaddr);
266 goto done;
267 }
268 offset = to - kaddr;
269 buf += copy;
270 kunmap_atomic(kaddr);
271 copy = min(bytes, iov->iov_len - skip);
272 }
273 /* Too bad - revert to non-atomic kmap */
274 kaddr = kmap(page);
275 to = kaddr + offset;
276 left = __copy_from_user(to, buf, copy);
277 copy -= left;
278 skip += copy;
279 to += copy;
280 bytes -= copy;
281 while (unlikely(!left && bytes)) {
282 iov++;
283 buf = iov->iov_base;
284 copy = min(bytes, iov->iov_len);
285 left = __copy_from_user(to, buf, copy);
286 copy -= left;
287 skip = copy;
288 to += copy;
289 bytes -= copy;
290 }
291 kunmap(page);
292 done:
293 if (skip == iov->iov_len) {
294 iov++;
295 skip = 0;
296 }
297 i->count -= wanted - bytes;
298 i->nr_segs -= iov - i->iov;
299 i->iov = iov;
300 i->iov_offset = skip;
301 return wanted - bytes;
302 }
303
304 /*
305 * Fault in the first iovec of the given iov_iter, to a maximum length
306 * of bytes. Returns 0 on success, or non-zero if the memory could not be
307 * accessed (ie. because it is an invalid address).
308 *
309 * writev-intensive code may want this to prefault several iovecs -- that
310 * would be possible (callers must not rely on the fact that _only_ the
311 * first iovec will be faulted with the current implementation).
312 */
313 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
314 {
315 if (!(i->type & (ITER_BVEC|ITER_KVEC))) {
316 char __user *buf = i->iov->iov_base + i->iov_offset;
317 bytes = min(bytes, i->iov->iov_len - i->iov_offset);
318 return fault_in_pages_readable(buf, bytes);
319 }
320 return 0;
321 }
322 EXPORT_SYMBOL(iov_iter_fault_in_readable);
323
324 /*
325 * Fault in one or more iovecs of the given iov_iter, to a maximum length of
326 * bytes. For each iovec, fault in each page that constitutes the iovec.
327 *
328 * Return 0 on success, or non-zero if the memory could not be accessed (i.e.
329 * because it is an invalid address).
330 */
331 int iov_iter_fault_in_multipages_readable(struct iov_iter *i, size_t bytes)
332 {
333 size_t skip = i->iov_offset;
334 const struct iovec *iov;
335 int err;
336 struct iovec v;
337
338 if (!(i->type & (ITER_BVEC|ITER_KVEC))) {
339 iterate_iovec(i, bytes, v, iov, skip, ({
340 err = fault_in_multipages_readable(v.iov_base,
341 v.iov_len);
342 if (unlikely(err))
343 return err;
344 0;}))
345 }
346 return 0;
347 }
348 EXPORT_SYMBOL(iov_iter_fault_in_multipages_readable);
349
350 void iov_iter_init(struct iov_iter *i, int direction,
351 const struct iovec *iov, unsigned long nr_segs,
352 size_t count)
353 {
354 /* It will get better. Eventually... */
355 if (segment_eq(get_fs(), KERNEL_DS)) {
356 direction |= ITER_KVEC;
357 i->type = direction;
358 i->kvec = (struct kvec *)iov;
359 } else {
360 i->type = direction;
361 i->iov = iov;
362 }
363 i->nr_segs = nr_segs;
364 i->iov_offset = 0;
365 i->count = count;
366 }
367 EXPORT_SYMBOL(iov_iter_init);
368
369 static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len)
370 {
371 char *from = kmap_atomic(page);
372 memcpy(to, from + offset, len);
373 kunmap_atomic(from);
374 }
375
376 static void memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len)
377 {
378 char *to = kmap_atomic(page);
379 memcpy(to + offset, from, len);
380 kunmap_atomic(to);
381 }
382
383 static void memzero_page(struct page *page, size_t offset, size_t len)
384 {
385 char *addr = kmap_atomic(page);
386 memset(addr + offset, 0, len);
387 kunmap_atomic(addr);
388 }
389
390 size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
391 {
392 const char *from = addr;
393 iterate_and_advance(i, bytes, v,
394 __copy_to_user(v.iov_base, (from += v.iov_len) - v.iov_len,
395 v.iov_len),
396 memcpy_to_page(v.bv_page, v.bv_offset,
397 (from += v.bv_len) - v.bv_len, v.bv_len),
398 memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len)
399 )
400
401 return bytes;
402 }
403 EXPORT_SYMBOL(copy_to_iter);
404
405 size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
406 {
407 char *to = addr;
408 iterate_and_advance(i, bytes, v,
409 __copy_from_user((to += v.iov_len) - v.iov_len, v.iov_base,
410 v.iov_len),
411 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
412 v.bv_offset, v.bv_len),
413 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
414 )
415
416 return bytes;
417 }
418 EXPORT_SYMBOL(copy_from_iter);
419
420 size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
421 {
422 char *to = addr;
423 iterate_and_advance(i, bytes, v,
424 __copy_from_user_nocache((to += v.iov_len) - v.iov_len,
425 v.iov_base, v.iov_len),
426 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
427 v.bv_offset, v.bv_len),
428 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
429 )
430
431 return bytes;
432 }
433 EXPORT_SYMBOL(copy_from_iter_nocache);
434
435 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
436 struct iov_iter *i)
437 {
438 if (i->type & (ITER_BVEC|ITER_KVEC)) {
439 void *kaddr = kmap_atomic(page);
440 size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
441 kunmap_atomic(kaddr);
442 return wanted;
443 } else
444 return copy_page_to_iter_iovec(page, offset, bytes, i);
445 }
446 EXPORT_SYMBOL(copy_page_to_iter);
447
448 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
449 struct iov_iter *i)
450 {
451 if (i->type & (ITER_BVEC|ITER_KVEC)) {
452 void *kaddr = kmap_atomic(page);
453 size_t wanted = copy_from_iter(kaddr + offset, bytes, i);
454 kunmap_atomic(kaddr);
455 return wanted;
456 } else
457 return copy_page_from_iter_iovec(page, offset, bytes, i);
458 }
459 EXPORT_SYMBOL(copy_page_from_iter);
460
461 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
462 {
463 iterate_and_advance(i, bytes, v,
464 __clear_user(v.iov_base, v.iov_len),
465 memzero_page(v.bv_page, v.bv_offset, v.bv_len),
466 memset(v.iov_base, 0, v.iov_len)
467 )
468
469 return bytes;
470 }
471 EXPORT_SYMBOL(iov_iter_zero);
472
473 size_t iov_iter_copy_from_user_atomic(struct page *page,
474 struct iov_iter *i, unsigned long offset, size_t bytes)
475 {
476 char *kaddr = kmap_atomic(page), *p = kaddr + offset;
477 iterate_all_kinds(i, bytes, v,
478 __copy_from_user_inatomic((p += v.iov_len) - v.iov_len,
479 v.iov_base, v.iov_len),
480 memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
481 v.bv_offset, v.bv_len),
482 memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
483 )
484 kunmap_atomic(kaddr);
485 return bytes;
486 }
487 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
488
489 void iov_iter_advance(struct iov_iter *i, size_t size)
490 {
491 iterate_and_advance(i, size, v, 0, 0, 0)
492 }
493 EXPORT_SYMBOL(iov_iter_advance);
494
495 /*
496 * Return the count of just the current iov_iter segment.
497 */
498 size_t iov_iter_single_seg_count(const struct iov_iter *i)
499 {
500 if (i->nr_segs == 1)
501 return i->count;
502 else if (i->type & ITER_BVEC)
503 return min(i->count, i->bvec->bv_len - i->iov_offset);
504 else
505 return min(i->count, i->iov->iov_len - i->iov_offset);
506 }
507 EXPORT_SYMBOL(iov_iter_single_seg_count);
508
509 void iov_iter_kvec(struct iov_iter *i, int direction,
510 const struct kvec *kvec, unsigned long nr_segs,
511 size_t count)
512 {
513 BUG_ON(!(direction & ITER_KVEC));
514 i->type = direction;
515 i->kvec = kvec;
516 i->nr_segs = nr_segs;
517 i->iov_offset = 0;
518 i->count = count;
519 }
520 EXPORT_SYMBOL(iov_iter_kvec);
521
522 void iov_iter_bvec(struct iov_iter *i, int direction,
523 const struct bio_vec *bvec, unsigned long nr_segs,
524 size_t count)
525 {
526 BUG_ON(!(direction & ITER_BVEC));
527 i->type = direction;
528 i->bvec = bvec;
529 i->nr_segs = nr_segs;
530 i->iov_offset = 0;
531 i->count = count;
532 }
533 EXPORT_SYMBOL(iov_iter_bvec);
534
535 unsigned long iov_iter_alignment(const struct iov_iter *i)
536 {
537 unsigned long res = 0;
538 size_t size = i->count;
539
540 if (!size)
541 return 0;
542
543 iterate_all_kinds(i, size, v,
544 (res |= (unsigned long)v.iov_base | v.iov_len, 0),
545 res |= v.bv_offset | v.bv_len,
546 res |= (unsigned long)v.iov_base | v.iov_len
547 )
548 return res;
549 }
550 EXPORT_SYMBOL(iov_iter_alignment);
551
552 unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
553 {
554 unsigned long res = 0;
555 size_t size = i->count;
556 if (!size)
557 return 0;
558
559 iterate_all_kinds(i, size, v,
560 (res |= (!res ? 0 : (unsigned long)v.iov_base) |
561 (size != v.iov_len ? size : 0), 0),
562 (res |= (!res ? 0 : (unsigned long)v.bv_offset) |
563 (size != v.bv_len ? size : 0)),
564 (res |= (!res ? 0 : (unsigned long)v.iov_base) |
565 (size != v.iov_len ? size : 0))
566 );
567 return res;
568 }
569 EXPORT_SYMBOL(iov_iter_gap_alignment);
570
571 ssize_t iov_iter_get_pages(struct iov_iter *i,
572 struct page **pages, size_t maxsize, unsigned maxpages,
573 size_t *start)
574 {
575 if (maxsize > i->count)
576 maxsize = i->count;
577
578 if (!maxsize)
579 return 0;
580
581 iterate_all_kinds(i, maxsize, v, ({
582 unsigned long addr = (unsigned long)v.iov_base;
583 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
584 int n;
585 int res;
586
587 if (len > maxpages * PAGE_SIZE)
588 len = maxpages * PAGE_SIZE;
589 addr &= ~(PAGE_SIZE - 1);
590 n = DIV_ROUND_UP(len, PAGE_SIZE);
591 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
592 if (unlikely(res < 0))
593 return res;
594 return (res == n ? len : res * PAGE_SIZE) - *start;
595 0;}),({
596 /* can't be more than PAGE_SIZE */
597 *start = v.bv_offset;
598 get_page(*pages = v.bv_page);
599 return v.bv_len;
600 }),({
601 return -EFAULT;
602 })
603 )
604 return 0;
605 }
606 EXPORT_SYMBOL(iov_iter_get_pages);
607
608 static struct page **get_pages_array(size_t n)
609 {
610 struct page **p = kmalloc(n * sizeof(struct page *), GFP_KERNEL);
611 if (!p)
612 p = vmalloc(n * sizeof(struct page *));
613 return p;
614 }
615
616 ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
617 struct page ***pages, size_t maxsize,
618 size_t *start)
619 {
620 struct page **p;
621
622 if (maxsize > i->count)
623 maxsize = i->count;
624
625 if (!maxsize)
626 return 0;
627
628 iterate_all_kinds(i, maxsize, v, ({
629 unsigned long addr = (unsigned long)v.iov_base;
630 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
631 int n;
632 int res;
633
634 addr &= ~(PAGE_SIZE - 1);
635 n = DIV_ROUND_UP(len, PAGE_SIZE);
636 p = get_pages_array(n);
637 if (!p)
638 return -ENOMEM;
639 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);
640 if (unlikely(res < 0)) {
641 kvfree(p);
642 return res;
643 }
644 *pages = p;
645 return (res == n ? len : res * PAGE_SIZE) - *start;
646 0;}),({
647 /* can't be more than PAGE_SIZE */
648 *start = v.bv_offset;
649 *pages = p = get_pages_array(1);
650 if (!p)
651 return -ENOMEM;
652 get_page(*p = v.bv_page);
653 return v.bv_len;
654 }),({
655 return -EFAULT;
656 })
657 )
658 return 0;
659 }
660 EXPORT_SYMBOL(iov_iter_get_pages_alloc);
661
662 size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,
663 struct iov_iter *i)
664 {
665 char *to = addr;
666 __wsum sum, next;
667 size_t off = 0;
668 sum = *csum;
669 iterate_and_advance(i, bytes, v, ({
670 int err = 0;
671 next = csum_and_copy_from_user(v.iov_base,
672 (to += v.iov_len) - v.iov_len,
673 v.iov_len, 0, &err);
674 if (!err) {
675 sum = csum_block_add(sum, next, off);
676 off += v.iov_len;
677 }
678 err ? v.iov_len : 0;
679 }), ({
680 char *p = kmap_atomic(v.bv_page);
681 next = csum_partial_copy_nocheck(p + v.bv_offset,
682 (to += v.bv_len) - v.bv_len,
683 v.bv_len, 0);
684 kunmap_atomic(p);
685 sum = csum_block_add(sum, next, off);
686 off += v.bv_len;
687 }),({
688 next = csum_partial_copy_nocheck(v.iov_base,
689 (to += v.iov_len) - v.iov_len,
690 v.iov_len, 0);
691 sum = csum_block_add(sum, next, off);
692 off += v.iov_len;
693 })
694 )
695 *csum = sum;
696 return bytes;
697 }
698 EXPORT_SYMBOL(csum_and_copy_from_iter);
699
700 size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum,
701 struct iov_iter *i)
702 {
703 const char *from = addr;
704 __wsum sum, next;
705 size_t off = 0;
706 sum = *csum;
707 iterate_and_advance(i, bytes, v, ({
708 int err = 0;
709 next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len,
710 v.iov_base,
711 v.iov_len, 0, &err);
712 if (!err) {
713 sum = csum_block_add(sum, next, off);
714 off += v.iov_len;
715 }
716 err ? v.iov_len : 0;
717 }), ({
718 char *p = kmap_atomic(v.bv_page);
719 next = csum_partial_copy_nocheck((from += v.bv_len) - v.bv_len,
720 p + v.bv_offset,
721 v.bv_len, 0);
722 kunmap_atomic(p);
723 sum = csum_block_add(sum, next, off);
724 off += v.bv_len;
725 }),({
726 next = csum_partial_copy_nocheck((from += v.iov_len) - v.iov_len,
727 v.iov_base,
728 v.iov_len, 0);
729 sum = csum_block_add(sum, next, off);
730 off += v.iov_len;
731 })
732 )
733 *csum = sum;
734 return bytes;
735 }
736 EXPORT_SYMBOL(csum_and_copy_to_iter);
737
738 int iov_iter_npages(const struct iov_iter *i, int maxpages)
739 {
740 size_t size = i->count;
741 int npages = 0;
742
743 if (!size)
744 return 0;
745
746 iterate_all_kinds(i, size, v, ({
747 unsigned long p = (unsigned long)v.iov_base;
748 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
749 - p / PAGE_SIZE;
750 if (npages >= maxpages)
751 return maxpages;
752 0;}),({
753 npages++;
754 if (npages >= maxpages)
755 return maxpages;
756 }),({
757 unsigned long p = (unsigned long)v.iov_base;
758 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
759 - p / PAGE_SIZE;
760 if (npages >= maxpages)
761 return maxpages;
762 })
763 )
764 return npages;
765 }
766 EXPORT_SYMBOL(iov_iter_npages);
767
768 const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags)
769 {
770 *new = *old;
771 if (new->type & ITER_BVEC)
772 return new->bvec = kmemdup(new->bvec,
773 new->nr_segs * sizeof(struct bio_vec),
774 flags);
775 else
776 /* iovec and kvec have identical layout */
777 return new->iov = kmemdup(new->iov,
778 new->nr_segs * sizeof(struct iovec),
779 flags);
780 }
781 EXPORT_SYMBOL(dup_iter);
782
783 int import_iovec(int type, const struct iovec __user * uvector,
784 unsigned nr_segs, unsigned fast_segs,
785 struct iovec **iov, struct iov_iter *i)
786 {
787 ssize_t n;
788 struct iovec *p;
789 n = rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
790 *iov, &p);
791 if (n < 0) {
792 if (p != *iov)
793 kfree(p);
794 *iov = NULL;
795 return n;
796 }
797 iov_iter_init(i, type, p, nr_segs, n);
798 *iov = p == *iov ? NULL : p;
799 return 0;
800 }
801 EXPORT_SYMBOL(import_iovec);
802
803 #ifdef CONFIG_COMPAT
804 #include <linux/compat.h>
805
806 int compat_import_iovec(int type, const struct compat_iovec __user * uvector,
807 unsigned nr_segs, unsigned fast_segs,
808 struct iovec **iov, struct iov_iter *i)
809 {
810 ssize_t n;
811 struct iovec *p;
812 n = compat_rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
813 *iov, &p);
814 if (n < 0) {
815 if (p != *iov)
816 kfree(p);
817 *iov = NULL;
818 return n;
819 }
820 iov_iter_init(i, type, p, nr_segs, n);
821 *iov = p == *iov ? NULL : p;
822 return 0;
823 }
824 #endif
825
826 int import_single_range(int rw, void __user *buf, size_t len,
827 struct iovec *iov, struct iov_iter *i)
828 {
829 if (len > MAX_RW_COUNT)
830 len = MAX_RW_COUNT;
831 if (unlikely(!access_ok(!rw, buf, len)))
832 return -EFAULT;
833
834 iov->iov_base = buf;
835 iov->iov_len = len;
836 iov_iter_init(i, rw, iov, 1, len);
837 return 0;
838 }
839 EXPORT_SYMBOL(import_single_range);