1 #include <linux/export.h>
3 #include <linux/pagemap.h>
4 #include <linux/slab.h>
5 #include <linux/vmalloc.h>
7 #define iterate_iovec(i, n, __v, __p, skip, STEP) { \
11 __v.iov_len = min(n, __p->iov_len - skip); \
12 if (likely(__v.iov_len)) { \
13 __v.iov_base = __p->iov_base + skip; \
15 __v.iov_len -= left; \
16 skip += __v.iov_len; \
21 while (unlikely(!left && n)) { \
23 __v.iov_len = min(n, __p->iov_len); \
24 if (unlikely(!__v.iov_len)) \
26 __v.iov_base = __p->iov_base; \
28 __v.iov_len -= left; \
35 #define iterate_bvec(i, n, __v, __p, skip, STEP) { \
38 __v.bv_len = min_t(size_t, n, __p->bv_len - skip); \
39 if (likely(__v.bv_len)) { \
40 __v.bv_page = __p->bv_page; \
41 __v.bv_offset = __p->bv_offset + skip; \
46 while (unlikely(n)) { \
48 __v.bv_len = min_t(size_t, n, __p->bv_len); \
49 if (unlikely(!__v.bv_len)) \
51 __v.bv_page = __p->bv_page; \
52 __v.bv_offset = __p->bv_offset; \
60 #define iterate_all_kinds(i, n, v, I, B) { \
61 size_t skip = i->iov_offset; \
62 if (unlikely(i->type & ITER_BVEC)) { \
63 const struct bio_vec *bvec; \
65 iterate_bvec(i, n, v, bvec, skip, (B)) \
67 const struct iovec *iov; \
69 iterate_iovec(i, n, v, iov, skip, (I)) \
73 static size_t copy_to_iter_iovec(void *from
, size_t bytes
, struct iov_iter
*i
)
75 size_t skip
, copy
, left
, wanted
;
76 const struct iovec
*iov
;
79 if (unlikely(bytes
> i
->count
))
88 buf
= iov
->iov_base
+ skip
;
89 copy
= min(bytes
, iov
->iov_len
- skip
);
91 left
= __copy_to_user(buf
, from
, copy
);
96 while (unlikely(!left
&& bytes
)) {
99 copy
= min(bytes
, iov
->iov_len
);
100 left
= __copy_to_user(buf
, from
, copy
);
107 if (skip
== iov
->iov_len
) {
111 i
->count
-= wanted
- bytes
;
112 i
->nr_segs
-= iov
- i
->iov
;
114 i
->iov_offset
= skip
;
115 return wanted
- bytes
;
118 static size_t copy_from_iter_iovec(void *to
, size_t bytes
, struct iov_iter
*i
)
120 size_t skip
, copy
, left
, wanted
;
121 const struct iovec
*iov
;
124 if (unlikely(bytes
> i
->count
))
127 if (unlikely(!bytes
))
132 skip
= i
->iov_offset
;
133 buf
= iov
->iov_base
+ skip
;
134 copy
= min(bytes
, iov
->iov_len
- skip
);
136 left
= __copy_from_user(to
, buf
, copy
);
141 while (unlikely(!left
&& bytes
)) {
144 copy
= min(bytes
, iov
->iov_len
);
145 left
= __copy_from_user(to
, buf
, copy
);
152 if (skip
== iov
->iov_len
) {
156 i
->count
-= wanted
- bytes
;
157 i
->nr_segs
-= iov
- i
->iov
;
159 i
->iov_offset
= skip
;
160 return wanted
- bytes
;
163 static size_t copy_page_to_iter_iovec(struct page
*page
, size_t offset
, size_t bytes
,
166 size_t skip
, copy
, left
, wanted
;
167 const struct iovec
*iov
;
171 if (unlikely(bytes
> i
->count
))
174 if (unlikely(!bytes
))
179 skip
= i
->iov_offset
;
180 buf
= iov
->iov_base
+ skip
;
181 copy
= min(bytes
, iov
->iov_len
- skip
);
183 if (!fault_in_pages_writeable(buf
, copy
)) {
184 kaddr
= kmap_atomic(page
);
185 from
= kaddr
+ offset
;
187 /* first chunk, usually the only one */
188 left
= __copy_to_user_inatomic(buf
, from
, copy
);
194 while (unlikely(!left
&& bytes
)) {
197 copy
= min(bytes
, iov
->iov_len
);
198 left
= __copy_to_user_inatomic(buf
, from
, copy
);
204 if (likely(!bytes
)) {
205 kunmap_atomic(kaddr
);
208 offset
= from
- kaddr
;
210 kunmap_atomic(kaddr
);
211 copy
= min(bytes
, iov
->iov_len
- skip
);
213 /* Too bad - revert to non-atomic kmap */
215 from
= kaddr
+ offset
;
216 left
= __copy_to_user(buf
, from
, copy
);
221 while (unlikely(!left
&& bytes
)) {
224 copy
= min(bytes
, iov
->iov_len
);
225 left
= __copy_to_user(buf
, from
, copy
);
233 if (skip
== iov
->iov_len
) {
237 i
->count
-= wanted
- bytes
;
238 i
->nr_segs
-= iov
- i
->iov
;
240 i
->iov_offset
= skip
;
241 return wanted
- bytes
;
244 static size_t copy_page_from_iter_iovec(struct page
*page
, size_t offset
, size_t bytes
,
247 size_t skip
, copy
, left
, wanted
;
248 const struct iovec
*iov
;
252 if (unlikely(bytes
> i
->count
))
255 if (unlikely(!bytes
))
260 skip
= i
->iov_offset
;
261 buf
= iov
->iov_base
+ skip
;
262 copy
= min(bytes
, iov
->iov_len
- skip
);
264 if (!fault_in_pages_readable(buf
, copy
)) {
265 kaddr
= kmap_atomic(page
);
268 /* first chunk, usually the only one */
269 left
= __copy_from_user_inatomic(to
, buf
, copy
);
275 while (unlikely(!left
&& bytes
)) {
278 copy
= min(bytes
, iov
->iov_len
);
279 left
= __copy_from_user_inatomic(to
, buf
, copy
);
285 if (likely(!bytes
)) {
286 kunmap_atomic(kaddr
);
291 kunmap_atomic(kaddr
);
292 copy
= min(bytes
, iov
->iov_len
- skip
);
294 /* Too bad - revert to non-atomic kmap */
297 left
= __copy_from_user(to
, buf
, copy
);
302 while (unlikely(!left
&& bytes
)) {
305 copy
= min(bytes
, iov
->iov_len
);
306 left
= __copy_from_user(to
, buf
, copy
);
314 if (skip
== iov
->iov_len
) {
318 i
->count
-= wanted
- bytes
;
319 i
->nr_segs
-= iov
- i
->iov
;
321 i
->iov_offset
= skip
;
322 return wanted
- bytes
;
325 static size_t zero_iovec(size_t bytes
, struct iov_iter
*i
)
327 size_t skip
, copy
, left
, wanted
;
328 const struct iovec
*iov
;
331 if (unlikely(bytes
> i
->count
))
334 if (unlikely(!bytes
))
339 skip
= i
->iov_offset
;
340 buf
= iov
->iov_base
+ skip
;
341 copy
= min(bytes
, iov
->iov_len
- skip
);
343 left
= __clear_user(buf
, copy
);
348 while (unlikely(!left
&& bytes
)) {
351 copy
= min(bytes
, iov
->iov_len
);
352 left
= __clear_user(buf
, copy
);
358 if (skip
== iov
->iov_len
) {
362 i
->count
-= wanted
- bytes
;
363 i
->nr_segs
-= iov
- i
->iov
;
365 i
->iov_offset
= skip
;
366 return wanted
- bytes
;
369 static void advance_iovec(struct iov_iter
*i
, size_t bytes
)
371 BUG_ON(i
->count
< bytes
);
373 if (likely(i
->nr_segs
== 1)) {
374 i
->iov_offset
+= bytes
;
377 const struct iovec
*iov
= i
->iov
;
378 size_t base
= i
->iov_offset
;
379 unsigned long nr_segs
= i
->nr_segs
;
382 * The !iov->iov_len check ensures we skip over unlikely
383 * zero-length segments (without overruning the iovec).
385 while (bytes
|| unlikely(i
->count
&& !iov
->iov_len
)) {
388 copy
= min(bytes
, iov
->iov_len
- base
);
389 BUG_ON(!i
->count
|| i
->count
< copy
);
393 if (iov
->iov_len
== base
) {
400 i
->iov_offset
= base
;
401 i
->nr_segs
= nr_segs
;
406 * Fault in the first iovec of the given iov_iter, to a maximum length
407 * of bytes. Returns 0 on success, or non-zero if the memory could not be
408 * accessed (ie. because it is an invalid address).
410 * writev-intensive code may want this to prefault several iovecs -- that
411 * would be possible (callers must not rely on the fact that _only_ the
412 * first iovec will be faulted with the current implementation).
414 int iov_iter_fault_in_readable(struct iov_iter
*i
, size_t bytes
)
416 if (!(i
->type
& ITER_BVEC
)) {
417 char __user
*buf
= i
->iov
->iov_base
+ i
->iov_offset
;
418 bytes
= min(bytes
, i
->iov
->iov_len
- i
->iov_offset
);
419 return fault_in_pages_readable(buf
, bytes
);
423 EXPORT_SYMBOL(iov_iter_fault_in_readable
);
425 void iov_iter_init(struct iov_iter
*i
, int direction
,
426 const struct iovec
*iov
, unsigned long nr_segs
,
429 /* It will get better. Eventually... */
430 if (segment_eq(get_fs(), KERNEL_DS
))
431 direction
|= ITER_KVEC
;
434 i
->nr_segs
= nr_segs
;
438 EXPORT_SYMBOL(iov_iter_init
);
440 static ssize_t
get_pages_iovec(struct iov_iter
*i
,
441 struct page
**pages
, size_t maxsize
, unsigned maxpages
,
444 size_t offset
= i
->iov_offset
;
445 const struct iovec
*iov
= i
->iov
;
451 len
= iov
->iov_len
- offset
;
456 addr
= (unsigned long)iov
->iov_base
+ offset
;
457 len
+= *start
= addr
& (PAGE_SIZE
- 1);
458 if (len
> maxpages
* PAGE_SIZE
)
459 len
= maxpages
* PAGE_SIZE
;
460 addr
&= ~(PAGE_SIZE
- 1);
461 n
= (len
+ PAGE_SIZE
- 1) / PAGE_SIZE
;
462 res
= get_user_pages_fast(addr
, n
, (i
->type
& WRITE
) != WRITE
, pages
);
463 if (unlikely(res
< 0))
465 return (res
== n
? len
: res
* PAGE_SIZE
) - *start
;
468 static ssize_t
get_pages_alloc_iovec(struct iov_iter
*i
,
469 struct page
***pages
, size_t maxsize
,
472 size_t offset
= i
->iov_offset
;
473 const struct iovec
*iov
= i
->iov
;
480 len
= iov
->iov_len
- offset
;
485 addr
= (unsigned long)iov
->iov_base
+ offset
;
486 len
+= *start
= addr
& (PAGE_SIZE
- 1);
487 addr
&= ~(PAGE_SIZE
- 1);
488 n
= (len
+ PAGE_SIZE
- 1) / PAGE_SIZE
;
490 p
= kmalloc(n
* sizeof(struct page
*), GFP_KERNEL
);
492 p
= vmalloc(n
* sizeof(struct page
*));
496 res
= get_user_pages_fast(addr
, n
, (i
->type
& WRITE
) != WRITE
, p
);
497 if (unlikely(res
< 0)) {
502 return (res
== n
? len
: res
* PAGE_SIZE
) - *start
;
505 static int iov_iter_npages_iovec(const struct iov_iter
*i
, int maxpages
)
507 size_t offset
= i
->iov_offset
;
508 size_t size
= i
->count
;
509 const struct iovec
*iov
= i
->iov
;
513 for (n
= 0; size
&& n
< i
->nr_segs
; n
++, iov
++) {
514 unsigned long addr
= (unsigned long)iov
->iov_base
+ offset
;
515 size_t len
= iov
->iov_len
- offset
;
517 if (unlikely(!len
)) /* empty segment */
521 npages
+= (addr
+ len
+ PAGE_SIZE
- 1) / PAGE_SIZE
523 if (npages
>= maxpages
) /* don't bother going further */
528 return min(npages
, maxpages
);
531 static void memcpy_from_page(char *to
, struct page
*page
, size_t offset
, size_t len
)
533 char *from
= kmap_atomic(page
);
534 memcpy(to
, from
+ offset
, len
);
538 static void memcpy_to_page(struct page
*page
, size_t offset
, char *from
, size_t len
)
540 char *to
= kmap_atomic(page
);
541 memcpy(to
+ offset
, from
, len
);
545 static void memzero_page(struct page
*page
, size_t offset
, size_t len
)
547 char *addr
= kmap_atomic(page
);
548 memset(addr
+ offset
, 0, len
);
552 static size_t copy_to_iter_bvec(void *from
, size_t bytes
, struct iov_iter
*i
)
554 size_t skip
, copy
, wanted
;
555 const struct bio_vec
*bvec
;
557 if (unlikely(bytes
> i
->count
))
560 if (unlikely(!bytes
))
565 skip
= i
->iov_offset
;
566 copy
= min_t(size_t, bytes
, bvec
->bv_len
- skip
);
568 memcpy_to_page(bvec
->bv_page
, skip
+ bvec
->bv_offset
, from
, copy
);
574 copy
= min(bytes
, (size_t)bvec
->bv_len
);
575 memcpy_to_page(bvec
->bv_page
, bvec
->bv_offset
, from
, copy
);
580 if (skip
== bvec
->bv_len
) {
584 i
->count
-= wanted
- bytes
;
585 i
->nr_segs
-= bvec
- i
->bvec
;
587 i
->iov_offset
= skip
;
588 return wanted
- bytes
;
591 static size_t copy_from_iter_bvec(void *to
, size_t bytes
, struct iov_iter
*i
)
593 size_t skip
, copy
, wanted
;
594 const struct bio_vec
*bvec
;
596 if (unlikely(bytes
> i
->count
))
599 if (unlikely(!bytes
))
604 skip
= i
->iov_offset
;
606 copy
= min(bytes
, bvec
->bv_len
- skip
);
608 memcpy_from_page(to
, bvec
->bv_page
, bvec
->bv_offset
+ skip
, copy
);
616 copy
= min(bytes
, (size_t)bvec
->bv_len
);
617 memcpy_from_page(to
, bvec
->bv_page
, bvec
->bv_offset
, copy
);
622 if (skip
== bvec
->bv_len
) {
627 i
->nr_segs
-= bvec
- i
->bvec
;
629 i
->iov_offset
= skip
;
633 static size_t copy_page_to_iter_bvec(struct page
*page
, size_t offset
,
634 size_t bytes
, struct iov_iter
*i
)
636 void *kaddr
= kmap_atomic(page
);
637 size_t wanted
= copy_to_iter_bvec(kaddr
+ offset
, bytes
, i
);
638 kunmap_atomic(kaddr
);
642 static size_t copy_page_from_iter_bvec(struct page
*page
, size_t offset
,
643 size_t bytes
, struct iov_iter
*i
)
645 void *kaddr
= kmap_atomic(page
);
646 size_t wanted
= copy_from_iter_bvec(kaddr
+ offset
, bytes
, i
);
647 kunmap_atomic(kaddr
);
651 static size_t zero_bvec(size_t bytes
, struct iov_iter
*i
)
653 size_t skip
, copy
, wanted
;
654 const struct bio_vec
*bvec
;
656 if (unlikely(bytes
> i
->count
))
659 if (unlikely(!bytes
))
664 skip
= i
->iov_offset
;
665 copy
= min_t(size_t, bytes
, bvec
->bv_len
- skip
);
667 memzero_page(bvec
->bv_page
, skip
+ bvec
->bv_offset
, copy
);
672 copy
= min(bytes
, (size_t)bvec
->bv_len
);
673 memzero_page(bvec
->bv_page
, bvec
->bv_offset
, copy
);
677 if (skip
== bvec
->bv_len
) {
681 i
->count
-= wanted
- bytes
;
682 i
->nr_segs
-= bvec
- i
->bvec
;
684 i
->iov_offset
= skip
;
685 return wanted
- bytes
;
688 static void advance_bvec(struct iov_iter
*i
, size_t bytes
)
690 BUG_ON(i
->count
< bytes
);
692 if (likely(i
->nr_segs
== 1)) {
693 i
->iov_offset
+= bytes
;
696 const struct bio_vec
*bvec
= i
->bvec
;
697 size_t base
= i
->iov_offset
;
698 unsigned long nr_segs
= i
->nr_segs
;
701 * The !iov->iov_len check ensures we skip over unlikely
702 * zero-length segments (without overruning the iovec).
704 while (bytes
|| unlikely(i
->count
&& !bvec
->bv_len
)) {
707 copy
= min(bytes
, bvec
->bv_len
- base
);
708 BUG_ON(!i
->count
|| i
->count
< copy
);
712 if (bvec
->bv_len
== base
) {
719 i
->iov_offset
= base
;
720 i
->nr_segs
= nr_segs
;
724 static ssize_t
get_pages_bvec(struct iov_iter
*i
,
725 struct page
**pages
, size_t maxsize
, unsigned maxpages
,
728 const struct bio_vec
*bvec
= i
->bvec
;
729 size_t len
= bvec
->bv_len
- i
->iov_offset
;
734 /* can't be more than PAGE_SIZE */
735 *start
= bvec
->bv_offset
+ i
->iov_offset
;
737 get_page(*pages
= bvec
->bv_page
);
742 static ssize_t
get_pages_alloc_bvec(struct iov_iter
*i
,
743 struct page
***pages
, size_t maxsize
,
746 const struct bio_vec
*bvec
= i
->bvec
;
747 size_t len
= bvec
->bv_len
- i
->iov_offset
;
752 *start
= bvec
->bv_offset
+ i
->iov_offset
;
754 *pages
= kmalloc(sizeof(struct page
*), GFP_KERNEL
);
758 get_page(**pages
= bvec
->bv_page
);
763 static int iov_iter_npages_bvec(const struct iov_iter
*i
, int maxpages
)
765 size_t offset
= i
->iov_offset
;
766 size_t size
= i
->count
;
767 const struct bio_vec
*bvec
= i
->bvec
;
771 for (n
= 0; size
&& n
< i
->nr_segs
; n
++, bvec
++) {
772 size_t len
= bvec
->bv_len
- offset
;
774 if (unlikely(!len
)) /* empty segment */
779 if (npages
>= maxpages
) /* don't bother going further */
784 return min(npages
, maxpages
);
787 size_t copy_page_to_iter(struct page
*page
, size_t offset
, size_t bytes
,
790 if (i
->type
& ITER_BVEC
)
791 return copy_page_to_iter_bvec(page
, offset
, bytes
, i
);
793 return copy_page_to_iter_iovec(page
, offset
, bytes
, i
);
795 EXPORT_SYMBOL(copy_page_to_iter
);
797 size_t copy_page_from_iter(struct page
*page
, size_t offset
, size_t bytes
,
800 if (i
->type
& ITER_BVEC
)
801 return copy_page_from_iter_bvec(page
, offset
, bytes
, i
);
803 return copy_page_from_iter_iovec(page
, offset
, bytes
, i
);
805 EXPORT_SYMBOL(copy_page_from_iter
);
807 size_t copy_to_iter(void *addr
, size_t bytes
, struct iov_iter
*i
)
809 if (i
->type
& ITER_BVEC
)
810 return copy_to_iter_bvec(addr
, bytes
, i
);
812 return copy_to_iter_iovec(addr
, bytes
, i
);
814 EXPORT_SYMBOL(copy_to_iter
);
816 size_t copy_from_iter(void *addr
, size_t bytes
, struct iov_iter
*i
)
818 if (i
->type
& ITER_BVEC
)
819 return copy_from_iter_bvec(addr
, bytes
, i
);
821 return copy_from_iter_iovec(addr
, bytes
, i
);
823 EXPORT_SYMBOL(copy_from_iter
);
825 size_t iov_iter_zero(size_t bytes
, struct iov_iter
*i
)
827 if (i
->type
& ITER_BVEC
) {
828 return zero_bvec(bytes
, i
);
830 return zero_iovec(bytes
, i
);
833 EXPORT_SYMBOL(iov_iter_zero
);
835 size_t iov_iter_copy_from_user_atomic(struct page
*page
,
836 struct iov_iter
*i
, unsigned long offset
, size_t bytes
)
838 char *kaddr
= kmap_atomic(page
), *p
= kaddr
+ offset
;
839 iterate_all_kinds(i
, bytes
, v
,
840 __copy_from_user_inatomic((p
+= v
.iov_len
) - v
.iov_len
,
841 v
.iov_base
, v
.iov_len
),
842 memcpy_from_page((p
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
843 v
.bv_offset
, v
.bv_len
)
845 kunmap_atomic(kaddr
);
848 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic
);
850 void iov_iter_advance(struct iov_iter
*i
, size_t size
)
852 if (i
->type
& ITER_BVEC
)
853 advance_bvec(i
, size
);
855 advance_iovec(i
, size
);
857 EXPORT_SYMBOL(iov_iter_advance
);
860 * Return the count of just the current iov_iter segment.
862 size_t iov_iter_single_seg_count(const struct iov_iter
*i
)
866 else if (i
->type
& ITER_BVEC
)
867 return min(i
->count
, i
->bvec
->bv_len
- i
->iov_offset
);
869 return min(i
->count
, i
->iov
->iov_len
- i
->iov_offset
);
871 EXPORT_SYMBOL(iov_iter_single_seg_count
);
873 unsigned long iov_iter_alignment(const struct iov_iter
*i
)
875 unsigned long res
= 0;
876 size_t size
= i
->count
;
881 iterate_all_kinds(i
, size
, v
,
882 (res
|= (unsigned long)v
.iov_base
| v
.iov_len
, 0),
883 res
|= v
.bv_offset
| v
.bv_len
887 EXPORT_SYMBOL(iov_iter_alignment
);
889 ssize_t
iov_iter_get_pages(struct iov_iter
*i
,
890 struct page
**pages
, size_t maxsize
, unsigned maxpages
,
893 if (i
->type
& ITER_BVEC
)
894 return get_pages_bvec(i
, pages
, maxsize
, maxpages
, start
);
896 return get_pages_iovec(i
, pages
, maxsize
, maxpages
, start
);
898 EXPORT_SYMBOL(iov_iter_get_pages
);
900 ssize_t
iov_iter_get_pages_alloc(struct iov_iter
*i
,
901 struct page
***pages
, size_t maxsize
,
904 if (i
->type
& ITER_BVEC
)
905 return get_pages_alloc_bvec(i
, pages
, maxsize
, start
);
907 return get_pages_alloc_iovec(i
, pages
, maxsize
, start
);
909 EXPORT_SYMBOL(iov_iter_get_pages_alloc
);
911 int iov_iter_npages(const struct iov_iter
*i
, int maxpages
)
913 if (i
->type
& ITER_BVEC
)
914 return iov_iter_npages_bvec(i
, maxpages
);
916 return iov_iter_npages_iovec(i
, maxpages
);
918 EXPORT_SYMBOL(iov_iter_npages
);