1 #include <linux/export.h>
2 #include <linux/bvec.h>
4 #include <linux/pagemap.h>
5 #include <linux/slab.h>
6 #include <linux/vmalloc.h>
7 #include <linux/splice.h>
8 #include <net/checksum.h>
10 #define PIPE_PARANOIA /* for now */
12 #define iterate_iovec(i, n, __v, __p, skip, STEP) { \
16 __v.iov_len = min(n, __p->iov_len - skip); \
17 if (likely(__v.iov_len)) { \
18 __v.iov_base = __p->iov_base + skip; \
20 __v.iov_len -= left; \
21 skip += __v.iov_len; \
26 while (unlikely(!left && n)) { \
28 __v.iov_len = min(n, __p->iov_len); \
29 if (unlikely(!__v.iov_len)) \
31 __v.iov_base = __p->iov_base; \
33 __v.iov_len -= left; \
40 #define iterate_kvec(i, n, __v, __p, skip, STEP) { \
43 __v.iov_len = min(n, __p->iov_len - skip); \
44 if (likely(__v.iov_len)) { \
45 __v.iov_base = __p->iov_base + skip; \
47 skip += __v.iov_len; \
50 while (unlikely(n)) { \
52 __v.iov_len = min(n, __p->iov_len); \
53 if (unlikely(!__v.iov_len)) \
55 __v.iov_base = __p->iov_base; \
63 #define iterate_bvec(i, n, __v, __bi, skip, STEP) { \
64 struct bvec_iter __start; \
65 __start.bi_size = n; \
66 __start.bi_bvec_done = skip; \
68 for_each_bvec(__v, i->bvec, __bi, __start) { \
75 #define iterate_all_kinds(i, n, v, I, B, K) { \
77 size_t skip = i->iov_offset; \
78 if (unlikely(i->type & ITER_BVEC)) { \
80 struct bvec_iter __bi; \
81 iterate_bvec(i, n, v, __bi, skip, (B)) \
82 } else if (unlikely(i->type & ITER_KVEC)) { \
83 const struct kvec *kvec; \
85 iterate_kvec(i, n, v, kvec, skip, (K)) \
86 } else if (unlikely(i->type & ITER_DISCARD)) { \
88 const struct iovec *iov; \
90 iterate_iovec(i, n, v, iov, skip, (I)) \
95 #define iterate_and_advance(i, n, v, I, B, K) { \
96 if (unlikely(i->count < n)) \
99 size_t skip = i->iov_offset; \
100 if (unlikely(i->type & ITER_BVEC)) { \
101 const struct bio_vec *bvec = i->bvec; \
103 struct bvec_iter __bi; \
104 iterate_bvec(i, n, v, __bi, skip, (B)) \
105 i->bvec = __bvec_iter_bvec(i->bvec, __bi); \
106 i->nr_segs -= i->bvec - bvec; \
107 skip = __bi.bi_bvec_done; \
108 } else if (unlikely(i->type & ITER_KVEC)) { \
109 const struct kvec *kvec; \
111 iterate_kvec(i, n, v, kvec, skip, (K)) \
112 if (skip == kvec->iov_len) { \
116 i->nr_segs -= kvec - i->kvec; \
118 } else if (unlikely(i->type & ITER_DISCARD)) { \
121 const struct iovec *iov; \
123 iterate_iovec(i, n, v, iov, skip, (I)) \
124 if (skip == iov->iov_len) { \
128 i->nr_segs -= iov - i->iov; \
132 i->iov_offset = skip; \
136 static int copyout(void __user
*to
, const void *from
, size_t n
)
138 if (access_ok(VERIFY_WRITE
, to
, n
)) {
139 kasan_check_read(from
, n
);
140 n
= raw_copy_to_user(to
, from
, n
);
145 static int copyin(void *to
, const void __user
*from
, size_t n
)
147 if (access_ok(VERIFY_READ
, from
, n
)) {
148 kasan_check_write(to
, n
);
149 n
= raw_copy_from_user(to
, from
, n
);
154 static size_t copy_page_to_iter_iovec(struct page
*page
, size_t offset
, size_t bytes
,
157 size_t skip
, copy
, left
, wanted
;
158 const struct iovec
*iov
;
162 if (unlikely(bytes
> i
->count
))
165 if (unlikely(!bytes
))
171 skip
= i
->iov_offset
;
172 buf
= iov
->iov_base
+ skip
;
173 copy
= min(bytes
, iov
->iov_len
- skip
);
175 if (IS_ENABLED(CONFIG_HIGHMEM
) && !fault_in_pages_writeable(buf
, copy
)) {
176 kaddr
= kmap_atomic(page
);
177 from
= kaddr
+ offset
;
179 /* first chunk, usually the only one */
180 left
= copyout(buf
, from
, copy
);
186 while (unlikely(!left
&& bytes
)) {
189 copy
= min(bytes
, iov
->iov_len
);
190 left
= copyout(buf
, from
, copy
);
196 if (likely(!bytes
)) {
197 kunmap_atomic(kaddr
);
200 offset
= from
- kaddr
;
202 kunmap_atomic(kaddr
);
203 copy
= min(bytes
, iov
->iov_len
- skip
);
205 /* Too bad - revert to non-atomic kmap */
208 from
= kaddr
+ offset
;
209 left
= copyout(buf
, from
, copy
);
214 while (unlikely(!left
&& bytes
)) {
217 copy
= min(bytes
, iov
->iov_len
);
218 left
= copyout(buf
, from
, copy
);
227 if (skip
== iov
->iov_len
) {
231 i
->count
-= wanted
- bytes
;
232 i
->nr_segs
-= iov
- i
->iov
;
234 i
->iov_offset
= skip
;
235 return wanted
- bytes
;
238 static size_t copy_page_from_iter_iovec(struct page
*page
, size_t offset
, size_t bytes
,
241 size_t skip
, copy
, left
, wanted
;
242 const struct iovec
*iov
;
246 if (unlikely(bytes
> i
->count
))
249 if (unlikely(!bytes
))
255 skip
= i
->iov_offset
;
256 buf
= iov
->iov_base
+ skip
;
257 copy
= min(bytes
, iov
->iov_len
- skip
);
259 if (IS_ENABLED(CONFIG_HIGHMEM
) && !fault_in_pages_readable(buf
, copy
)) {
260 kaddr
= kmap_atomic(page
);
263 /* first chunk, usually the only one */
264 left
= copyin(to
, buf
, copy
);
270 while (unlikely(!left
&& bytes
)) {
273 copy
= min(bytes
, iov
->iov_len
);
274 left
= copyin(to
, buf
, copy
);
280 if (likely(!bytes
)) {
281 kunmap_atomic(kaddr
);
286 kunmap_atomic(kaddr
);
287 copy
= min(bytes
, iov
->iov_len
- skip
);
289 /* Too bad - revert to non-atomic kmap */
293 left
= copyin(to
, buf
, copy
);
298 while (unlikely(!left
&& bytes
)) {
301 copy
= min(bytes
, iov
->iov_len
);
302 left
= copyin(to
, buf
, copy
);
311 if (skip
== iov
->iov_len
) {
315 i
->count
-= wanted
- bytes
;
316 i
->nr_segs
-= iov
- i
->iov
;
318 i
->iov_offset
= skip
;
319 return wanted
- bytes
;
323 static bool sanity(const struct iov_iter
*i
)
325 struct pipe_inode_info
*pipe
= i
->pipe
;
327 int next
= pipe
->curbuf
+ pipe
->nrbufs
;
329 struct pipe_buffer
*p
;
330 if (unlikely(!pipe
->nrbufs
))
331 goto Bad
; // pipe must be non-empty
332 if (unlikely(idx
!= ((next
- 1) & (pipe
->buffers
- 1))))
333 goto Bad
; // must be at the last buffer...
335 p
= &pipe
->bufs
[idx
];
336 if (unlikely(p
->offset
+ p
->len
!= i
->iov_offset
))
337 goto Bad
; // ... at the end of segment
339 if (idx
!= (next
& (pipe
->buffers
- 1)))
340 goto Bad
; // must be right after the last buffer
344 printk(KERN_ERR
"idx = %d, offset = %zd\n", i
->idx
, i
->iov_offset
);
345 printk(KERN_ERR
"curbuf = %d, nrbufs = %d, buffers = %d\n",
346 pipe
->curbuf
, pipe
->nrbufs
, pipe
->buffers
);
347 for (idx
= 0; idx
< pipe
->buffers
; idx
++)
348 printk(KERN_ERR
"[%p %p %d %d]\n",
350 pipe
->bufs
[idx
].page
,
351 pipe
->bufs
[idx
].offset
,
352 pipe
->bufs
[idx
].len
);
357 #define sanity(i) true
360 static inline int next_idx(int idx
, struct pipe_inode_info
*pipe
)
362 return (idx
+ 1) & (pipe
->buffers
- 1);
365 static size_t copy_page_to_iter_pipe(struct page
*page
, size_t offset
, size_t bytes
,
368 struct pipe_inode_info
*pipe
= i
->pipe
;
369 struct pipe_buffer
*buf
;
373 if (unlikely(bytes
> i
->count
))
376 if (unlikely(!bytes
))
384 buf
= &pipe
->bufs
[idx
];
386 if (offset
== off
&& buf
->page
== page
) {
387 /* merge with the last one */
389 i
->iov_offset
+= bytes
;
392 idx
= next_idx(idx
, pipe
);
393 buf
= &pipe
->bufs
[idx
];
395 if (idx
== pipe
->curbuf
&& pipe
->nrbufs
)
398 buf
->ops
= &page_cache_pipe_buf_ops
;
399 get_page(buf
->page
= page
);
400 buf
->offset
= offset
;
402 i
->iov_offset
= offset
+ bytes
;
410 * Fault in one or more iovecs of the given iov_iter, to a maximum length of
411 * bytes. For each iovec, fault in each page that constitutes the iovec.
413 * Return 0 on success, or non-zero if the memory could not be accessed (i.e.
414 * because it is an invalid address).
416 int iov_iter_fault_in_readable(struct iov_iter
*i
, size_t bytes
)
418 size_t skip
= i
->iov_offset
;
419 const struct iovec
*iov
;
423 if (!(i
->type
& (ITER_BVEC
|ITER_KVEC
))) {
424 iterate_iovec(i
, bytes
, v
, iov
, skip
, ({
425 err
= fault_in_pages_readable(v
.iov_base
, v
.iov_len
);
432 EXPORT_SYMBOL(iov_iter_fault_in_readable
);
434 void iov_iter_init(struct iov_iter
*i
, unsigned int direction
,
435 const struct iovec
*iov
, unsigned long nr_segs
,
438 WARN_ON(direction
& ~(READ
| WRITE
));
439 direction
&= READ
| WRITE
;
441 /* It will get better. Eventually... */
442 if (uaccess_kernel()) {
443 i
->type
= ITER_KVEC
| direction
;
444 i
->kvec
= (struct kvec
*)iov
;
446 i
->type
= ITER_IOVEC
| direction
;
449 i
->nr_segs
= nr_segs
;
453 EXPORT_SYMBOL(iov_iter_init
);
455 static void memcpy_from_page(char *to
, struct page
*page
, size_t offset
, size_t len
)
457 char *from
= kmap_atomic(page
);
458 memcpy(to
, from
+ offset
, len
);
462 static void memcpy_to_page(struct page
*page
, size_t offset
, const char *from
, size_t len
)
464 char *to
= kmap_atomic(page
);
465 memcpy(to
+ offset
, from
, len
);
469 static void memzero_page(struct page
*page
, size_t offset
, size_t len
)
471 char *addr
= kmap_atomic(page
);
472 memset(addr
+ offset
, 0, len
);
476 static inline bool allocated(struct pipe_buffer
*buf
)
478 return buf
->ops
== &default_pipe_buf_ops
;
481 static inline void data_start(const struct iov_iter
*i
, int *idxp
, size_t *offp
)
483 size_t off
= i
->iov_offset
;
485 if (off
&& (!allocated(&i
->pipe
->bufs
[idx
]) || off
== PAGE_SIZE
)) {
486 idx
= next_idx(idx
, i
->pipe
);
493 static size_t push_pipe(struct iov_iter
*i
, size_t size
,
494 int *idxp
, size_t *offp
)
496 struct pipe_inode_info
*pipe
= i
->pipe
;
501 if (unlikely(size
> i
->count
))
507 data_start(i
, &idx
, &off
);
511 left
-= PAGE_SIZE
- off
;
513 pipe
->bufs
[idx
].len
+= size
;
516 pipe
->bufs
[idx
].len
= PAGE_SIZE
;
517 idx
= next_idx(idx
, pipe
);
519 while (idx
!= pipe
->curbuf
|| !pipe
->nrbufs
) {
520 struct page
*page
= alloc_page(GFP_USER
);
524 pipe
->bufs
[idx
].ops
= &default_pipe_buf_ops
;
525 pipe
->bufs
[idx
].page
= page
;
526 pipe
->bufs
[idx
].offset
= 0;
527 if (left
<= PAGE_SIZE
) {
528 pipe
->bufs
[idx
].len
= left
;
531 pipe
->bufs
[idx
].len
= PAGE_SIZE
;
533 idx
= next_idx(idx
, pipe
);
538 static size_t copy_pipe_to_iter(const void *addr
, size_t bytes
,
541 struct pipe_inode_info
*pipe
= i
->pipe
;
548 bytes
= n
= push_pipe(i
, bytes
, &idx
, &off
);
551 for ( ; n
; idx
= next_idx(idx
, pipe
), off
= 0) {
552 size_t chunk
= min_t(size_t, n
, PAGE_SIZE
- off
);
553 memcpy_to_page(pipe
->bufs
[idx
].page
, off
, addr
, chunk
);
555 i
->iov_offset
= off
+ chunk
;
563 static size_t csum_and_copy_to_pipe_iter(const void *addr
, size_t bytes
,
564 __wsum
*csum
, struct iov_iter
*i
)
566 struct pipe_inode_info
*pipe
= i
->pipe
;
569 __wsum sum
= *csum
, next
;
575 bytes
= n
= push_pipe(i
, bytes
, &idx
, &r
);
578 for ( ; n
; idx
= next_idx(idx
, pipe
), r
= 0) {
579 size_t chunk
= min_t(size_t, n
, PAGE_SIZE
- r
);
580 char *p
= kmap_atomic(pipe
->bufs
[idx
].page
);
581 next
= csum_partial_copy_nocheck(addr
, p
+ r
, chunk
, 0);
582 sum
= csum_block_add(sum
, next
, off
);
585 i
->iov_offset
= r
+ chunk
;
595 size_t _copy_to_iter(const void *addr
, size_t bytes
, struct iov_iter
*i
)
597 const char *from
= addr
;
598 if (unlikely(iov_iter_is_pipe(i
)))
599 return copy_pipe_to_iter(addr
, bytes
, i
);
600 if (iter_is_iovec(i
))
602 iterate_and_advance(i
, bytes
, v
,
603 copyout(v
.iov_base
, (from
+= v
.iov_len
) - v
.iov_len
, v
.iov_len
),
604 memcpy_to_page(v
.bv_page
, v
.bv_offset
,
605 (from
+= v
.bv_len
) - v
.bv_len
, v
.bv_len
),
606 memcpy(v
.iov_base
, (from
+= v
.iov_len
) - v
.iov_len
, v
.iov_len
)
611 EXPORT_SYMBOL(_copy_to_iter
);
613 #ifdef CONFIG_ARCH_HAS_UACCESS_MCSAFE
614 static int copyout_mcsafe(void __user
*to
, const void *from
, size_t n
)
616 if (access_ok(VERIFY_WRITE
, to
, n
)) {
617 kasan_check_read(from
, n
);
618 n
= copy_to_user_mcsafe((__force
void *) to
, from
, n
);
623 static unsigned long memcpy_mcsafe_to_page(struct page
*page
, size_t offset
,
624 const char *from
, size_t len
)
629 to
= kmap_atomic(page
);
630 ret
= memcpy_mcsafe(to
+ offset
, from
, len
);
636 static size_t copy_pipe_to_iter_mcsafe(const void *addr
, size_t bytes
,
639 struct pipe_inode_info
*pipe
= i
->pipe
;
640 size_t n
, off
, xfer
= 0;
646 bytes
= n
= push_pipe(i
, bytes
, &idx
, &off
);
649 for ( ; n
; idx
= next_idx(idx
, pipe
), off
= 0) {
650 size_t chunk
= min_t(size_t, n
, PAGE_SIZE
- off
);
653 rem
= memcpy_mcsafe_to_page(pipe
->bufs
[idx
].page
, off
, addr
,
656 i
->iov_offset
= off
+ chunk
- rem
;
668 * _copy_to_iter_mcsafe - copy to user with source-read error exception handling
669 * @addr: source kernel address
670 * @bytes: total transfer length
671 * @iter: destination iterator
673 * The pmem driver arranges for filesystem-dax to use this facility via
674 * dax_copy_to_iter() for protecting read/write to persistent memory.
675 * Unless / until an architecture can guarantee identical performance
676 * between _copy_to_iter_mcsafe() and _copy_to_iter() it would be a
677 * performance regression to switch more users to the mcsafe version.
679 * Otherwise, the main differences between this and typical _copy_to_iter().
681 * * Typical tail/residue handling after a fault retries the copy
682 * byte-by-byte until the fault happens again. Re-triggering machine
683 * checks is potentially fatal so the implementation uses source
684 * alignment and poison alignment assumptions to avoid re-triggering
685 * hardware exceptions.
687 * * ITER_KVEC, ITER_PIPE, and ITER_BVEC can return short copies.
688 * Compare to copy_to_iter() where only ITER_IOVEC attempts might return
691 * See MCSAFE_TEST for self-test.
693 size_t _copy_to_iter_mcsafe(const void *addr
, size_t bytes
, struct iov_iter
*i
)
695 const char *from
= addr
;
696 unsigned long rem
, curr_addr
, s_addr
= (unsigned long) addr
;
698 if (unlikely(iov_iter_is_pipe(i
)))
699 return copy_pipe_to_iter_mcsafe(addr
, bytes
, i
);
700 if (iter_is_iovec(i
))
702 iterate_and_advance(i
, bytes
, v
,
703 copyout_mcsafe(v
.iov_base
, (from
+= v
.iov_len
) - v
.iov_len
, v
.iov_len
),
705 rem
= memcpy_mcsafe_to_page(v
.bv_page
, v
.bv_offset
,
706 (from
+= v
.bv_len
) - v
.bv_len
, v
.bv_len
);
708 curr_addr
= (unsigned long) from
;
709 bytes
= curr_addr
- s_addr
- rem
;
714 rem
= memcpy_mcsafe(v
.iov_base
, (from
+= v
.iov_len
) - v
.iov_len
,
717 curr_addr
= (unsigned long) from
;
718 bytes
= curr_addr
- s_addr
- rem
;
726 EXPORT_SYMBOL_GPL(_copy_to_iter_mcsafe
);
727 #endif /* CONFIG_ARCH_HAS_UACCESS_MCSAFE */
729 size_t _copy_from_iter(void *addr
, size_t bytes
, struct iov_iter
*i
)
732 if (unlikely(iov_iter_is_pipe(i
))) {
736 if (iter_is_iovec(i
))
738 iterate_and_advance(i
, bytes
, v
,
739 copyin((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
),
740 memcpy_from_page((to
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
741 v
.bv_offset
, v
.bv_len
),
742 memcpy((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
)
747 EXPORT_SYMBOL(_copy_from_iter
);
749 bool _copy_from_iter_full(void *addr
, size_t bytes
, struct iov_iter
*i
)
752 if (unlikely(iov_iter_is_pipe(i
))) {
756 if (unlikely(i
->count
< bytes
))
759 if (iter_is_iovec(i
))
761 iterate_all_kinds(i
, bytes
, v
, ({
762 if (copyin((to
+= v
.iov_len
) - v
.iov_len
,
763 v
.iov_base
, v
.iov_len
))
766 memcpy_from_page((to
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
767 v
.bv_offset
, v
.bv_len
),
768 memcpy((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
)
771 iov_iter_advance(i
, bytes
);
774 EXPORT_SYMBOL(_copy_from_iter_full
);
776 size_t _copy_from_iter_nocache(void *addr
, size_t bytes
, struct iov_iter
*i
)
779 if (unlikely(iov_iter_is_pipe(i
))) {
783 iterate_and_advance(i
, bytes
, v
,
784 __copy_from_user_inatomic_nocache((to
+= v
.iov_len
) - v
.iov_len
,
785 v
.iov_base
, v
.iov_len
),
786 memcpy_from_page((to
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
787 v
.bv_offset
, v
.bv_len
),
788 memcpy((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
)
793 EXPORT_SYMBOL(_copy_from_iter_nocache
);
795 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
797 * _copy_from_iter_flushcache - write destination through cpu cache
798 * @addr: destination kernel address
799 * @bytes: total transfer length
800 * @iter: source iterator
802 * The pmem driver arranges for filesystem-dax to use this facility via
803 * dax_copy_from_iter() for ensuring that writes to persistent memory
804 * are flushed through the CPU cache. It is differentiated from
805 * _copy_from_iter_nocache() in that guarantees all data is flushed for
806 * all iterator types. The _copy_from_iter_nocache() only attempts to
807 * bypass the cache for the ITER_IOVEC case, and on some archs may use
808 * instructions that strand dirty-data in the cache.
810 size_t _copy_from_iter_flushcache(void *addr
, size_t bytes
, struct iov_iter
*i
)
813 if (unlikely(iov_iter_is_pipe(i
))) {
817 iterate_and_advance(i
, bytes
, v
,
818 __copy_from_user_flushcache((to
+= v
.iov_len
) - v
.iov_len
,
819 v
.iov_base
, v
.iov_len
),
820 memcpy_page_flushcache((to
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
821 v
.bv_offset
, v
.bv_len
),
822 memcpy_flushcache((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
,
828 EXPORT_SYMBOL_GPL(_copy_from_iter_flushcache
);
831 bool _copy_from_iter_full_nocache(void *addr
, size_t bytes
, struct iov_iter
*i
)
834 if (unlikely(iov_iter_is_pipe(i
))) {
838 if (unlikely(i
->count
< bytes
))
840 iterate_all_kinds(i
, bytes
, v
, ({
841 if (__copy_from_user_inatomic_nocache((to
+= v
.iov_len
) - v
.iov_len
,
842 v
.iov_base
, v
.iov_len
))
845 memcpy_from_page((to
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
846 v
.bv_offset
, v
.bv_len
),
847 memcpy((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
)
850 iov_iter_advance(i
, bytes
);
853 EXPORT_SYMBOL(_copy_from_iter_full_nocache
);
855 static inline bool page_copy_sane(struct page
*page
, size_t offset
, size_t n
)
857 struct page
*head
= compound_head(page
);
858 size_t v
= n
+ offset
+ page_address(page
) - page_address(head
);
860 if (likely(n
<= v
&& v
<= (PAGE_SIZE
<< compound_order(head
))))
866 size_t copy_page_to_iter(struct page
*page
, size_t offset
, size_t bytes
,
869 if (unlikely(!page_copy_sane(page
, offset
, bytes
)))
871 if (i
->type
& (ITER_BVEC
|ITER_KVEC
)) {
872 void *kaddr
= kmap_atomic(page
);
873 size_t wanted
= copy_to_iter(kaddr
+ offset
, bytes
, i
);
874 kunmap_atomic(kaddr
);
876 } else if (unlikely(iov_iter_is_discard(i
)))
878 else if (likely(!iov_iter_is_pipe(i
)))
879 return copy_page_to_iter_iovec(page
, offset
, bytes
, i
);
881 return copy_page_to_iter_pipe(page
, offset
, bytes
, i
);
883 EXPORT_SYMBOL(copy_page_to_iter
);
885 size_t copy_page_from_iter(struct page
*page
, size_t offset
, size_t bytes
,
888 if (unlikely(!page_copy_sane(page
, offset
, bytes
)))
890 if (unlikely(iov_iter_is_pipe(i
) || iov_iter_is_discard(i
))) {
894 if (i
->type
& (ITER_BVEC
|ITER_KVEC
)) {
895 void *kaddr
= kmap_atomic(page
);
896 size_t wanted
= _copy_from_iter(kaddr
+ offset
, bytes
, i
);
897 kunmap_atomic(kaddr
);
900 return copy_page_from_iter_iovec(page
, offset
, bytes
, i
);
902 EXPORT_SYMBOL(copy_page_from_iter
);
904 static size_t pipe_zero(size_t bytes
, struct iov_iter
*i
)
906 struct pipe_inode_info
*pipe
= i
->pipe
;
913 bytes
= n
= push_pipe(i
, bytes
, &idx
, &off
);
917 for ( ; n
; idx
= next_idx(idx
, pipe
), off
= 0) {
918 size_t chunk
= min_t(size_t, n
, PAGE_SIZE
- off
);
919 memzero_page(pipe
->bufs
[idx
].page
, off
, chunk
);
921 i
->iov_offset
= off
+ chunk
;
928 size_t iov_iter_zero(size_t bytes
, struct iov_iter
*i
)
930 if (unlikely(iov_iter_is_pipe(i
)))
931 return pipe_zero(bytes
, i
);
932 iterate_and_advance(i
, bytes
, v
,
933 clear_user(v
.iov_base
, v
.iov_len
),
934 memzero_page(v
.bv_page
, v
.bv_offset
, v
.bv_len
),
935 memset(v
.iov_base
, 0, v
.iov_len
)
940 EXPORT_SYMBOL(iov_iter_zero
);
942 size_t iov_iter_copy_from_user_atomic(struct page
*page
,
943 struct iov_iter
*i
, unsigned long offset
, size_t bytes
)
945 char *kaddr
= kmap_atomic(page
), *p
= kaddr
+ offset
;
946 if (unlikely(!page_copy_sane(page
, offset
, bytes
))) {
947 kunmap_atomic(kaddr
);
950 if (unlikely(iov_iter_is_pipe(i
) || iov_iter_is_discard(i
))) {
951 kunmap_atomic(kaddr
);
955 iterate_all_kinds(i
, bytes
, v
,
956 copyin((p
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
),
957 memcpy_from_page((p
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
958 v
.bv_offset
, v
.bv_len
),
959 memcpy((p
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
)
961 kunmap_atomic(kaddr
);
964 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic
);
966 static inline void pipe_truncate(struct iov_iter
*i
)
968 struct pipe_inode_info
*pipe
= i
->pipe
;
970 size_t off
= i
->iov_offset
;
972 int nrbufs
= (idx
- pipe
->curbuf
) & (pipe
->buffers
- 1);
974 pipe
->bufs
[idx
].len
= off
- pipe
->bufs
[idx
].offset
;
975 idx
= next_idx(idx
, pipe
);
978 while (pipe
->nrbufs
> nrbufs
) {
979 pipe_buf_release(pipe
, &pipe
->bufs
[idx
]);
980 idx
= next_idx(idx
, pipe
);
986 static void pipe_advance(struct iov_iter
*i
, size_t size
)
988 struct pipe_inode_info
*pipe
= i
->pipe
;
989 if (unlikely(i
->count
< size
))
992 struct pipe_buffer
*buf
;
993 size_t off
= i
->iov_offset
, left
= size
;
995 if (off
) /* make it relative to the beginning of buffer */
996 left
+= off
- pipe
->bufs
[idx
].offset
;
998 buf
= &pipe
->bufs
[idx
];
999 if (left
<= buf
->len
)
1002 idx
= next_idx(idx
, pipe
);
1005 i
->iov_offset
= buf
->offset
+ left
;
1008 /* ... and discard everything past that point */
1012 void iov_iter_advance(struct iov_iter
*i
, size_t size
)
1014 if (unlikely(iov_iter_is_pipe(i
))) {
1015 pipe_advance(i
, size
);
1018 if (unlikely(iov_iter_is_discard(i
))) {
1022 iterate_and_advance(i
, size
, v
, 0, 0, 0)
1024 EXPORT_SYMBOL(iov_iter_advance
);
1026 void iov_iter_revert(struct iov_iter
*i
, size_t unroll
)
1030 if (WARN_ON(unroll
> MAX_RW_COUNT
))
1033 if (unlikely(iov_iter_is_pipe(i
))) {
1034 struct pipe_inode_info
*pipe
= i
->pipe
;
1036 size_t off
= i
->iov_offset
;
1038 size_t n
= off
- pipe
->bufs
[idx
].offset
;
1044 if (!unroll
&& idx
== i
->start_idx
) {
1049 idx
= pipe
->buffers
- 1;
1050 off
= pipe
->bufs
[idx
].offset
+ pipe
->bufs
[idx
].len
;
1052 i
->iov_offset
= off
;
1057 if (unlikely(iov_iter_is_discard(i
)))
1059 if (unroll
<= i
->iov_offset
) {
1060 i
->iov_offset
-= unroll
;
1063 unroll
-= i
->iov_offset
;
1064 if (iov_iter_is_bvec(i
)) {
1065 const struct bio_vec
*bvec
= i
->bvec
;
1067 size_t n
= (--bvec
)->bv_len
;
1071 i
->iov_offset
= n
- unroll
;
1076 } else { /* same logics for iovec and kvec */
1077 const struct iovec
*iov
= i
->iov
;
1079 size_t n
= (--iov
)->iov_len
;
1083 i
->iov_offset
= n
- unroll
;
1090 EXPORT_SYMBOL(iov_iter_revert
);
1093 * Return the count of just the current iov_iter segment.
1095 size_t iov_iter_single_seg_count(const struct iov_iter
*i
)
1097 if (unlikely(iov_iter_is_pipe(i
)))
1098 return i
->count
; // it is a silly place, anyway
1099 if (i
->nr_segs
== 1)
1101 if (unlikely(iov_iter_is_discard(i
)))
1103 else if (iov_iter_is_bvec(i
))
1104 return min(i
->count
, i
->bvec
->bv_len
- i
->iov_offset
);
1106 return min(i
->count
, i
->iov
->iov_len
- i
->iov_offset
);
1108 EXPORT_SYMBOL(iov_iter_single_seg_count
);
1110 void iov_iter_kvec(struct iov_iter
*i
, unsigned int direction
,
1111 const struct kvec
*kvec
, unsigned long nr_segs
,
1114 WARN_ON(direction
& ~(READ
| WRITE
));
1115 i
->type
= ITER_KVEC
| (direction
& (READ
| WRITE
));
1117 i
->nr_segs
= nr_segs
;
1121 EXPORT_SYMBOL(iov_iter_kvec
);
1123 void iov_iter_bvec(struct iov_iter
*i
, unsigned int direction
,
1124 const struct bio_vec
*bvec
, unsigned long nr_segs
,
1127 WARN_ON(direction
& ~(READ
| WRITE
));
1128 i
->type
= ITER_BVEC
| (direction
& (READ
| WRITE
));
1130 i
->nr_segs
= nr_segs
;
1134 EXPORT_SYMBOL(iov_iter_bvec
);
1136 void iov_iter_pipe(struct iov_iter
*i
, unsigned int direction
,
1137 struct pipe_inode_info
*pipe
,
1140 BUG_ON(direction
!= READ
);
1141 WARN_ON(pipe
->nrbufs
== pipe
->buffers
);
1142 i
->type
= ITER_PIPE
| READ
;
1144 i
->idx
= (pipe
->curbuf
+ pipe
->nrbufs
) & (pipe
->buffers
- 1);
1147 i
->start_idx
= i
->idx
;
1149 EXPORT_SYMBOL(iov_iter_pipe
);
1152 * iov_iter_discard - Initialise an I/O iterator that discards data
1153 * @i: The iterator to initialise.
1154 * @direction: The direction of the transfer.
1155 * @count: The size of the I/O buffer in bytes.
1157 * Set up an I/O iterator that just discards everything that's written to it.
1158 * It's only available as a READ iterator.
1160 void iov_iter_discard(struct iov_iter
*i
, unsigned int direction
, size_t count
)
1162 BUG_ON(direction
!= READ
);
1163 i
->type
= ITER_DISCARD
| READ
;
1167 EXPORT_SYMBOL(iov_iter_discard
);
1169 unsigned long iov_iter_alignment(const struct iov_iter
*i
)
1171 unsigned long res
= 0;
1172 size_t size
= i
->count
;
1174 if (unlikely(iov_iter_is_pipe(i
))) {
1175 if (size
&& i
->iov_offset
&& allocated(&i
->pipe
->bufs
[i
->idx
]))
1176 return size
| i
->iov_offset
;
1179 iterate_all_kinds(i
, size
, v
,
1180 (res
|= (unsigned long)v
.iov_base
| v
.iov_len
, 0),
1181 res
|= v
.bv_offset
| v
.bv_len
,
1182 res
|= (unsigned long)v
.iov_base
| v
.iov_len
1186 EXPORT_SYMBOL(iov_iter_alignment
);
1188 unsigned long iov_iter_gap_alignment(const struct iov_iter
*i
)
1190 unsigned long res
= 0;
1191 size_t size
= i
->count
;
1193 if (unlikely(iov_iter_is_pipe(i
) || iov_iter_is_discard(i
))) {
1198 iterate_all_kinds(i
, size
, v
,
1199 (res
|= (!res
? 0 : (unsigned long)v
.iov_base
) |
1200 (size
!= v
.iov_len
? size
: 0), 0),
1201 (res
|= (!res
? 0 : (unsigned long)v
.bv_offset
) |
1202 (size
!= v
.bv_len
? size
: 0)),
1203 (res
|= (!res
? 0 : (unsigned long)v
.iov_base
) |
1204 (size
!= v
.iov_len
? size
: 0))
1208 EXPORT_SYMBOL(iov_iter_gap_alignment
);
1210 static inline ssize_t
__pipe_get_pages(struct iov_iter
*i
,
1212 struct page
**pages
,
1216 struct pipe_inode_info
*pipe
= i
->pipe
;
1217 ssize_t n
= push_pipe(i
, maxsize
, &idx
, start
);
1224 get_page(*pages
++ = pipe
->bufs
[idx
].page
);
1225 idx
= next_idx(idx
, pipe
);
1232 static ssize_t
pipe_get_pages(struct iov_iter
*i
,
1233 struct page
**pages
, size_t maxsize
, unsigned maxpages
,
1246 data_start(i
, &idx
, start
);
1247 /* some of this one + all after this one */
1248 npages
= ((i
->pipe
->curbuf
- idx
- 1) & (i
->pipe
->buffers
- 1)) + 1;
1249 capacity
= min(npages
,maxpages
) * PAGE_SIZE
- *start
;
1251 return __pipe_get_pages(i
, min(maxsize
, capacity
), pages
, idx
, start
);
1254 ssize_t
iov_iter_get_pages(struct iov_iter
*i
,
1255 struct page
**pages
, size_t maxsize
, unsigned maxpages
,
1258 if (maxsize
> i
->count
)
1261 if (unlikely(iov_iter_is_pipe(i
)))
1262 return pipe_get_pages(i
, pages
, maxsize
, maxpages
, start
);
1263 if (unlikely(iov_iter_is_discard(i
)))
1266 iterate_all_kinds(i
, maxsize
, v
, ({
1267 unsigned long addr
= (unsigned long)v
.iov_base
;
1268 size_t len
= v
.iov_len
+ (*start
= addr
& (PAGE_SIZE
- 1));
1272 if (len
> maxpages
* PAGE_SIZE
)
1273 len
= maxpages
* PAGE_SIZE
;
1274 addr
&= ~(PAGE_SIZE
- 1);
1275 n
= DIV_ROUND_UP(len
, PAGE_SIZE
);
1276 res
= get_user_pages_fast(addr
, n
, iov_iter_rw(i
) != WRITE
, pages
);
1277 if (unlikely(res
< 0))
1279 return (res
== n
? len
: res
* PAGE_SIZE
) - *start
;
1281 /* can't be more than PAGE_SIZE */
1282 *start
= v
.bv_offset
;
1283 get_page(*pages
= v
.bv_page
);
1291 EXPORT_SYMBOL(iov_iter_get_pages
);
1293 static struct page
**get_pages_array(size_t n
)
1295 return kvmalloc_array(n
, sizeof(struct page
*), GFP_KERNEL
);
1298 static ssize_t
pipe_get_pages_alloc(struct iov_iter
*i
,
1299 struct page
***pages
, size_t maxsize
,
1313 data_start(i
, &idx
, start
);
1314 /* some of this one + all after this one */
1315 npages
= ((i
->pipe
->curbuf
- idx
- 1) & (i
->pipe
->buffers
- 1)) + 1;
1316 n
= npages
* PAGE_SIZE
- *start
;
1320 npages
= DIV_ROUND_UP(maxsize
+ *start
, PAGE_SIZE
);
1321 p
= get_pages_array(npages
);
1324 n
= __pipe_get_pages(i
, maxsize
, p
, idx
, start
);
1332 ssize_t
iov_iter_get_pages_alloc(struct iov_iter
*i
,
1333 struct page
***pages
, size_t maxsize
,
1338 if (maxsize
> i
->count
)
1341 if (unlikely(iov_iter_is_pipe(i
)))
1342 return pipe_get_pages_alloc(i
, pages
, maxsize
, start
);
1343 if (unlikely(iov_iter_is_discard(i
)))
1346 iterate_all_kinds(i
, maxsize
, v
, ({
1347 unsigned long addr
= (unsigned long)v
.iov_base
;
1348 size_t len
= v
.iov_len
+ (*start
= addr
& (PAGE_SIZE
- 1));
1352 addr
&= ~(PAGE_SIZE
- 1);
1353 n
= DIV_ROUND_UP(len
, PAGE_SIZE
);
1354 p
= get_pages_array(n
);
1357 res
= get_user_pages_fast(addr
, n
, iov_iter_rw(i
) != WRITE
, p
);
1358 if (unlikely(res
< 0)) {
1363 return (res
== n
? len
: res
* PAGE_SIZE
) - *start
;
1365 /* can't be more than PAGE_SIZE */
1366 *start
= v
.bv_offset
;
1367 *pages
= p
= get_pages_array(1);
1370 get_page(*p
= v
.bv_page
);
1378 EXPORT_SYMBOL(iov_iter_get_pages_alloc
);
1380 size_t csum_and_copy_from_iter(void *addr
, size_t bytes
, __wsum
*csum
,
1387 if (unlikely(iov_iter_is_pipe(i
) || iov_iter_is_discard(i
))) {
1391 iterate_and_advance(i
, bytes
, v
, ({
1393 next
= csum_and_copy_from_user(v
.iov_base
,
1394 (to
+= v
.iov_len
) - v
.iov_len
,
1395 v
.iov_len
, 0, &err
);
1397 sum
= csum_block_add(sum
, next
, off
);
1400 err
? v
.iov_len
: 0;
1402 char *p
= kmap_atomic(v
.bv_page
);
1403 next
= csum_partial_copy_nocheck(p
+ v
.bv_offset
,
1404 (to
+= v
.bv_len
) - v
.bv_len
,
1407 sum
= csum_block_add(sum
, next
, off
);
1410 next
= csum_partial_copy_nocheck(v
.iov_base
,
1411 (to
+= v
.iov_len
) - v
.iov_len
,
1413 sum
= csum_block_add(sum
, next
, off
);
1420 EXPORT_SYMBOL(csum_and_copy_from_iter
);
1422 bool csum_and_copy_from_iter_full(void *addr
, size_t bytes
, __wsum
*csum
,
1429 if (unlikely(iov_iter_is_pipe(i
) || iov_iter_is_discard(i
))) {
1433 if (unlikely(i
->count
< bytes
))
1435 iterate_all_kinds(i
, bytes
, v
, ({
1437 next
= csum_and_copy_from_user(v
.iov_base
,
1438 (to
+= v
.iov_len
) - v
.iov_len
,
1439 v
.iov_len
, 0, &err
);
1442 sum
= csum_block_add(sum
, next
, off
);
1446 char *p
= kmap_atomic(v
.bv_page
);
1447 next
= csum_partial_copy_nocheck(p
+ v
.bv_offset
,
1448 (to
+= v
.bv_len
) - v
.bv_len
,
1451 sum
= csum_block_add(sum
, next
, off
);
1454 next
= csum_partial_copy_nocheck(v
.iov_base
,
1455 (to
+= v
.iov_len
) - v
.iov_len
,
1457 sum
= csum_block_add(sum
, next
, off
);
1462 iov_iter_advance(i
, bytes
);
1465 EXPORT_SYMBOL(csum_and_copy_from_iter_full
);
1467 size_t csum_and_copy_to_iter(const void *addr
, size_t bytes
, __wsum
*csum
,
1470 const char *from
= addr
;
1474 if (unlikely(iov_iter_is_pipe(i
)))
1475 return csum_and_copy_to_pipe_iter(addr
, bytes
, csum
, i
);
1478 if (unlikely(iov_iter_is_discard(i
))) {
1479 WARN_ON(1); /* for now */
1482 iterate_and_advance(i
, bytes
, v
, ({
1484 next
= csum_and_copy_to_user((from
+= v
.iov_len
) - v
.iov_len
,
1486 v
.iov_len
, 0, &err
);
1488 sum
= csum_block_add(sum
, next
, off
);
1491 err
? v
.iov_len
: 0;
1493 char *p
= kmap_atomic(v
.bv_page
);
1494 next
= csum_partial_copy_nocheck((from
+= v
.bv_len
) - v
.bv_len
,
1498 sum
= csum_block_add(sum
, next
, off
);
1501 next
= csum_partial_copy_nocheck((from
+= v
.iov_len
) - v
.iov_len
,
1504 sum
= csum_block_add(sum
, next
, off
);
1511 EXPORT_SYMBOL(csum_and_copy_to_iter
);
1513 int iov_iter_npages(const struct iov_iter
*i
, int maxpages
)
1515 size_t size
= i
->count
;
1520 if (unlikely(iov_iter_is_discard(i
)))
1523 if (unlikely(iov_iter_is_pipe(i
))) {
1524 struct pipe_inode_info
*pipe
= i
->pipe
;
1531 data_start(i
, &idx
, &off
);
1532 /* some of this one + all after this one */
1533 npages
= ((pipe
->curbuf
- idx
- 1) & (pipe
->buffers
- 1)) + 1;
1534 if (npages
>= maxpages
)
1536 } else iterate_all_kinds(i
, size
, v
, ({
1537 unsigned long p
= (unsigned long)v
.iov_base
;
1538 npages
+= DIV_ROUND_UP(p
+ v
.iov_len
, PAGE_SIZE
)
1540 if (npages
>= maxpages
)
1544 if (npages
>= maxpages
)
1547 unsigned long p
= (unsigned long)v
.iov_base
;
1548 npages
+= DIV_ROUND_UP(p
+ v
.iov_len
, PAGE_SIZE
)
1550 if (npages
>= maxpages
)
1556 EXPORT_SYMBOL(iov_iter_npages
);
1558 const void *dup_iter(struct iov_iter
*new, struct iov_iter
*old
, gfp_t flags
)
1561 if (unlikely(iov_iter_is_pipe(new))) {
1565 if (unlikely(iov_iter_is_discard(new)))
1567 if (iov_iter_is_bvec(new))
1568 return new->bvec
= kmemdup(new->bvec
,
1569 new->nr_segs
* sizeof(struct bio_vec
),
1572 /* iovec and kvec have identical layout */
1573 return new->iov
= kmemdup(new->iov
,
1574 new->nr_segs
* sizeof(struct iovec
),
1577 EXPORT_SYMBOL(dup_iter
);
1580 * import_iovec() - Copy an array of &struct iovec from userspace
1581 * into the kernel, check that it is valid, and initialize a new
1582 * &struct iov_iter iterator to access it.
1584 * @type: One of %READ or %WRITE.
1585 * @uvector: Pointer to the userspace array.
1586 * @nr_segs: Number of elements in userspace array.
1587 * @fast_segs: Number of elements in @iov.
1588 * @iov: (input and output parameter) Pointer to pointer to (usually small
1589 * on-stack) kernel array.
1590 * @i: Pointer to iterator that will be initialized on success.
1592 * If the array pointed to by *@iov is large enough to hold all @nr_segs,
1593 * then this function places %NULL in *@iov on return. Otherwise, a new
1594 * array will be allocated and the result placed in *@iov. This means that
1595 * the caller may call kfree() on *@iov regardless of whether the small
1596 * on-stack array was used or not (and regardless of whether this function
1597 * returns an error or not).
1599 * Return: 0 on success or negative error code on error.
1601 int import_iovec(int type
, const struct iovec __user
* uvector
,
1602 unsigned nr_segs
, unsigned fast_segs
,
1603 struct iovec
**iov
, struct iov_iter
*i
)
1607 n
= rw_copy_check_uvector(type
, uvector
, nr_segs
, fast_segs
,
1615 iov_iter_init(i
, type
, p
, nr_segs
, n
);
1616 *iov
= p
== *iov
? NULL
: p
;
1619 EXPORT_SYMBOL(import_iovec
);
1621 #ifdef CONFIG_COMPAT
1622 #include <linux/compat.h>
1624 int compat_import_iovec(int type
, const struct compat_iovec __user
* uvector
,
1625 unsigned nr_segs
, unsigned fast_segs
,
1626 struct iovec
**iov
, struct iov_iter
*i
)
1630 n
= compat_rw_copy_check_uvector(type
, uvector
, nr_segs
, fast_segs
,
1638 iov_iter_init(i
, type
, p
, nr_segs
, n
);
1639 *iov
= p
== *iov
? NULL
: p
;
1644 int import_single_range(int rw
, void __user
*buf
, size_t len
,
1645 struct iovec
*iov
, struct iov_iter
*i
)
1647 if (len
> MAX_RW_COUNT
)
1649 if (unlikely(!access_ok(!rw
, buf
, len
)))
1652 iov
->iov_base
= buf
;
1654 iov_iter_init(i
, rw
, iov
, 1, len
);
1657 EXPORT_SYMBOL(import_single_range
);
1659 int iov_iter_for_each_range(struct iov_iter
*i
, size_t bytes
,
1660 int (*f
)(struct kvec
*vec
, void *context
),
1668 iterate_all_kinds(i
, bytes
, v
, -EINVAL
, ({
1669 w
.iov_base
= kmap(v
.bv_page
) + v
.bv_offset
;
1670 w
.iov_len
= v
.bv_len
;
1671 err
= f(&w
, context
);
1675 err
= f(&w
, context
);})
1679 EXPORT_SYMBOL(iov_iter_for_each_range
);