1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/export.h>
3 #include <linux/bvec.h>
5 #include <linux/pagemap.h>
6 #include <linux/slab.h>
7 #include <linux/vmalloc.h>
8 #include <linux/splice.h>
9 #include <net/checksum.h>
10 #include <linux/scatterlist.h>
12 #define PIPE_PARANOIA /* for now */
14 #define iterate_iovec(i, n, __v, __p, skip, STEP) { \
18 __v.iov_len = min(n, __p->iov_len - skip); \
19 if (likely(__v.iov_len)) { \
20 __v.iov_base = __p->iov_base + skip; \
22 __v.iov_len -= left; \
23 skip += __v.iov_len; \
28 while (unlikely(!left && n)) { \
30 __v.iov_len = min(n, __p->iov_len); \
31 if (unlikely(!__v.iov_len)) \
33 __v.iov_base = __p->iov_base; \
35 __v.iov_len -= left; \
42 #define iterate_kvec(i, n, __v, __p, skip, STEP) { \
45 __v.iov_len = min(n, __p->iov_len - skip); \
46 if (likely(__v.iov_len)) { \
47 __v.iov_base = __p->iov_base + skip; \
49 skip += __v.iov_len; \
52 while (unlikely(n)) { \
54 __v.iov_len = min(n, __p->iov_len); \
55 if (unlikely(!__v.iov_len)) \
57 __v.iov_base = __p->iov_base; \
65 #define iterate_bvec(i, n, __v, __bi, skip, STEP) { \
66 struct bvec_iter __start; \
67 __start.bi_size = n; \
68 __start.bi_bvec_done = skip; \
70 for_each_bvec(__v, i->bvec, __bi, __start) { \
77 #define iterate_all_kinds(i, n, v, I, B, K) { \
79 size_t skip = i->iov_offset; \
80 if (unlikely(i->type & ITER_BVEC)) { \
82 struct bvec_iter __bi; \
83 iterate_bvec(i, n, v, __bi, skip, (B)) \
84 } else if (unlikely(i->type & ITER_KVEC)) { \
85 const struct kvec *kvec; \
87 iterate_kvec(i, n, v, kvec, skip, (K)) \
88 } else if (unlikely(i->type & ITER_DISCARD)) { \
90 const struct iovec *iov; \
92 iterate_iovec(i, n, v, iov, skip, (I)) \
97 #define iterate_and_advance(i, n, v, I, B, K) { \
98 if (unlikely(i->count < n)) \
101 size_t skip = i->iov_offset; \
102 if (unlikely(i->type & ITER_BVEC)) { \
103 const struct bio_vec *bvec = i->bvec; \
105 struct bvec_iter __bi; \
106 iterate_bvec(i, n, v, __bi, skip, (B)) \
107 i->bvec = __bvec_iter_bvec(i->bvec, __bi); \
108 i->nr_segs -= i->bvec - bvec; \
109 skip = __bi.bi_bvec_done; \
110 } else if (unlikely(i->type & ITER_KVEC)) { \
111 const struct kvec *kvec; \
113 iterate_kvec(i, n, v, kvec, skip, (K)) \
114 if (skip == kvec->iov_len) { \
118 i->nr_segs -= kvec - i->kvec; \
120 } else if (unlikely(i->type & ITER_DISCARD)) { \
123 const struct iovec *iov; \
125 iterate_iovec(i, n, v, iov, skip, (I)) \
126 if (skip == iov->iov_len) { \
130 i->nr_segs -= iov - i->iov; \
134 i->iov_offset = skip; \
138 static int copyout(void __user
*to
, const void *from
, size_t n
)
140 if (access_ok(to
, n
)) {
141 kasan_check_read(from
, n
);
142 n
= raw_copy_to_user(to
, from
, n
);
147 static int copyin(void *to
, const void __user
*from
, size_t n
)
149 if (access_ok(from
, n
)) {
150 kasan_check_write(to
, n
);
151 n
= raw_copy_from_user(to
, from
, n
);
156 static size_t copy_page_to_iter_iovec(struct page
*page
, size_t offset
, size_t bytes
,
159 size_t skip
, copy
, left
, wanted
;
160 const struct iovec
*iov
;
164 if (unlikely(bytes
> i
->count
))
167 if (unlikely(!bytes
))
173 skip
= i
->iov_offset
;
174 buf
= iov
->iov_base
+ skip
;
175 copy
= min(bytes
, iov
->iov_len
- skip
);
177 if (IS_ENABLED(CONFIG_HIGHMEM
) && !fault_in_pages_writeable(buf
, copy
)) {
178 kaddr
= kmap_atomic(page
);
179 from
= kaddr
+ offset
;
181 /* first chunk, usually the only one */
182 left
= copyout(buf
, from
, copy
);
188 while (unlikely(!left
&& bytes
)) {
191 copy
= min(bytes
, iov
->iov_len
);
192 left
= copyout(buf
, from
, copy
);
198 if (likely(!bytes
)) {
199 kunmap_atomic(kaddr
);
202 offset
= from
- kaddr
;
204 kunmap_atomic(kaddr
);
205 copy
= min(bytes
, iov
->iov_len
- skip
);
207 /* Too bad - revert to non-atomic kmap */
210 from
= kaddr
+ offset
;
211 left
= copyout(buf
, from
, copy
);
216 while (unlikely(!left
&& bytes
)) {
219 copy
= min(bytes
, iov
->iov_len
);
220 left
= copyout(buf
, from
, copy
);
229 if (skip
== iov
->iov_len
) {
233 i
->count
-= wanted
- bytes
;
234 i
->nr_segs
-= iov
- i
->iov
;
236 i
->iov_offset
= skip
;
237 return wanted
- bytes
;
240 static size_t copy_page_from_iter_iovec(struct page
*page
, size_t offset
, size_t bytes
,
243 size_t skip
, copy
, left
, wanted
;
244 const struct iovec
*iov
;
248 if (unlikely(bytes
> i
->count
))
251 if (unlikely(!bytes
))
257 skip
= i
->iov_offset
;
258 buf
= iov
->iov_base
+ skip
;
259 copy
= min(bytes
, iov
->iov_len
- skip
);
261 if (IS_ENABLED(CONFIG_HIGHMEM
) && !fault_in_pages_readable(buf
, copy
)) {
262 kaddr
= kmap_atomic(page
);
265 /* first chunk, usually the only one */
266 left
= copyin(to
, buf
, copy
);
272 while (unlikely(!left
&& bytes
)) {
275 copy
= min(bytes
, iov
->iov_len
);
276 left
= copyin(to
, buf
, copy
);
282 if (likely(!bytes
)) {
283 kunmap_atomic(kaddr
);
288 kunmap_atomic(kaddr
);
289 copy
= min(bytes
, iov
->iov_len
- skip
);
291 /* Too bad - revert to non-atomic kmap */
295 left
= copyin(to
, buf
, copy
);
300 while (unlikely(!left
&& bytes
)) {
303 copy
= min(bytes
, iov
->iov_len
);
304 left
= copyin(to
, buf
, copy
);
313 if (skip
== iov
->iov_len
) {
317 i
->count
-= wanted
- bytes
;
318 i
->nr_segs
-= iov
- i
->iov
;
320 i
->iov_offset
= skip
;
321 return wanted
- bytes
;
325 static bool sanity(const struct iov_iter
*i
)
327 struct pipe_inode_info
*pipe
= i
->pipe
;
328 unsigned int p_head
= pipe
->head
;
329 unsigned int p_tail
= pipe
->tail
;
330 unsigned int p_mask
= pipe
->ring_size
- 1;
331 unsigned int p_occupancy
= pipe_occupancy(p_head
, p_tail
);
332 unsigned int i_head
= i
->head
;
336 struct pipe_buffer
*p
;
337 if (unlikely(p_occupancy
== 0))
338 goto Bad
; // pipe must be non-empty
339 if (unlikely(i_head
!= p_head
- 1))
340 goto Bad
; // must be at the last buffer...
342 p
= &pipe
->bufs
[i_head
& p_mask
];
343 if (unlikely(p
->offset
+ p
->len
!= i
->iov_offset
))
344 goto Bad
; // ... at the end of segment
346 if (i_head
!= p_head
)
347 goto Bad
; // must be right after the last buffer
351 printk(KERN_ERR
"idx = %d, offset = %zd\n", i_head
, i
->iov_offset
);
352 printk(KERN_ERR
"head = %d, tail = %d, buffers = %d\n",
353 p_head
, p_tail
, pipe
->ring_size
);
354 for (idx
= 0; idx
< pipe
->ring_size
; idx
++)
355 printk(KERN_ERR
"[%p %p %d %d]\n",
357 pipe
->bufs
[idx
].page
,
358 pipe
->bufs
[idx
].offset
,
359 pipe
->bufs
[idx
].len
);
364 #define sanity(i) true
367 static size_t copy_page_to_iter_pipe(struct page
*page
, size_t offset
, size_t bytes
,
370 struct pipe_inode_info
*pipe
= i
->pipe
;
371 struct pipe_buffer
*buf
;
372 unsigned int p_tail
= pipe
->tail
;
373 unsigned int p_mask
= pipe
->ring_size
- 1;
374 unsigned int i_head
= i
->head
;
377 if (unlikely(bytes
> i
->count
))
380 if (unlikely(!bytes
))
387 buf
= &pipe
->bufs
[i_head
& p_mask
];
389 if (offset
== off
&& buf
->page
== page
) {
390 /* merge with the last one */
392 i
->iov_offset
+= bytes
;
396 buf
= &pipe
->bufs
[i_head
& p_mask
];
398 if (pipe_full(i_head
, p_tail
, pipe
->max_usage
))
401 buf
->ops
= &page_cache_pipe_buf_ops
;
404 buf
->offset
= offset
;
407 pipe
->head
= i_head
+ 1;
408 i
->iov_offset
= offset
+ bytes
;
416 * Fault in one or more iovecs of the given iov_iter, to a maximum length of
417 * bytes. For each iovec, fault in each page that constitutes the iovec.
419 * Return 0 on success, or non-zero if the memory could not be accessed (i.e.
420 * because it is an invalid address).
422 int iov_iter_fault_in_readable(struct iov_iter
*i
, size_t bytes
)
424 size_t skip
= i
->iov_offset
;
425 const struct iovec
*iov
;
429 if (!(i
->type
& (ITER_BVEC
|ITER_KVEC
))) {
430 iterate_iovec(i
, bytes
, v
, iov
, skip
, ({
431 err
= fault_in_pages_readable(v
.iov_base
, v
.iov_len
);
438 EXPORT_SYMBOL(iov_iter_fault_in_readable
);
440 void iov_iter_init(struct iov_iter
*i
, unsigned int direction
,
441 const struct iovec
*iov
, unsigned long nr_segs
,
444 WARN_ON(direction
& ~(READ
| WRITE
));
445 direction
&= READ
| WRITE
;
447 /* It will get better. Eventually... */
448 if (uaccess_kernel()) {
449 i
->type
= ITER_KVEC
| direction
;
450 i
->kvec
= (struct kvec
*)iov
;
452 i
->type
= ITER_IOVEC
| direction
;
455 i
->nr_segs
= nr_segs
;
459 EXPORT_SYMBOL(iov_iter_init
);
461 static void memcpy_from_page(char *to
, struct page
*page
, size_t offset
, size_t len
)
463 char *from
= kmap_atomic(page
);
464 memcpy(to
, from
+ offset
, len
);
468 static void memcpy_to_page(struct page
*page
, size_t offset
, const char *from
, size_t len
)
470 char *to
= kmap_atomic(page
);
471 memcpy(to
+ offset
, from
, len
);
475 static void memzero_page(struct page
*page
, size_t offset
, size_t len
)
477 char *addr
= kmap_atomic(page
);
478 memset(addr
+ offset
, 0, len
);
482 static inline bool allocated(struct pipe_buffer
*buf
)
484 return buf
->ops
== &default_pipe_buf_ops
;
487 static inline void data_start(const struct iov_iter
*i
,
488 unsigned int *iter_headp
, size_t *offp
)
490 unsigned int p_mask
= i
->pipe
->ring_size
- 1;
491 unsigned int iter_head
= i
->head
;
492 size_t off
= i
->iov_offset
;
494 if (off
&& (!allocated(&i
->pipe
->bufs
[iter_head
& p_mask
]) ||
499 *iter_headp
= iter_head
;
503 static size_t push_pipe(struct iov_iter
*i
, size_t size
,
504 int *iter_headp
, size_t *offp
)
506 struct pipe_inode_info
*pipe
= i
->pipe
;
507 unsigned int p_tail
= pipe
->tail
;
508 unsigned int p_mask
= pipe
->ring_size
- 1;
509 unsigned int iter_head
;
513 if (unlikely(size
> i
->count
))
519 data_start(i
, &iter_head
, &off
);
520 *iter_headp
= iter_head
;
523 left
-= PAGE_SIZE
- off
;
525 pipe
->bufs
[iter_head
& p_mask
].len
+= size
;
528 pipe
->bufs
[iter_head
& p_mask
].len
= PAGE_SIZE
;
531 while (!pipe_full(iter_head
, p_tail
, pipe
->max_usage
)) {
532 struct pipe_buffer
*buf
= &pipe
->bufs
[iter_head
& p_mask
];
533 struct page
*page
= alloc_page(GFP_USER
);
537 buf
->ops
= &default_pipe_buf_ops
;
540 buf
->len
= min_t(ssize_t
, left
, PAGE_SIZE
);
543 pipe
->head
= iter_head
;
551 static size_t copy_pipe_to_iter(const void *addr
, size_t bytes
,
554 struct pipe_inode_info
*pipe
= i
->pipe
;
555 unsigned int p_mask
= pipe
->ring_size
- 1;
562 bytes
= n
= push_pipe(i
, bytes
, &i_head
, &off
);
566 size_t chunk
= min_t(size_t, n
, PAGE_SIZE
- off
);
567 memcpy_to_page(pipe
->bufs
[i_head
& p_mask
].page
, off
, addr
, chunk
);
569 i
->iov_offset
= off
+ chunk
;
579 static __wsum
csum_and_memcpy(void *to
, const void *from
, size_t len
,
580 __wsum sum
, size_t off
)
582 __wsum next
= csum_partial_copy_nocheck(from
, to
, len
, 0);
583 return csum_block_add(sum
, next
, off
);
586 static size_t csum_and_copy_to_pipe_iter(const void *addr
, size_t bytes
,
587 __wsum
*csum
, struct iov_iter
*i
)
589 struct pipe_inode_info
*pipe
= i
->pipe
;
590 unsigned int p_mask
= pipe
->ring_size
- 1;
599 bytes
= n
= push_pipe(i
, bytes
, &i_head
, &r
);
603 size_t chunk
= min_t(size_t, n
, PAGE_SIZE
- r
);
604 char *p
= kmap_atomic(pipe
->bufs
[i_head
& p_mask
].page
);
605 sum
= csum_and_memcpy(p
+ r
, addr
, chunk
, sum
, off
);
608 i
->iov_offset
= r
+ chunk
;
620 size_t _copy_to_iter(const void *addr
, size_t bytes
, struct iov_iter
*i
)
622 const char *from
= addr
;
623 if (unlikely(iov_iter_is_pipe(i
)))
624 return copy_pipe_to_iter(addr
, bytes
, i
);
625 if (iter_is_iovec(i
))
627 iterate_and_advance(i
, bytes
, v
,
628 copyout(v
.iov_base
, (from
+= v
.iov_len
) - v
.iov_len
, v
.iov_len
),
629 memcpy_to_page(v
.bv_page
, v
.bv_offset
,
630 (from
+= v
.bv_len
) - v
.bv_len
, v
.bv_len
),
631 memcpy(v
.iov_base
, (from
+= v
.iov_len
) - v
.iov_len
, v
.iov_len
)
636 EXPORT_SYMBOL(_copy_to_iter
);
638 #ifdef CONFIG_ARCH_HAS_UACCESS_MCSAFE
639 static int copyout_mcsafe(void __user
*to
, const void *from
, size_t n
)
641 if (access_ok(to
, n
)) {
642 kasan_check_read(from
, n
);
643 n
= copy_to_user_mcsafe((__force
void *) to
, from
, n
);
648 static unsigned long memcpy_mcsafe_to_page(struct page
*page
, size_t offset
,
649 const char *from
, size_t len
)
654 to
= kmap_atomic(page
);
655 ret
= memcpy_mcsafe(to
+ offset
, from
, len
);
661 static size_t copy_pipe_to_iter_mcsafe(const void *addr
, size_t bytes
,
664 struct pipe_inode_info
*pipe
= i
->pipe
;
665 unsigned int p_mask
= pipe
->ring_size
- 1;
667 size_t n
, off
, xfer
= 0;
672 bytes
= n
= push_pipe(i
, bytes
, &i_head
, &off
);
676 size_t chunk
= min_t(size_t, n
, PAGE_SIZE
- off
);
679 rem
= memcpy_mcsafe_to_page(pipe
->bufs
[i_head
& p_mask
].page
,
682 i
->iov_offset
= off
+ chunk
- rem
;
696 * _copy_to_iter_mcsafe - copy to user with source-read error exception handling
697 * @addr: source kernel address
698 * @bytes: total transfer length
699 * @iter: destination iterator
701 * The pmem driver arranges for filesystem-dax to use this facility via
702 * dax_copy_to_iter() for protecting read/write to persistent memory.
703 * Unless / until an architecture can guarantee identical performance
704 * between _copy_to_iter_mcsafe() and _copy_to_iter() it would be a
705 * performance regression to switch more users to the mcsafe version.
707 * Otherwise, the main differences between this and typical _copy_to_iter().
709 * * Typical tail/residue handling after a fault retries the copy
710 * byte-by-byte until the fault happens again. Re-triggering machine
711 * checks is potentially fatal so the implementation uses source
712 * alignment and poison alignment assumptions to avoid re-triggering
713 * hardware exceptions.
715 * * ITER_KVEC, ITER_PIPE, and ITER_BVEC can return short copies.
716 * Compare to copy_to_iter() where only ITER_IOVEC attempts might return
719 * See MCSAFE_TEST for self-test.
721 size_t _copy_to_iter_mcsafe(const void *addr
, size_t bytes
, struct iov_iter
*i
)
723 const char *from
= addr
;
724 unsigned long rem
, curr_addr
, s_addr
= (unsigned long) addr
;
726 if (unlikely(iov_iter_is_pipe(i
)))
727 return copy_pipe_to_iter_mcsafe(addr
, bytes
, i
);
728 if (iter_is_iovec(i
))
730 iterate_and_advance(i
, bytes
, v
,
731 copyout_mcsafe(v
.iov_base
, (from
+= v
.iov_len
) - v
.iov_len
, v
.iov_len
),
733 rem
= memcpy_mcsafe_to_page(v
.bv_page
, v
.bv_offset
,
734 (from
+= v
.bv_len
) - v
.bv_len
, v
.bv_len
);
736 curr_addr
= (unsigned long) from
;
737 bytes
= curr_addr
- s_addr
- rem
;
742 rem
= memcpy_mcsafe(v
.iov_base
, (from
+= v
.iov_len
) - v
.iov_len
,
745 curr_addr
= (unsigned long) from
;
746 bytes
= curr_addr
- s_addr
- rem
;
754 EXPORT_SYMBOL_GPL(_copy_to_iter_mcsafe
);
755 #endif /* CONFIG_ARCH_HAS_UACCESS_MCSAFE */
757 size_t _copy_from_iter(void *addr
, size_t bytes
, struct iov_iter
*i
)
760 if (unlikely(iov_iter_is_pipe(i
))) {
764 if (iter_is_iovec(i
))
766 iterate_and_advance(i
, bytes
, v
,
767 copyin((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
),
768 memcpy_from_page((to
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
769 v
.bv_offset
, v
.bv_len
),
770 memcpy((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
)
775 EXPORT_SYMBOL(_copy_from_iter
);
777 bool _copy_from_iter_full(void *addr
, size_t bytes
, struct iov_iter
*i
)
780 if (unlikely(iov_iter_is_pipe(i
))) {
784 if (unlikely(i
->count
< bytes
))
787 if (iter_is_iovec(i
))
789 iterate_all_kinds(i
, bytes
, v
, ({
790 if (copyin((to
+= v
.iov_len
) - v
.iov_len
,
791 v
.iov_base
, v
.iov_len
))
794 memcpy_from_page((to
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
795 v
.bv_offset
, v
.bv_len
),
796 memcpy((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
)
799 iov_iter_advance(i
, bytes
);
802 EXPORT_SYMBOL(_copy_from_iter_full
);
804 size_t _copy_from_iter_nocache(void *addr
, size_t bytes
, struct iov_iter
*i
)
807 if (unlikely(iov_iter_is_pipe(i
))) {
811 iterate_and_advance(i
, bytes
, v
,
812 __copy_from_user_inatomic_nocache((to
+= v
.iov_len
) - v
.iov_len
,
813 v
.iov_base
, v
.iov_len
),
814 memcpy_from_page((to
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
815 v
.bv_offset
, v
.bv_len
),
816 memcpy((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
)
821 EXPORT_SYMBOL(_copy_from_iter_nocache
);
823 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
825 * _copy_from_iter_flushcache - write destination through cpu cache
826 * @addr: destination kernel address
827 * @bytes: total transfer length
828 * @iter: source iterator
830 * The pmem driver arranges for filesystem-dax to use this facility via
831 * dax_copy_from_iter() for ensuring that writes to persistent memory
832 * are flushed through the CPU cache. It is differentiated from
833 * _copy_from_iter_nocache() in that guarantees all data is flushed for
834 * all iterator types. The _copy_from_iter_nocache() only attempts to
835 * bypass the cache for the ITER_IOVEC case, and on some archs may use
836 * instructions that strand dirty-data in the cache.
838 size_t _copy_from_iter_flushcache(void *addr
, size_t bytes
, struct iov_iter
*i
)
841 if (unlikely(iov_iter_is_pipe(i
))) {
845 iterate_and_advance(i
, bytes
, v
,
846 __copy_from_user_flushcache((to
+= v
.iov_len
) - v
.iov_len
,
847 v
.iov_base
, v
.iov_len
),
848 memcpy_page_flushcache((to
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
849 v
.bv_offset
, v
.bv_len
),
850 memcpy_flushcache((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
,
856 EXPORT_SYMBOL_GPL(_copy_from_iter_flushcache
);
859 bool _copy_from_iter_full_nocache(void *addr
, size_t bytes
, struct iov_iter
*i
)
862 if (unlikely(iov_iter_is_pipe(i
))) {
866 if (unlikely(i
->count
< bytes
))
868 iterate_all_kinds(i
, bytes
, v
, ({
869 if (__copy_from_user_inatomic_nocache((to
+= v
.iov_len
) - v
.iov_len
,
870 v
.iov_base
, v
.iov_len
))
873 memcpy_from_page((to
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
874 v
.bv_offset
, v
.bv_len
),
875 memcpy((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
)
878 iov_iter_advance(i
, bytes
);
881 EXPORT_SYMBOL(_copy_from_iter_full_nocache
);
883 static inline bool page_copy_sane(struct page
*page
, size_t offset
, size_t n
)
886 size_t v
= n
+ offset
;
889 * The general case needs to access the page order in order
890 * to compute the page size.
891 * However, we mostly deal with order-0 pages and thus can
892 * avoid a possible cache line miss for requests that fit all
895 if (n
<= v
&& v
<= PAGE_SIZE
)
898 head
= compound_head(page
);
899 v
+= (page
- head
) << PAGE_SHIFT
;
901 if (likely(n
<= v
&& v
<= (page_size(head
))))
907 size_t copy_page_to_iter(struct page
*page
, size_t offset
, size_t bytes
,
910 if (unlikely(!page_copy_sane(page
, offset
, bytes
)))
912 if (i
->type
& (ITER_BVEC
|ITER_KVEC
)) {
913 void *kaddr
= kmap_atomic(page
);
914 size_t wanted
= copy_to_iter(kaddr
+ offset
, bytes
, i
);
915 kunmap_atomic(kaddr
);
917 } else if (unlikely(iov_iter_is_discard(i
)))
919 else if (likely(!iov_iter_is_pipe(i
)))
920 return copy_page_to_iter_iovec(page
, offset
, bytes
, i
);
922 return copy_page_to_iter_pipe(page
, offset
, bytes
, i
);
924 EXPORT_SYMBOL(copy_page_to_iter
);
926 size_t copy_page_from_iter(struct page
*page
, size_t offset
, size_t bytes
,
929 if (unlikely(!page_copy_sane(page
, offset
, bytes
)))
931 if (unlikely(iov_iter_is_pipe(i
) || iov_iter_is_discard(i
))) {
935 if (i
->type
& (ITER_BVEC
|ITER_KVEC
)) {
936 void *kaddr
= kmap_atomic(page
);
937 size_t wanted
= _copy_from_iter(kaddr
+ offset
, bytes
, i
);
938 kunmap_atomic(kaddr
);
941 return copy_page_from_iter_iovec(page
, offset
, bytes
, i
);
943 EXPORT_SYMBOL(copy_page_from_iter
);
945 static size_t pipe_zero(size_t bytes
, struct iov_iter
*i
)
947 struct pipe_inode_info
*pipe
= i
->pipe
;
948 unsigned int p_mask
= pipe
->ring_size
- 1;
955 bytes
= n
= push_pipe(i
, bytes
, &i_head
, &off
);
960 size_t chunk
= min_t(size_t, n
, PAGE_SIZE
- off
);
961 memzero_page(pipe
->bufs
[i_head
& p_mask
].page
, off
, chunk
);
963 i
->iov_offset
= off
+ chunk
;
972 size_t iov_iter_zero(size_t bytes
, struct iov_iter
*i
)
974 if (unlikely(iov_iter_is_pipe(i
)))
975 return pipe_zero(bytes
, i
);
976 iterate_and_advance(i
, bytes
, v
,
977 clear_user(v
.iov_base
, v
.iov_len
),
978 memzero_page(v
.bv_page
, v
.bv_offset
, v
.bv_len
),
979 memset(v
.iov_base
, 0, v
.iov_len
)
984 EXPORT_SYMBOL(iov_iter_zero
);
986 size_t iov_iter_copy_from_user_atomic(struct page
*page
,
987 struct iov_iter
*i
, unsigned long offset
, size_t bytes
)
989 char *kaddr
= kmap_atomic(page
), *p
= kaddr
+ offset
;
990 if (unlikely(!page_copy_sane(page
, offset
, bytes
))) {
991 kunmap_atomic(kaddr
);
994 if (unlikely(iov_iter_is_pipe(i
) || iov_iter_is_discard(i
))) {
995 kunmap_atomic(kaddr
);
999 iterate_all_kinds(i
, bytes
, v
,
1000 copyin((p
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
),
1001 memcpy_from_page((p
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
1002 v
.bv_offset
, v
.bv_len
),
1003 memcpy((p
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
)
1005 kunmap_atomic(kaddr
);
1008 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic
);
1010 static inline void pipe_truncate(struct iov_iter
*i
)
1012 struct pipe_inode_info
*pipe
= i
->pipe
;
1013 unsigned int p_tail
= pipe
->tail
;
1014 unsigned int p_head
= pipe
->head
;
1015 unsigned int p_mask
= pipe
->ring_size
- 1;
1017 if (!pipe_empty(p_head
, p_tail
)) {
1018 struct pipe_buffer
*buf
;
1019 unsigned int i_head
= i
->head
;
1020 size_t off
= i
->iov_offset
;
1023 buf
= &pipe
->bufs
[i_head
& p_mask
];
1024 buf
->len
= off
- buf
->offset
;
1027 while (p_head
!= i_head
) {
1029 pipe_buf_release(pipe
, &pipe
->bufs
[p_head
& p_mask
]);
1032 pipe
->head
= p_head
;
1036 static void pipe_advance(struct iov_iter
*i
, size_t size
)
1038 struct pipe_inode_info
*pipe
= i
->pipe
;
1039 if (unlikely(i
->count
< size
))
1042 struct pipe_buffer
*buf
;
1043 unsigned int p_mask
= pipe
->ring_size
- 1;
1044 unsigned int i_head
= i
->head
;
1045 size_t off
= i
->iov_offset
, left
= size
;
1047 if (off
) /* make it relative to the beginning of buffer */
1048 left
+= off
- pipe
->bufs
[i_head
& p_mask
].offset
;
1050 buf
= &pipe
->bufs
[i_head
& p_mask
];
1051 if (left
<= buf
->len
)
1057 i
->iov_offset
= buf
->offset
+ left
;
1060 /* ... and discard everything past that point */
1064 void iov_iter_advance(struct iov_iter
*i
, size_t size
)
1066 if (unlikely(iov_iter_is_pipe(i
))) {
1067 pipe_advance(i
, size
);
1070 if (unlikely(iov_iter_is_discard(i
))) {
1074 iterate_and_advance(i
, size
, v
, 0, 0, 0)
1076 EXPORT_SYMBOL(iov_iter_advance
);
1078 void iov_iter_revert(struct iov_iter
*i
, size_t unroll
)
1082 if (WARN_ON(unroll
> MAX_RW_COUNT
))
1085 if (unlikely(iov_iter_is_pipe(i
))) {
1086 struct pipe_inode_info
*pipe
= i
->pipe
;
1087 unsigned int p_mask
= pipe
->ring_size
- 1;
1088 unsigned int i_head
= i
->head
;
1089 size_t off
= i
->iov_offset
;
1091 struct pipe_buffer
*b
= &pipe
->bufs
[i_head
& p_mask
];
1092 size_t n
= off
- b
->offset
;
1098 if (!unroll
&& i_head
== i
->start_head
) {
1103 b
= &pipe
->bufs
[i_head
& p_mask
];
1104 off
= b
->offset
+ b
->len
;
1106 i
->iov_offset
= off
;
1111 if (unlikely(iov_iter_is_discard(i
)))
1113 if (unroll
<= i
->iov_offset
) {
1114 i
->iov_offset
-= unroll
;
1117 unroll
-= i
->iov_offset
;
1118 if (iov_iter_is_bvec(i
)) {
1119 const struct bio_vec
*bvec
= i
->bvec
;
1121 size_t n
= (--bvec
)->bv_len
;
1125 i
->iov_offset
= n
- unroll
;
1130 } else { /* same logics for iovec and kvec */
1131 const struct iovec
*iov
= i
->iov
;
1133 size_t n
= (--iov
)->iov_len
;
1137 i
->iov_offset
= n
- unroll
;
1144 EXPORT_SYMBOL(iov_iter_revert
);
1147 * Return the count of just the current iov_iter segment.
1149 size_t iov_iter_single_seg_count(const struct iov_iter
*i
)
1151 if (unlikely(iov_iter_is_pipe(i
)))
1152 return i
->count
; // it is a silly place, anyway
1153 if (i
->nr_segs
== 1)
1155 if (unlikely(iov_iter_is_discard(i
)))
1157 else if (iov_iter_is_bvec(i
))
1158 return min(i
->count
, i
->bvec
->bv_len
- i
->iov_offset
);
1160 return min(i
->count
, i
->iov
->iov_len
- i
->iov_offset
);
1162 EXPORT_SYMBOL(iov_iter_single_seg_count
);
1164 void iov_iter_kvec(struct iov_iter
*i
, unsigned int direction
,
1165 const struct kvec
*kvec
, unsigned long nr_segs
,
1168 WARN_ON(direction
& ~(READ
| WRITE
));
1169 i
->type
= ITER_KVEC
| (direction
& (READ
| WRITE
));
1171 i
->nr_segs
= nr_segs
;
1175 EXPORT_SYMBOL(iov_iter_kvec
);
1177 void iov_iter_bvec(struct iov_iter
*i
, unsigned int direction
,
1178 const struct bio_vec
*bvec
, unsigned long nr_segs
,
1181 WARN_ON(direction
& ~(READ
| WRITE
));
1182 i
->type
= ITER_BVEC
| (direction
& (READ
| WRITE
));
1184 i
->nr_segs
= nr_segs
;
1188 EXPORT_SYMBOL(iov_iter_bvec
);
1190 void iov_iter_pipe(struct iov_iter
*i
, unsigned int direction
,
1191 struct pipe_inode_info
*pipe
,
1194 BUG_ON(direction
!= READ
);
1195 WARN_ON(pipe_full(pipe
->head
, pipe
->tail
, pipe
->ring_size
));
1196 i
->type
= ITER_PIPE
| READ
;
1198 i
->head
= pipe
->head
;
1201 i
->start_head
= i
->head
;
1203 EXPORT_SYMBOL(iov_iter_pipe
);
1206 * iov_iter_discard - Initialise an I/O iterator that discards data
1207 * @i: The iterator to initialise.
1208 * @direction: The direction of the transfer.
1209 * @count: The size of the I/O buffer in bytes.
1211 * Set up an I/O iterator that just discards everything that's written to it.
1212 * It's only available as a READ iterator.
1214 void iov_iter_discard(struct iov_iter
*i
, unsigned int direction
, size_t count
)
1216 BUG_ON(direction
!= READ
);
1217 i
->type
= ITER_DISCARD
| READ
;
1221 EXPORT_SYMBOL(iov_iter_discard
);
1223 unsigned long iov_iter_alignment(const struct iov_iter
*i
)
1225 unsigned int p_mask
= i
->pipe
->ring_size
- 1;
1226 unsigned long res
= 0;
1227 size_t size
= i
->count
;
1229 if (unlikely(iov_iter_is_pipe(i
))) {
1230 if (size
&& i
->iov_offset
&& allocated(&i
->pipe
->bufs
[i
->head
& p_mask
]))
1231 return size
| i
->iov_offset
;
1234 iterate_all_kinds(i
, size
, v
,
1235 (res
|= (unsigned long)v
.iov_base
| v
.iov_len
, 0),
1236 res
|= v
.bv_offset
| v
.bv_len
,
1237 res
|= (unsigned long)v
.iov_base
| v
.iov_len
1241 EXPORT_SYMBOL(iov_iter_alignment
);
1243 unsigned long iov_iter_gap_alignment(const struct iov_iter
*i
)
1245 unsigned long res
= 0;
1246 size_t size
= i
->count
;
1248 if (unlikely(iov_iter_is_pipe(i
) || iov_iter_is_discard(i
))) {
1253 iterate_all_kinds(i
, size
, v
,
1254 (res
|= (!res
? 0 : (unsigned long)v
.iov_base
) |
1255 (size
!= v
.iov_len
? size
: 0), 0),
1256 (res
|= (!res
? 0 : (unsigned long)v
.bv_offset
) |
1257 (size
!= v
.bv_len
? size
: 0)),
1258 (res
|= (!res
? 0 : (unsigned long)v
.iov_base
) |
1259 (size
!= v
.iov_len
? size
: 0))
1263 EXPORT_SYMBOL(iov_iter_gap_alignment
);
1265 static inline ssize_t
__pipe_get_pages(struct iov_iter
*i
,
1267 struct page
**pages
,
1271 struct pipe_inode_info
*pipe
= i
->pipe
;
1272 unsigned int p_mask
= pipe
->ring_size
- 1;
1273 ssize_t n
= push_pipe(i
, maxsize
, &iter_head
, start
);
1280 get_page(*pages
++ = pipe
->bufs
[iter_head
& p_mask
].page
);
1288 static ssize_t
pipe_get_pages(struct iov_iter
*i
,
1289 struct page
**pages
, size_t maxsize
, unsigned maxpages
,
1292 unsigned int iter_head
, npages
;
1301 data_start(i
, &iter_head
, start
);
1302 /* Amount of free space: some of this one + all after this one */
1303 npages
= pipe_space_for_user(iter_head
, i
->pipe
->tail
, i
->pipe
);
1304 capacity
= min(npages
, maxpages
) * PAGE_SIZE
- *start
;
1306 return __pipe_get_pages(i
, min(maxsize
, capacity
), pages
, iter_head
, start
);
1309 ssize_t
iov_iter_get_pages(struct iov_iter
*i
,
1310 struct page
**pages
, size_t maxsize
, unsigned maxpages
,
1313 if (maxsize
> i
->count
)
1316 if (unlikely(iov_iter_is_pipe(i
)))
1317 return pipe_get_pages(i
, pages
, maxsize
, maxpages
, start
);
1318 if (unlikely(iov_iter_is_discard(i
)))
1321 iterate_all_kinds(i
, maxsize
, v
, ({
1322 unsigned long addr
= (unsigned long)v
.iov_base
;
1323 size_t len
= v
.iov_len
+ (*start
= addr
& (PAGE_SIZE
- 1));
1327 if (len
> maxpages
* PAGE_SIZE
)
1328 len
= maxpages
* PAGE_SIZE
;
1329 addr
&= ~(PAGE_SIZE
- 1);
1330 n
= DIV_ROUND_UP(len
, PAGE_SIZE
);
1331 res
= get_user_pages_fast(addr
, n
,
1332 iov_iter_rw(i
) != WRITE
? FOLL_WRITE
: 0,
1334 if (unlikely(res
< 0))
1336 return (res
== n
? len
: res
* PAGE_SIZE
) - *start
;
1338 /* can't be more than PAGE_SIZE */
1339 *start
= v
.bv_offset
;
1340 get_page(*pages
= v
.bv_page
);
1348 EXPORT_SYMBOL(iov_iter_get_pages
);
1350 static struct page
**get_pages_array(size_t n
)
1352 return kvmalloc_array(n
, sizeof(struct page
*), GFP_KERNEL
);
1355 static ssize_t
pipe_get_pages_alloc(struct iov_iter
*i
,
1356 struct page
***pages
, size_t maxsize
,
1360 unsigned int iter_head
, npages
;
1369 data_start(i
, &iter_head
, start
);
1370 /* Amount of free space: some of this one + all after this one */
1371 npages
= pipe_space_for_user(iter_head
, i
->pipe
->tail
, i
->pipe
);
1372 n
= npages
* PAGE_SIZE
- *start
;
1376 npages
= DIV_ROUND_UP(maxsize
+ *start
, PAGE_SIZE
);
1377 p
= get_pages_array(npages
);
1380 n
= __pipe_get_pages(i
, maxsize
, p
, iter_head
, start
);
1388 ssize_t
iov_iter_get_pages_alloc(struct iov_iter
*i
,
1389 struct page
***pages
, size_t maxsize
,
1394 if (maxsize
> i
->count
)
1397 if (unlikely(iov_iter_is_pipe(i
)))
1398 return pipe_get_pages_alloc(i
, pages
, maxsize
, start
);
1399 if (unlikely(iov_iter_is_discard(i
)))
1402 iterate_all_kinds(i
, maxsize
, v
, ({
1403 unsigned long addr
= (unsigned long)v
.iov_base
;
1404 size_t len
= v
.iov_len
+ (*start
= addr
& (PAGE_SIZE
- 1));
1408 addr
&= ~(PAGE_SIZE
- 1);
1409 n
= DIV_ROUND_UP(len
, PAGE_SIZE
);
1410 p
= get_pages_array(n
);
1413 res
= get_user_pages_fast(addr
, n
,
1414 iov_iter_rw(i
) != WRITE
? FOLL_WRITE
: 0, p
);
1415 if (unlikely(res
< 0)) {
1420 return (res
== n
? len
: res
* PAGE_SIZE
) - *start
;
1422 /* can't be more than PAGE_SIZE */
1423 *start
= v
.bv_offset
;
1424 *pages
= p
= get_pages_array(1);
1427 get_page(*p
= v
.bv_page
);
1435 EXPORT_SYMBOL(iov_iter_get_pages_alloc
);
1437 size_t csum_and_copy_from_iter(void *addr
, size_t bytes
, __wsum
*csum
,
1444 if (unlikely(iov_iter_is_pipe(i
) || iov_iter_is_discard(i
))) {
1448 iterate_and_advance(i
, bytes
, v
, ({
1450 next
= csum_and_copy_from_user(v
.iov_base
,
1451 (to
+= v
.iov_len
) - v
.iov_len
,
1452 v
.iov_len
, 0, &err
);
1454 sum
= csum_block_add(sum
, next
, off
);
1457 err
? v
.iov_len
: 0;
1459 char *p
= kmap_atomic(v
.bv_page
);
1460 sum
= csum_and_memcpy((to
+= v
.bv_len
) - v
.bv_len
,
1461 p
+ v
.bv_offset
, v
.bv_len
,
1466 sum
= csum_and_memcpy((to
+= v
.iov_len
) - v
.iov_len
,
1467 v
.iov_base
, v
.iov_len
,
1475 EXPORT_SYMBOL(csum_and_copy_from_iter
);
1477 bool csum_and_copy_from_iter_full(void *addr
, size_t bytes
, __wsum
*csum
,
1484 if (unlikely(iov_iter_is_pipe(i
) || iov_iter_is_discard(i
))) {
1488 if (unlikely(i
->count
< bytes
))
1490 iterate_all_kinds(i
, bytes
, v
, ({
1492 next
= csum_and_copy_from_user(v
.iov_base
,
1493 (to
+= v
.iov_len
) - v
.iov_len
,
1494 v
.iov_len
, 0, &err
);
1497 sum
= csum_block_add(sum
, next
, off
);
1501 char *p
= kmap_atomic(v
.bv_page
);
1502 sum
= csum_and_memcpy((to
+= v
.bv_len
) - v
.bv_len
,
1503 p
+ v
.bv_offset
, v
.bv_len
,
1508 sum
= csum_and_memcpy((to
+= v
.iov_len
) - v
.iov_len
,
1509 v
.iov_base
, v
.iov_len
,
1515 iov_iter_advance(i
, bytes
);
1518 EXPORT_SYMBOL(csum_and_copy_from_iter_full
);
1520 size_t csum_and_copy_to_iter(const void *addr
, size_t bytes
, void *csump
,
1523 const char *from
= addr
;
1524 __wsum
*csum
= csump
;
1528 if (unlikely(iov_iter_is_pipe(i
)))
1529 return csum_and_copy_to_pipe_iter(addr
, bytes
, csum
, i
);
1532 if (unlikely(iov_iter_is_discard(i
))) {
1533 WARN_ON(1); /* for now */
1536 iterate_and_advance(i
, bytes
, v
, ({
1538 next
= csum_and_copy_to_user((from
+= v
.iov_len
) - v
.iov_len
,
1540 v
.iov_len
, 0, &err
);
1542 sum
= csum_block_add(sum
, next
, off
);
1545 err
? v
.iov_len
: 0;
1547 char *p
= kmap_atomic(v
.bv_page
);
1548 sum
= csum_and_memcpy(p
+ v
.bv_offset
,
1549 (from
+= v
.bv_len
) - v
.bv_len
,
1550 v
.bv_len
, sum
, off
);
1554 sum
= csum_and_memcpy(v
.iov_base
,
1555 (from
+= v
.iov_len
) - v
.iov_len
,
1556 v
.iov_len
, sum
, off
);
1563 EXPORT_SYMBOL(csum_and_copy_to_iter
);
1565 size_t hash_and_copy_to_iter(const void *addr
, size_t bytes
, void *hashp
,
1568 #ifdef CONFIG_CRYPTO
1569 struct ahash_request
*hash
= hashp
;
1570 struct scatterlist sg
;
1573 copied
= copy_to_iter(addr
, bytes
, i
);
1574 sg_init_one(&sg
, addr
, copied
);
1575 ahash_request_set_crypt(hash
, &sg
, NULL
, copied
);
1576 crypto_ahash_update(hash
);
1582 EXPORT_SYMBOL(hash_and_copy_to_iter
);
1584 int iov_iter_npages(const struct iov_iter
*i
, int maxpages
)
1586 size_t size
= i
->count
;
1591 if (unlikely(iov_iter_is_discard(i
)))
1594 if (unlikely(iov_iter_is_pipe(i
))) {
1595 struct pipe_inode_info
*pipe
= i
->pipe
;
1596 unsigned int iter_head
;
1602 data_start(i
, &iter_head
, &off
);
1603 /* some of this one + all after this one */
1604 npages
= pipe_space_for_user(iter_head
, pipe
->tail
, pipe
);
1605 if (npages
>= maxpages
)
1607 } else iterate_all_kinds(i
, size
, v
, ({
1608 unsigned long p
= (unsigned long)v
.iov_base
;
1609 npages
+= DIV_ROUND_UP(p
+ v
.iov_len
, PAGE_SIZE
)
1611 if (npages
>= maxpages
)
1615 if (npages
>= maxpages
)
1618 unsigned long p
= (unsigned long)v
.iov_base
;
1619 npages
+= DIV_ROUND_UP(p
+ v
.iov_len
, PAGE_SIZE
)
1621 if (npages
>= maxpages
)
1627 EXPORT_SYMBOL(iov_iter_npages
);
1629 const void *dup_iter(struct iov_iter
*new, struct iov_iter
*old
, gfp_t flags
)
1632 if (unlikely(iov_iter_is_pipe(new))) {
1636 if (unlikely(iov_iter_is_discard(new)))
1638 if (iov_iter_is_bvec(new))
1639 return new->bvec
= kmemdup(new->bvec
,
1640 new->nr_segs
* sizeof(struct bio_vec
),
1643 /* iovec and kvec have identical layout */
1644 return new->iov
= kmemdup(new->iov
,
1645 new->nr_segs
* sizeof(struct iovec
),
1648 EXPORT_SYMBOL(dup_iter
);
1651 * import_iovec() - Copy an array of &struct iovec from userspace
1652 * into the kernel, check that it is valid, and initialize a new
1653 * &struct iov_iter iterator to access it.
1655 * @type: One of %READ or %WRITE.
1656 * @uvector: Pointer to the userspace array.
1657 * @nr_segs: Number of elements in userspace array.
1658 * @fast_segs: Number of elements in @iov.
1659 * @iov: (input and output parameter) Pointer to pointer to (usually small
1660 * on-stack) kernel array.
1661 * @i: Pointer to iterator that will be initialized on success.
1663 * If the array pointed to by *@iov is large enough to hold all @nr_segs,
1664 * then this function places %NULL in *@iov on return. Otherwise, a new
1665 * array will be allocated and the result placed in *@iov. This means that
1666 * the caller may call kfree() on *@iov regardless of whether the small
1667 * on-stack array was used or not (and regardless of whether this function
1668 * returns an error or not).
1670 * Return: Negative error code on error, bytes imported on success
1672 ssize_t
import_iovec(int type
, const struct iovec __user
* uvector
,
1673 unsigned nr_segs
, unsigned fast_segs
,
1674 struct iovec
**iov
, struct iov_iter
*i
)
1678 n
= rw_copy_check_uvector(type
, uvector
, nr_segs
, fast_segs
,
1686 iov_iter_init(i
, type
, p
, nr_segs
, n
);
1687 *iov
= p
== *iov
? NULL
: p
;
1690 EXPORT_SYMBOL(import_iovec
);
1692 #ifdef CONFIG_COMPAT
1693 #include <linux/compat.h>
1695 ssize_t
compat_import_iovec(int type
,
1696 const struct compat_iovec __user
* uvector
,
1697 unsigned nr_segs
, unsigned fast_segs
,
1698 struct iovec
**iov
, struct iov_iter
*i
)
1702 n
= compat_rw_copy_check_uvector(type
, uvector
, nr_segs
, fast_segs
,
1710 iov_iter_init(i
, type
, p
, nr_segs
, n
);
1711 *iov
= p
== *iov
? NULL
: p
;
1714 EXPORT_SYMBOL(compat_import_iovec
);
1717 int import_single_range(int rw
, void __user
*buf
, size_t len
,
1718 struct iovec
*iov
, struct iov_iter
*i
)
1720 if (len
> MAX_RW_COUNT
)
1722 if (unlikely(!access_ok(buf
, len
)))
1725 iov
->iov_base
= buf
;
1727 iov_iter_init(i
, rw
, iov
, 1, len
);
1730 EXPORT_SYMBOL(import_single_range
);
1732 int iov_iter_for_each_range(struct iov_iter
*i
, size_t bytes
,
1733 int (*f
)(struct kvec
*vec
, void *context
),
1741 iterate_all_kinds(i
, bytes
, v
, -EINVAL
, ({
1742 w
.iov_base
= kmap(v
.bv_page
) + v
.bv_offset
;
1743 w
.iov_len
= v
.bv_len
;
1744 err
= f(&w
, context
);
1748 err
= f(&w
, context
);})
1752 EXPORT_SYMBOL(iov_iter_for_each_range
);