1 #include <linux/export.h>
2 #include <linux/bvec.h>
4 #include <linux/pagemap.h>
5 #include <linux/slab.h>
6 #include <linux/vmalloc.h>
7 #include <linux/splice.h>
8 #include <net/checksum.h>
10 #define PIPE_PARANOIA /* for now */
12 #define iterate_iovec(i, n, __v, __p, skip, STEP) { \
16 __v.iov_len = min(n, __p->iov_len - skip); \
17 if (likely(__v.iov_len)) { \
18 __v.iov_base = __p->iov_base + skip; \
20 __v.iov_len -= left; \
21 skip += __v.iov_len; \
26 while (unlikely(!left && n)) { \
28 __v.iov_len = min(n, __p->iov_len); \
29 if (unlikely(!__v.iov_len)) \
31 __v.iov_base = __p->iov_base; \
33 __v.iov_len -= left; \
40 #define iterate_kvec(i, n, __v, __p, skip, STEP) { \
43 __v.iov_len = min(n, __p->iov_len - skip); \
44 if (likely(__v.iov_len)) { \
45 __v.iov_base = __p->iov_base + skip; \
47 skip += __v.iov_len; \
50 while (unlikely(n)) { \
52 __v.iov_len = min(n, __p->iov_len); \
53 if (unlikely(!__v.iov_len)) \
55 __v.iov_base = __p->iov_base; \
63 #define iterate_bvec(i, n, __v, __bi, skip, STEP) { \
64 struct bvec_iter __start; \
65 __start.bi_size = n; \
66 __start.bi_bvec_done = skip; \
68 for_each_bvec(__v, i->bvec, __bi, __start) { \
75 #define iterate_all_kinds(i, n, v, I, B, K) { \
77 size_t skip = i->iov_offset; \
78 if (unlikely(i->type & ITER_BVEC)) { \
80 struct bvec_iter __bi; \
81 iterate_bvec(i, n, v, __bi, skip, (B)) \
82 } else if (unlikely(i->type & ITER_KVEC)) { \
83 const struct kvec *kvec; \
85 iterate_kvec(i, n, v, kvec, skip, (K)) \
87 const struct iovec *iov; \
89 iterate_iovec(i, n, v, iov, skip, (I)) \
94 #define iterate_and_advance(i, n, v, I, B, K) { \
95 if (unlikely(i->count < n)) \
98 size_t skip = i->iov_offset; \
99 if (unlikely(i->type & ITER_BVEC)) { \
100 const struct bio_vec *bvec = i->bvec; \
102 struct bvec_iter __bi; \
103 iterate_bvec(i, n, v, __bi, skip, (B)) \
104 i->bvec = __bvec_iter_bvec(i->bvec, __bi); \
105 i->nr_segs -= i->bvec - bvec; \
106 skip = __bi.bi_bvec_done; \
107 } else if (unlikely(i->type & ITER_KVEC)) { \
108 const struct kvec *kvec; \
110 iterate_kvec(i, n, v, kvec, skip, (K)) \
111 if (skip == kvec->iov_len) { \
115 i->nr_segs -= kvec - i->kvec; \
118 const struct iovec *iov; \
120 iterate_iovec(i, n, v, iov, skip, (I)) \
121 if (skip == iov->iov_len) { \
125 i->nr_segs -= iov - i->iov; \
129 i->iov_offset = skip; \
133 static size_t copy_page_to_iter_iovec(struct page
*page
, size_t offset
, size_t bytes
,
136 size_t skip
, copy
, left
, wanted
;
137 const struct iovec
*iov
;
141 if (unlikely(bytes
> i
->count
))
144 if (unlikely(!bytes
))
149 skip
= i
->iov_offset
;
150 buf
= iov
->iov_base
+ skip
;
151 copy
= min(bytes
, iov
->iov_len
- skip
);
153 if (IS_ENABLED(CONFIG_HIGHMEM
) && !fault_in_pages_writeable(buf
, copy
)) {
154 kaddr
= kmap_atomic(page
);
155 from
= kaddr
+ offset
;
157 /* first chunk, usually the only one */
158 left
= __copy_to_user_inatomic(buf
, from
, copy
);
164 while (unlikely(!left
&& bytes
)) {
167 copy
= min(bytes
, iov
->iov_len
);
168 left
= __copy_to_user_inatomic(buf
, from
, copy
);
174 if (likely(!bytes
)) {
175 kunmap_atomic(kaddr
);
178 offset
= from
- kaddr
;
180 kunmap_atomic(kaddr
);
181 copy
= min(bytes
, iov
->iov_len
- skip
);
183 /* Too bad - revert to non-atomic kmap */
186 from
= kaddr
+ offset
;
187 left
= __copy_to_user(buf
, from
, copy
);
192 while (unlikely(!left
&& bytes
)) {
195 copy
= min(bytes
, iov
->iov_len
);
196 left
= __copy_to_user(buf
, from
, copy
);
205 if (skip
== iov
->iov_len
) {
209 i
->count
-= wanted
- bytes
;
210 i
->nr_segs
-= iov
- i
->iov
;
212 i
->iov_offset
= skip
;
213 return wanted
- bytes
;
216 static size_t copy_page_from_iter_iovec(struct page
*page
, size_t offset
, size_t bytes
,
219 size_t skip
, copy
, left
, wanted
;
220 const struct iovec
*iov
;
224 if (unlikely(bytes
> i
->count
))
227 if (unlikely(!bytes
))
232 skip
= i
->iov_offset
;
233 buf
= iov
->iov_base
+ skip
;
234 copy
= min(bytes
, iov
->iov_len
- skip
);
236 if (IS_ENABLED(CONFIG_HIGHMEM
) && !fault_in_pages_readable(buf
, copy
)) {
237 kaddr
= kmap_atomic(page
);
240 /* first chunk, usually the only one */
241 left
= __copy_from_user_inatomic(to
, buf
, copy
);
247 while (unlikely(!left
&& bytes
)) {
250 copy
= min(bytes
, iov
->iov_len
);
251 left
= __copy_from_user_inatomic(to
, buf
, copy
);
257 if (likely(!bytes
)) {
258 kunmap_atomic(kaddr
);
263 kunmap_atomic(kaddr
);
264 copy
= min(bytes
, iov
->iov_len
- skip
);
266 /* Too bad - revert to non-atomic kmap */
270 left
= __copy_from_user(to
, buf
, copy
);
275 while (unlikely(!left
&& bytes
)) {
278 copy
= min(bytes
, iov
->iov_len
);
279 left
= __copy_from_user(to
, buf
, copy
);
288 if (skip
== iov
->iov_len
) {
292 i
->count
-= wanted
- bytes
;
293 i
->nr_segs
-= iov
- i
->iov
;
295 i
->iov_offset
= skip
;
296 return wanted
- bytes
;
300 static bool sanity(const struct iov_iter
*i
)
302 struct pipe_inode_info
*pipe
= i
->pipe
;
304 int next
= pipe
->curbuf
+ pipe
->nrbufs
;
306 struct pipe_buffer
*p
;
307 if (unlikely(!pipe
->nrbufs
))
308 goto Bad
; // pipe must be non-empty
309 if (unlikely(idx
!= ((next
- 1) & (pipe
->buffers
- 1))))
310 goto Bad
; // must be at the last buffer...
312 p
= &pipe
->bufs
[idx
];
313 if (unlikely(p
->offset
+ p
->len
!= i
->iov_offset
))
314 goto Bad
; // ... at the end of segment
316 if (idx
!= (next
& (pipe
->buffers
- 1)))
317 goto Bad
; // must be right after the last buffer
321 printk(KERN_ERR
"idx = %d, offset = %zd\n", i
->idx
, i
->iov_offset
);
322 printk(KERN_ERR
"curbuf = %d, nrbufs = %d, buffers = %d\n",
323 pipe
->curbuf
, pipe
->nrbufs
, pipe
->buffers
);
324 for (idx
= 0; idx
< pipe
->buffers
; idx
++)
325 printk(KERN_ERR
"[%p %p %d %d]\n",
327 pipe
->bufs
[idx
].page
,
328 pipe
->bufs
[idx
].offset
,
329 pipe
->bufs
[idx
].len
);
334 #define sanity(i) true
337 static inline int next_idx(int idx
, struct pipe_inode_info
*pipe
)
339 return (idx
+ 1) & (pipe
->buffers
- 1);
342 static size_t copy_page_to_iter_pipe(struct page
*page
, size_t offset
, size_t bytes
,
345 struct pipe_inode_info
*pipe
= i
->pipe
;
346 struct pipe_buffer
*buf
;
350 if (unlikely(bytes
> i
->count
))
353 if (unlikely(!bytes
))
361 buf
= &pipe
->bufs
[idx
];
363 if (offset
== off
&& buf
->page
== page
) {
364 /* merge with the last one */
366 i
->iov_offset
+= bytes
;
369 idx
= next_idx(idx
, pipe
);
370 buf
= &pipe
->bufs
[idx
];
372 if (idx
== pipe
->curbuf
&& pipe
->nrbufs
)
375 buf
->ops
= &page_cache_pipe_buf_ops
;
376 get_page(buf
->page
= page
);
377 buf
->offset
= offset
;
379 i
->iov_offset
= offset
+ bytes
;
387 * Fault in one or more iovecs of the given iov_iter, to a maximum length of
388 * bytes. For each iovec, fault in each page that constitutes the iovec.
390 * Return 0 on success, or non-zero if the memory could not be accessed (i.e.
391 * because it is an invalid address).
393 int iov_iter_fault_in_readable(struct iov_iter
*i
, size_t bytes
)
395 size_t skip
= i
->iov_offset
;
396 const struct iovec
*iov
;
400 if (!(i
->type
& (ITER_BVEC
|ITER_KVEC
))) {
401 iterate_iovec(i
, bytes
, v
, iov
, skip
, ({
402 err
= fault_in_pages_readable(v
.iov_base
, v
.iov_len
);
409 EXPORT_SYMBOL(iov_iter_fault_in_readable
);
411 void iov_iter_init(struct iov_iter
*i
, int direction
,
412 const struct iovec
*iov
, unsigned long nr_segs
,
415 /* It will get better. Eventually... */
416 if (uaccess_kernel()) {
417 direction
|= ITER_KVEC
;
419 i
->kvec
= (struct kvec
*)iov
;
424 i
->nr_segs
= nr_segs
;
428 EXPORT_SYMBOL(iov_iter_init
);
430 static void memcpy_from_page(char *to
, struct page
*page
, size_t offset
, size_t len
)
432 char *from
= kmap_atomic(page
);
433 memcpy(to
, from
+ offset
, len
);
437 static void memcpy_to_page(struct page
*page
, size_t offset
, const char *from
, size_t len
)
439 char *to
= kmap_atomic(page
);
440 memcpy(to
+ offset
, from
, len
);
444 static void memzero_page(struct page
*page
, size_t offset
, size_t len
)
446 char *addr
= kmap_atomic(page
);
447 memset(addr
+ offset
, 0, len
);
451 static inline bool allocated(struct pipe_buffer
*buf
)
453 return buf
->ops
== &default_pipe_buf_ops
;
456 static inline void data_start(const struct iov_iter
*i
, int *idxp
, size_t *offp
)
458 size_t off
= i
->iov_offset
;
460 if (off
&& (!allocated(&i
->pipe
->bufs
[idx
]) || off
== PAGE_SIZE
)) {
461 idx
= next_idx(idx
, i
->pipe
);
468 static size_t push_pipe(struct iov_iter
*i
, size_t size
,
469 int *idxp
, size_t *offp
)
471 struct pipe_inode_info
*pipe
= i
->pipe
;
476 if (unlikely(size
> i
->count
))
482 data_start(i
, &idx
, &off
);
486 left
-= PAGE_SIZE
- off
;
488 pipe
->bufs
[idx
].len
+= size
;
491 pipe
->bufs
[idx
].len
= PAGE_SIZE
;
492 idx
= next_idx(idx
, pipe
);
494 while (idx
!= pipe
->curbuf
|| !pipe
->nrbufs
) {
495 struct page
*page
= alloc_page(GFP_USER
);
499 pipe
->bufs
[idx
].ops
= &default_pipe_buf_ops
;
500 pipe
->bufs
[idx
].page
= page
;
501 pipe
->bufs
[idx
].offset
= 0;
502 if (left
<= PAGE_SIZE
) {
503 pipe
->bufs
[idx
].len
= left
;
506 pipe
->bufs
[idx
].len
= PAGE_SIZE
;
508 idx
= next_idx(idx
, pipe
);
513 static size_t copy_pipe_to_iter(const void *addr
, size_t bytes
,
516 struct pipe_inode_info
*pipe
= i
->pipe
;
523 bytes
= n
= push_pipe(i
, bytes
, &idx
, &off
);
526 for ( ; n
; idx
= next_idx(idx
, pipe
), off
= 0) {
527 size_t chunk
= min_t(size_t, n
, PAGE_SIZE
- off
);
528 memcpy_to_page(pipe
->bufs
[idx
].page
, off
, addr
, chunk
);
530 i
->iov_offset
= off
+ chunk
;
538 size_t copy_to_iter(const void *addr
, size_t bytes
, struct iov_iter
*i
)
540 const char *from
= addr
;
541 if (unlikely(i
->type
& ITER_PIPE
))
542 return copy_pipe_to_iter(addr
, bytes
, i
);
543 iterate_and_advance(i
, bytes
, v
,
544 __copy_to_user(v
.iov_base
, (from
+= v
.iov_len
) - v
.iov_len
,
546 memcpy_to_page(v
.bv_page
, v
.bv_offset
,
547 (from
+= v
.bv_len
) - v
.bv_len
, v
.bv_len
),
548 memcpy(v
.iov_base
, (from
+= v
.iov_len
) - v
.iov_len
, v
.iov_len
)
553 EXPORT_SYMBOL(copy_to_iter
);
555 size_t copy_from_iter(void *addr
, size_t bytes
, struct iov_iter
*i
)
558 if (unlikely(i
->type
& ITER_PIPE
)) {
562 iterate_and_advance(i
, bytes
, v
,
563 __copy_from_user((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
,
565 memcpy_from_page((to
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
566 v
.bv_offset
, v
.bv_len
),
567 memcpy((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
)
572 EXPORT_SYMBOL(copy_from_iter
);
574 bool copy_from_iter_full(void *addr
, size_t bytes
, struct iov_iter
*i
)
577 if (unlikely(i
->type
& ITER_PIPE
)) {
581 if (unlikely(i
->count
< bytes
))
584 iterate_all_kinds(i
, bytes
, v
, ({
585 if (__copy_from_user((to
+= v
.iov_len
) - v
.iov_len
,
586 v
.iov_base
, v
.iov_len
))
589 memcpy_from_page((to
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
590 v
.bv_offset
, v
.bv_len
),
591 memcpy((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
)
594 iov_iter_advance(i
, bytes
);
597 EXPORT_SYMBOL(copy_from_iter_full
);
599 size_t copy_from_iter_nocache(void *addr
, size_t bytes
, struct iov_iter
*i
)
602 if (unlikely(i
->type
& ITER_PIPE
)) {
606 iterate_and_advance(i
, bytes
, v
,
607 __copy_from_user_inatomic_nocache((to
+= v
.iov_len
) - v
.iov_len
,
608 v
.iov_base
, v
.iov_len
),
609 memcpy_from_page((to
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
610 v
.bv_offset
, v
.bv_len
),
611 memcpy((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
)
616 EXPORT_SYMBOL(copy_from_iter_nocache
);
618 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
619 size_t copy_from_iter_flushcache(void *addr
, size_t bytes
, struct iov_iter
*i
)
622 if (unlikely(i
->type
& ITER_PIPE
)) {
626 iterate_and_advance(i
, bytes
, v
,
627 __copy_from_user_flushcache((to
+= v
.iov_len
) - v
.iov_len
,
628 v
.iov_base
, v
.iov_len
),
629 memcpy_page_flushcache((to
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
630 v
.bv_offset
, v
.bv_len
),
631 memcpy_flushcache((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
,
637 EXPORT_SYMBOL_GPL(copy_from_iter_flushcache
);
640 bool copy_from_iter_full_nocache(void *addr
, size_t bytes
, struct iov_iter
*i
)
643 if (unlikely(i
->type
& ITER_PIPE
)) {
647 if (unlikely(i
->count
< bytes
))
649 iterate_all_kinds(i
, bytes
, v
, ({
650 if (__copy_from_user_inatomic_nocache((to
+= v
.iov_len
) - v
.iov_len
,
651 v
.iov_base
, v
.iov_len
))
654 memcpy_from_page((to
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
655 v
.bv_offset
, v
.bv_len
),
656 memcpy((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
)
659 iov_iter_advance(i
, bytes
);
662 EXPORT_SYMBOL(copy_from_iter_full_nocache
);
664 size_t copy_page_to_iter(struct page
*page
, size_t offset
, size_t bytes
,
667 if (i
->type
& (ITER_BVEC
|ITER_KVEC
)) {
668 void *kaddr
= kmap_atomic(page
);
669 size_t wanted
= copy_to_iter(kaddr
+ offset
, bytes
, i
);
670 kunmap_atomic(kaddr
);
672 } else if (likely(!(i
->type
& ITER_PIPE
)))
673 return copy_page_to_iter_iovec(page
, offset
, bytes
, i
);
675 return copy_page_to_iter_pipe(page
, offset
, bytes
, i
);
677 EXPORT_SYMBOL(copy_page_to_iter
);
679 size_t copy_page_from_iter(struct page
*page
, size_t offset
, size_t bytes
,
682 if (unlikely(i
->type
& ITER_PIPE
)) {
686 if (i
->type
& (ITER_BVEC
|ITER_KVEC
)) {
687 void *kaddr
= kmap_atomic(page
);
688 size_t wanted
= copy_from_iter(kaddr
+ offset
, bytes
, i
);
689 kunmap_atomic(kaddr
);
692 return copy_page_from_iter_iovec(page
, offset
, bytes
, i
);
694 EXPORT_SYMBOL(copy_page_from_iter
);
696 static size_t pipe_zero(size_t bytes
, struct iov_iter
*i
)
698 struct pipe_inode_info
*pipe
= i
->pipe
;
705 bytes
= n
= push_pipe(i
, bytes
, &idx
, &off
);
709 for ( ; n
; idx
= next_idx(idx
, pipe
), off
= 0) {
710 size_t chunk
= min_t(size_t, n
, PAGE_SIZE
- off
);
711 memzero_page(pipe
->bufs
[idx
].page
, off
, chunk
);
713 i
->iov_offset
= off
+ chunk
;
720 size_t iov_iter_zero(size_t bytes
, struct iov_iter
*i
)
722 if (unlikely(i
->type
& ITER_PIPE
))
723 return pipe_zero(bytes
, i
);
724 iterate_and_advance(i
, bytes
, v
,
725 __clear_user(v
.iov_base
, v
.iov_len
),
726 memzero_page(v
.bv_page
, v
.bv_offset
, v
.bv_len
),
727 memset(v
.iov_base
, 0, v
.iov_len
)
732 EXPORT_SYMBOL(iov_iter_zero
);
734 size_t iov_iter_copy_from_user_atomic(struct page
*page
,
735 struct iov_iter
*i
, unsigned long offset
, size_t bytes
)
737 char *kaddr
= kmap_atomic(page
), *p
= kaddr
+ offset
;
738 if (unlikely(i
->type
& ITER_PIPE
)) {
739 kunmap_atomic(kaddr
);
743 iterate_all_kinds(i
, bytes
, v
,
744 __copy_from_user_inatomic((p
+= v
.iov_len
) - v
.iov_len
,
745 v
.iov_base
, v
.iov_len
),
746 memcpy_from_page((p
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
747 v
.bv_offset
, v
.bv_len
),
748 memcpy((p
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
)
750 kunmap_atomic(kaddr
);
753 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic
);
755 static inline void pipe_truncate(struct iov_iter
*i
)
757 struct pipe_inode_info
*pipe
= i
->pipe
;
759 size_t off
= i
->iov_offset
;
761 int nrbufs
= (idx
- pipe
->curbuf
) & (pipe
->buffers
- 1);
763 pipe
->bufs
[idx
].len
= off
- pipe
->bufs
[idx
].offset
;
764 idx
= next_idx(idx
, pipe
);
767 while (pipe
->nrbufs
> nrbufs
) {
768 pipe_buf_release(pipe
, &pipe
->bufs
[idx
]);
769 idx
= next_idx(idx
, pipe
);
775 static void pipe_advance(struct iov_iter
*i
, size_t size
)
777 struct pipe_inode_info
*pipe
= i
->pipe
;
778 if (unlikely(i
->count
< size
))
781 struct pipe_buffer
*buf
;
782 size_t off
= i
->iov_offset
, left
= size
;
784 if (off
) /* make it relative to the beginning of buffer */
785 left
+= off
- pipe
->bufs
[idx
].offset
;
787 buf
= &pipe
->bufs
[idx
];
788 if (left
<= buf
->len
)
791 idx
= next_idx(idx
, pipe
);
794 i
->iov_offset
= buf
->offset
+ left
;
797 /* ... and discard everything past that point */
801 void iov_iter_advance(struct iov_iter
*i
, size_t size
)
803 if (unlikely(i
->type
& ITER_PIPE
)) {
804 pipe_advance(i
, size
);
807 iterate_and_advance(i
, size
, v
, 0, 0, 0)
809 EXPORT_SYMBOL(iov_iter_advance
);
811 void iov_iter_revert(struct iov_iter
*i
, size_t unroll
)
815 if (WARN_ON(unroll
> MAX_RW_COUNT
))
818 if (unlikely(i
->type
& ITER_PIPE
)) {
819 struct pipe_inode_info
*pipe
= i
->pipe
;
821 size_t off
= i
->iov_offset
;
823 size_t n
= off
- pipe
->bufs
[idx
].offset
;
829 if (!unroll
&& idx
== i
->start_idx
) {
834 idx
= pipe
->buffers
- 1;
835 off
= pipe
->bufs
[idx
].offset
+ pipe
->bufs
[idx
].len
;
842 if (unroll
<= i
->iov_offset
) {
843 i
->iov_offset
-= unroll
;
846 unroll
-= i
->iov_offset
;
847 if (i
->type
& ITER_BVEC
) {
848 const struct bio_vec
*bvec
= i
->bvec
;
850 size_t n
= (--bvec
)->bv_len
;
854 i
->iov_offset
= n
- unroll
;
859 } else { /* same logics for iovec and kvec */
860 const struct iovec
*iov
= i
->iov
;
862 size_t n
= (--iov
)->iov_len
;
866 i
->iov_offset
= n
- unroll
;
873 EXPORT_SYMBOL(iov_iter_revert
);
876 * Return the count of just the current iov_iter segment.
878 size_t iov_iter_single_seg_count(const struct iov_iter
*i
)
880 if (unlikely(i
->type
& ITER_PIPE
))
881 return i
->count
; // it is a silly place, anyway
884 else if (i
->type
& ITER_BVEC
)
885 return min(i
->count
, i
->bvec
->bv_len
- i
->iov_offset
);
887 return min(i
->count
, i
->iov
->iov_len
- i
->iov_offset
);
889 EXPORT_SYMBOL(iov_iter_single_seg_count
);
891 void iov_iter_kvec(struct iov_iter
*i
, int direction
,
892 const struct kvec
*kvec
, unsigned long nr_segs
,
895 BUG_ON(!(direction
& ITER_KVEC
));
898 i
->nr_segs
= nr_segs
;
902 EXPORT_SYMBOL(iov_iter_kvec
);
904 void iov_iter_bvec(struct iov_iter
*i
, int direction
,
905 const struct bio_vec
*bvec
, unsigned long nr_segs
,
908 BUG_ON(!(direction
& ITER_BVEC
));
911 i
->nr_segs
= nr_segs
;
915 EXPORT_SYMBOL(iov_iter_bvec
);
917 void iov_iter_pipe(struct iov_iter
*i
, int direction
,
918 struct pipe_inode_info
*pipe
,
921 BUG_ON(direction
!= ITER_PIPE
);
922 WARN_ON(pipe
->nrbufs
== pipe
->buffers
);
925 i
->idx
= (pipe
->curbuf
+ pipe
->nrbufs
) & (pipe
->buffers
- 1);
928 i
->start_idx
= i
->idx
;
930 EXPORT_SYMBOL(iov_iter_pipe
);
932 unsigned long iov_iter_alignment(const struct iov_iter
*i
)
934 unsigned long res
= 0;
935 size_t size
= i
->count
;
937 if (unlikely(i
->type
& ITER_PIPE
)) {
938 if (size
&& i
->iov_offset
&& allocated(&i
->pipe
->bufs
[i
->idx
]))
939 return size
| i
->iov_offset
;
942 iterate_all_kinds(i
, size
, v
,
943 (res
|= (unsigned long)v
.iov_base
| v
.iov_len
, 0),
944 res
|= v
.bv_offset
| v
.bv_len
,
945 res
|= (unsigned long)v
.iov_base
| v
.iov_len
949 EXPORT_SYMBOL(iov_iter_alignment
);
951 unsigned long iov_iter_gap_alignment(const struct iov_iter
*i
)
953 unsigned long res
= 0;
954 size_t size
= i
->count
;
956 if (unlikely(i
->type
& ITER_PIPE
)) {
961 iterate_all_kinds(i
, size
, v
,
962 (res
|= (!res
? 0 : (unsigned long)v
.iov_base
) |
963 (size
!= v
.iov_len
? size
: 0), 0),
964 (res
|= (!res
? 0 : (unsigned long)v
.bv_offset
) |
965 (size
!= v
.bv_len
? size
: 0)),
966 (res
|= (!res
? 0 : (unsigned long)v
.iov_base
) |
967 (size
!= v
.iov_len
? size
: 0))
971 EXPORT_SYMBOL(iov_iter_gap_alignment
);
973 static inline size_t __pipe_get_pages(struct iov_iter
*i
,
979 struct pipe_inode_info
*pipe
= i
->pipe
;
980 ssize_t n
= push_pipe(i
, maxsize
, &idx
, start
);
987 get_page(*pages
++ = pipe
->bufs
[idx
].page
);
988 idx
= next_idx(idx
, pipe
);
995 static ssize_t
pipe_get_pages(struct iov_iter
*i
,
996 struct page
**pages
, size_t maxsize
, unsigned maxpages
,
1009 data_start(i
, &idx
, start
);
1010 /* some of this one + all after this one */
1011 npages
= ((i
->pipe
->curbuf
- idx
- 1) & (i
->pipe
->buffers
- 1)) + 1;
1012 capacity
= min(npages
,maxpages
) * PAGE_SIZE
- *start
;
1014 return __pipe_get_pages(i
, min(maxsize
, capacity
), pages
, idx
, start
);
1017 ssize_t
iov_iter_get_pages(struct iov_iter
*i
,
1018 struct page
**pages
, size_t maxsize
, unsigned maxpages
,
1021 if (maxsize
> i
->count
)
1024 if (unlikely(i
->type
& ITER_PIPE
))
1025 return pipe_get_pages(i
, pages
, maxsize
, maxpages
, start
);
1026 iterate_all_kinds(i
, maxsize
, v
, ({
1027 unsigned long addr
= (unsigned long)v
.iov_base
;
1028 size_t len
= v
.iov_len
+ (*start
= addr
& (PAGE_SIZE
- 1));
1032 if (len
> maxpages
* PAGE_SIZE
)
1033 len
= maxpages
* PAGE_SIZE
;
1034 addr
&= ~(PAGE_SIZE
- 1);
1035 n
= DIV_ROUND_UP(len
, PAGE_SIZE
);
1036 res
= get_user_pages_fast(addr
, n
, (i
->type
& WRITE
) != WRITE
, pages
);
1037 if (unlikely(res
< 0))
1039 return (res
== n
? len
: res
* PAGE_SIZE
) - *start
;
1041 /* can't be more than PAGE_SIZE */
1042 *start
= v
.bv_offset
;
1043 get_page(*pages
= v
.bv_page
);
1051 EXPORT_SYMBOL(iov_iter_get_pages
);
1053 static struct page
**get_pages_array(size_t n
)
1055 return kvmalloc_array(n
, sizeof(struct page
*), GFP_KERNEL
);
1058 static ssize_t
pipe_get_pages_alloc(struct iov_iter
*i
,
1059 struct page
***pages
, size_t maxsize
,
1073 data_start(i
, &idx
, start
);
1074 /* some of this one + all after this one */
1075 npages
= ((i
->pipe
->curbuf
- idx
- 1) & (i
->pipe
->buffers
- 1)) + 1;
1076 n
= npages
* PAGE_SIZE
- *start
;
1080 npages
= DIV_ROUND_UP(maxsize
+ *start
, PAGE_SIZE
);
1081 p
= get_pages_array(npages
);
1084 n
= __pipe_get_pages(i
, maxsize
, p
, idx
, start
);
1092 ssize_t
iov_iter_get_pages_alloc(struct iov_iter
*i
,
1093 struct page
***pages
, size_t maxsize
,
1098 if (maxsize
> i
->count
)
1101 if (unlikely(i
->type
& ITER_PIPE
))
1102 return pipe_get_pages_alloc(i
, pages
, maxsize
, start
);
1103 iterate_all_kinds(i
, maxsize
, v
, ({
1104 unsigned long addr
= (unsigned long)v
.iov_base
;
1105 size_t len
= v
.iov_len
+ (*start
= addr
& (PAGE_SIZE
- 1));
1109 addr
&= ~(PAGE_SIZE
- 1);
1110 n
= DIV_ROUND_UP(len
, PAGE_SIZE
);
1111 p
= get_pages_array(n
);
1114 res
= get_user_pages_fast(addr
, n
, (i
->type
& WRITE
) != WRITE
, p
);
1115 if (unlikely(res
< 0)) {
1120 return (res
== n
? len
: res
* PAGE_SIZE
) - *start
;
1122 /* can't be more than PAGE_SIZE */
1123 *start
= v
.bv_offset
;
1124 *pages
= p
= get_pages_array(1);
1127 get_page(*p
= v
.bv_page
);
1135 EXPORT_SYMBOL(iov_iter_get_pages_alloc
);
1137 size_t csum_and_copy_from_iter(void *addr
, size_t bytes
, __wsum
*csum
,
1144 if (unlikely(i
->type
& ITER_PIPE
)) {
1148 iterate_and_advance(i
, bytes
, v
, ({
1150 next
= csum_and_copy_from_user(v
.iov_base
,
1151 (to
+= v
.iov_len
) - v
.iov_len
,
1152 v
.iov_len
, 0, &err
);
1154 sum
= csum_block_add(sum
, next
, off
);
1157 err
? v
.iov_len
: 0;
1159 char *p
= kmap_atomic(v
.bv_page
);
1160 next
= csum_partial_copy_nocheck(p
+ v
.bv_offset
,
1161 (to
+= v
.bv_len
) - v
.bv_len
,
1164 sum
= csum_block_add(sum
, next
, off
);
1167 next
= csum_partial_copy_nocheck(v
.iov_base
,
1168 (to
+= v
.iov_len
) - v
.iov_len
,
1170 sum
= csum_block_add(sum
, next
, off
);
1177 EXPORT_SYMBOL(csum_and_copy_from_iter
);
1179 bool csum_and_copy_from_iter_full(void *addr
, size_t bytes
, __wsum
*csum
,
1186 if (unlikely(i
->type
& ITER_PIPE
)) {
1190 if (unlikely(i
->count
< bytes
))
1192 iterate_all_kinds(i
, bytes
, v
, ({
1194 next
= csum_and_copy_from_user(v
.iov_base
,
1195 (to
+= v
.iov_len
) - v
.iov_len
,
1196 v
.iov_len
, 0, &err
);
1199 sum
= csum_block_add(sum
, next
, off
);
1203 char *p
= kmap_atomic(v
.bv_page
);
1204 next
= csum_partial_copy_nocheck(p
+ v
.bv_offset
,
1205 (to
+= v
.bv_len
) - v
.bv_len
,
1208 sum
= csum_block_add(sum
, next
, off
);
1211 next
= csum_partial_copy_nocheck(v
.iov_base
,
1212 (to
+= v
.iov_len
) - v
.iov_len
,
1214 sum
= csum_block_add(sum
, next
, off
);
1219 iov_iter_advance(i
, bytes
);
1222 EXPORT_SYMBOL(csum_and_copy_from_iter_full
);
1224 size_t csum_and_copy_to_iter(const void *addr
, size_t bytes
, __wsum
*csum
,
1227 const char *from
= addr
;
1231 if (unlikely(i
->type
& ITER_PIPE
)) {
1232 WARN_ON(1); /* for now */
1235 iterate_and_advance(i
, bytes
, v
, ({
1237 next
= csum_and_copy_to_user((from
+= v
.iov_len
) - v
.iov_len
,
1239 v
.iov_len
, 0, &err
);
1241 sum
= csum_block_add(sum
, next
, off
);
1244 err
? v
.iov_len
: 0;
1246 char *p
= kmap_atomic(v
.bv_page
);
1247 next
= csum_partial_copy_nocheck((from
+= v
.bv_len
) - v
.bv_len
,
1251 sum
= csum_block_add(sum
, next
, off
);
1254 next
= csum_partial_copy_nocheck((from
+= v
.iov_len
) - v
.iov_len
,
1257 sum
= csum_block_add(sum
, next
, off
);
1264 EXPORT_SYMBOL(csum_and_copy_to_iter
);
1266 int iov_iter_npages(const struct iov_iter
*i
, int maxpages
)
1268 size_t size
= i
->count
;
1274 if (unlikely(i
->type
& ITER_PIPE
)) {
1275 struct pipe_inode_info
*pipe
= i
->pipe
;
1282 data_start(i
, &idx
, &off
);
1283 /* some of this one + all after this one */
1284 npages
= ((pipe
->curbuf
- idx
- 1) & (pipe
->buffers
- 1)) + 1;
1285 if (npages
>= maxpages
)
1287 } else iterate_all_kinds(i
, size
, v
, ({
1288 unsigned long p
= (unsigned long)v
.iov_base
;
1289 npages
+= DIV_ROUND_UP(p
+ v
.iov_len
, PAGE_SIZE
)
1291 if (npages
>= maxpages
)
1295 if (npages
>= maxpages
)
1298 unsigned long p
= (unsigned long)v
.iov_base
;
1299 npages
+= DIV_ROUND_UP(p
+ v
.iov_len
, PAGE_SIZE
)
1301 if (npages
>= maxpages
)
1307 EXPORT_SYMBOL(iov_iter_npages
);
1309 const void *dup_iter(struct iov_iter
*new, struct iov_iter
*old
, gfp_t flags
)
1312 if (unlikely(new->type
& ITER_PIPE
)) {
1316 if (new->type
& ITER_BVEC
)
1317 return new->bvec
= kmemdup(new->bvec
,
1318 new->nr_segs
* sizeof(struct bio_vec
),
1321 /* iovec and kvec have identical layout */
1322 return new->iov
= kmemdup(new->iov
,
1323 new->nr_segs
* sizeof(struct iovec
),
1326 EXPORT_SYMBOL(dup_iter
);
1329 * import_iovec() - Copy an array of &struct iovec from userspace
1330 * into the kernel, check that it is valid, and initialize a new
1331 * &struct iov_iter iterator to access it.
1333 * @type: One of %READ or %WRITE.
1334 * @uvector: Pointer to the userspace array.
1335 * @nr_segs: Number of elements in userspace array.
1336 * @fast_segs: Number of elements in @iov.
1337 * @iov: (input and output parameter) Pointer to pointer to (usually small
1338 * on-stack) kernel array.
1339 * @i: Pointer to iterator that will be initialized on success.
1341 * If the array pointed to by *@iov is large enough to hold all @nr_segs,
1342 * then this function places %NULL in *@iov on return. Otherwise, a new
1343 * array will be allocated and the result placed in *@iov. This means that
1344 * the caller may call kfree() on *@iov regardless of whether the small
1345 * on-stack array was used or not (and regardless of whether this function
1346 * returns an error or not).
1348 * Return: 0 on success or negative error code on error.
1350 int import_iovec(int type
, const struct iovec __user
* uvector
,
1351 unsigned nr_segs
, unsigned fast_segs
,
1352 struct iovec
**iov
, struct iov_iter
*i
)
1356 n
= rw_copy_check_uvector(type
, uvector
, nr_segs
, fast_segs
,
1364 iov_iter_init(i
, type
, p
, nr_segs
, n
);
1365 *iov
= p
== *iov
? NULL
: p
;
1368 EXPORT_SYMBOL(import_iovec
);
1370 #ifdef CONFIG_COMPAT
1371 #include <linux/compat.h>
1373 int compat_import_iovec(int type
, const struct compat_iovec __user
* uvector
,
1374 unsigned nr_segs
, unsigned fast_segs
,
1375 struct iovec
**iov
, struct iov_iter
*i
)
1379 n
= compat_rw_copy_check_uvector(type
, uvector
, nr_segs
, fast_segs
,
1387 iov_iter_init(i
, type
, p
, nr_segs
, n
);
1388 *iov
= p
== *iov
? NULL
: p
;
1393 int import_single_range(int rw
, void __user
*buf
, size_t len
,
1394 struct iovec
*iov
, struct iov_iter
*i
)
1396 if (len
> MAX_RW_COUNT
)
1398 if (unlikely(!access_ok(!rw
, buf
, len
)))
1401 iov
->iov_base
= buf
;
1403 iov_iter_init(i
, rw
, iov
, 1, len
);
1406 EXPORT_SYMBOL(import_single_range
);