Merge git://git.kernel.org/pub/scm/linux/kernel/git/pablo/nf

[mirror_ubuntu-artful-kernel.git] / lib / iov_iter.c

diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index 228892dabba6f5579478988b84ef007fa04bd661..e68604ae3cedf41ce98bc06de2142629fa115cbd 100644 (file)
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -73,19 +73,21 @@
 }
 
 #define iterate_all_kinds(i, n, v, I, B, K) {                  \
-       size_t skip = i->iov_offset;                            \
-       if (unlikely(i->type & ITER_BVEC)) {                    \
-               struct bio_vec v;                               \
-               struct bvec_iter __bi;                          \
-               iterate_bvec(i, n, v, __bi, skip, (B))          \
-       } else if (unlikely(i->type & ITER_KVEC)) {             \
-               const struct kvec *kvec;                        \
-               struct kvec v;                                  \
-               iterate_kvec(i, n, v, kvec, skip, (K))          \
-       } else {                                                \
-               const struct iovec *iov;                        \
-               struct iovec v;                                 \
-               iterate_iovec(i, n, v, iov, skip, (I))          \
+       if (likely(n)) {                                        \
+               size_t skip = i->iov_offset;                    \
+               if (unlikely(i->type & ITER_BVEC)) {            \
+                       struct bio_vec v;                       \
+                       struct bvec_iter __bi;                  \
+                       iterate_bvec(i, n, v, __bi, skip, (B))  \
+               } else if (unlikely(i->type & ITER_KVEC)) {     \
+                       const struct kvec *kvec;                \
+                       struct kvec v;                          \
+                       iterate_kvec(i, n, v, kvec, skip, (K))  \
+               } else {                                        \
+                       const struct iovec *iov;                \
+                       struct iovec v;                         \
+                       iterate_iovec(i, n, v, iov, skip, (I))  \
+               }                                               \
        }                                                       \
 }
 
@@ -576,7 +578,7 @@ bool copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)
                WARN_ON(1);
                return false;
        }
-       if (unlikely(i->count < bytes))                         \
+       if (unlikely(i->count < bytes))
                return false;
 
        iterate_all_kinds(i, bytes, v, ({
@@ -620,7 +622,7 @@ bool copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)
                WARN_ON(1);
                return false;
        }
-       if (unlikely(i->count < bytes))                         \
+       if (unlikely(i->count < bytes))
                return false;
        iterate_all_kinds(i, bytes, v, ({
                if (__copy_from_user_nocache((to += v.iov_len) - v.iov_len,
@@ -728,43 +730,50 @@ size_t iov_iter_copy_from_user_atomic(struct page *page,
 }
 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
 
+static inline void pipe_truncate(struct iov_iter *i)
+{
+       struct pipe_inode_info *pipe = i->pipe;
+       if (pipe->nrbufs) {
+               size_t off = i->iov_offset;
+               int idx = i->idx;
+               int nrbufs = (idx - pipe->curbuf) & (pipe->buffers - 1);
+               if (off) {
+                       pipe->bufs[idx].len = off - pipe->bufs[idx].offset;
+                       idx = next_idx(idx, pipe);
+                       nrbufs++;
+               }
+               while (pipe->nrbufs > nrbufs) {
+                       pipe_buf_release(pipe, &pipe->bufs[idx]);
+                       idx = next_idx(idx, pipe);
+                       pipe->nrbufs--;
+               }
+       }
+}
+
 static void pipe_advance(struct iov_iter *i, size_t size)
 {
        struct pipe_inode_info *pipe = i->pipe;
-       struct pipe_buffer *buf;
-       int idx = i->idx;
-       size_t off = i->iov_offset, orig_sz;
-       
        if (unlikely(i->count < size))
                size = i->count;
-       orig_sz = size;
-
        if (size) {
+               struct pipe_buffer *buf;
+               size_t off = i->iov_offset, left = size;
+               int idx = i->idx;
                if (off) /* make it relative to the beginning of buffer */
-                       size += off - pipe->bufs[idx].offset;
+                       left += off - pipe->bufs[idx].offset;
                while (1) {
                        buf = &pipe->bufs[idx];
-                       if (size <= buf->len)
+                       if (left <= buf->len)
                                break;
-                       size -= buf->len;
+                       left -= buf->len;
                        idx = next_idx(idx, pipe);
                }
-               buf->len = size;
                i->idx = idx;
-               off = i->iov_offset = buf->offset + size;
-       }
-       if (off)
-               idx = next_idx(idx, pipe);
-       if (pipe->nrbufs) {
-               int unused = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
-               /* [curbuf,unused) is in use.  Free [idx,unused) */
-               while (idx != unused) {
-                       pipe_buf_release(pipe, &pipe->bufs[idx]);
-                       idx = next_idx(idx, pipe);
-                       pipe->nrbufs--;
-               }
+               i->iov_offset = buf->offset + left;
        }
-       i->count -= orig_sz;
+       i->count -= size;
+       /* ... and discard everything past that point */
+       pipe_truncate(i);
 }
 
 void iov_iter_advance(struct iov_iter *i, size_t size)
@@ -824,6 +833,7 @@ void iov_iter_pipe(struct iov_iter *i, int direction,
                        size_t count)
 {
        BUG_ON(direction != ITER_PIPE);
+       WARN_ON(pipe->nrbufs == pipe->buffers);
        i->type = direction;
        i->pipe = pipe;
        i->idx = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
@@ -837,11 +847,8 @@ unsigned long iov_iter_alignment(const struct iov_iter *i)
        unsigned long res = 0;
        size_t size = i->count;
 
-       if (!size)
-               return 0;
-
        if (unlikely(i->type & ITER_PIPE)) {
-               if (i->iov_offset && allocated(&i->pipe->bufs[i->idx]))
+               if (size && i->iov_offset && allocated(&i->pipe->bufs[i->idx]))
                        return size | i->iov_offset;
                return size;
        }
@@ -856,10 +863,8 @@ EXPORT_SYMBOL(iov_iter_alignment);
 
 unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
 {
-        unsigned long res = 0;
+       unsigned long res = 0;
        size_t size = i->count;
-       if (!size)
-               return 0;
 
        if (unlikely(i->type & ITER_PIPE)) {
                WARN_ON(1);
@@ -874,7 +879,7 @@ unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
                (res |= (!res ? 0 : (unsigned long)v.iov_base) |
                        (size != v.iov_len ? size : 0))
                );
-               return res;
+       return res;
 }
 EXPORT_SYMBOL(iov_iter_gap_alignment);
 
@@ -908,6 +913,9 @@ static ssize_t pipe_get_pages(struct iov_iter *i,
        size_t capacity;
        int idx;
 
+       if (!maxsize)
+               return 0;
+
        if (!sanity(i))
                return -EFAULT;
 
@@ -926,9 +934,6 @@ ssize_t iov_iter_get_pages(struct iov_iter *i,
        if (maxsize > i->count)
                maxsize = i->count;
 
-       if (!maxsize)
-               return 0;
-
        if (unlikely(i->type & ITER_PIPE))
                return pipe_get_pages(i, pages, maxsize, maxpages, start);
        iterate_all_kinds(i, maxsize, v, ({
@@ -975,6 +980,9 @@ static ssize_t pipe_get_pages_alloc(struct iov_iter *i,
        int idx;
        int npages;
 
+       if (!maxsize)
+               return 0;
+
        if (!sanity(i))
                return -EFAULT;
 
@@ -1006,9 +1014,6 @@ ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
        if (maxsize > i->count)
                maxsize = i->count;
 
-       if (!maxsize)
-               return 0;
-
        if (unlikely(i->type & ITER_PIPE))
                return pipe_get_pages_alloc(i, pages, maxsize, start);
        iterate_all_kinds(i, maxsize, v, ({