[mirror_ubuntu-zesty-kernel.git] / mm / iov_iter.c

#include <linux/export.h>
#include <linux/uio.h>
#include <linux/pagemap.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>

size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
			 struct iov_iter *i)
{
	size_t skip, copy, left, wanted;
	const struct iovec *iov;
	char __user *buf;
	void *kaddr, *from;

	if (unlikely(bytes > i->count))
		bytes = i->count;

	if (unlikely(!bytes))
		return 0;

	wanted = bytes;
	iov = i->iov;
	skip = i->iov_offset;
	buf = iov->iov_base + skip;
	copy = min(bytes, iov->iov_len - skip);

	if (!fault_in_pages_writeable(buf, copy)) {
		kaddr = kmap_atomic(page);
		from = kaddr + offset;

		/* first chunk, usually the only one */
		left = __copy_to_user_inatomic(buf, from, copy);
		copy -= left;
		skip += copy;
		from += copy;
		bytes -= copy;

		while (unlikely(!left && bytes)) {
			iov++;
			buf = iov->iov_base;
			copy = min(bytes, iov->iov_len);
			left = __copy_to_user_inatomic(buf, from, copy);
			copy -= left;
			skip = copy;
			from += copy;
			bytes -= copy;
		}
		if (likely(!bytes)) {
			kunmap_atomic(kaddr);
			goto done;
		}
		offset = from - kaddr;
		buf += copy;
		kunmap_atomic(kaddr);
		copy = min(bytes, iov->iov_len - skip);
	}
	/* Too bad - revert to non-atomic kmap */
	kaddr = kmap(page);
	from = kaddr + offset;
	left = __copy_to_user(buf, from, copy);
	copy -= left;
	skip += copy;
	from += copy;
	bytes -= copy;
	while (unlikely(!left && bytes)) {
		iov++;
		buf = iov->iov_base;
		copy = min(bytes, iov->iov_len);
		left = __copy_to_user(buf, from, copy);
		copy -= left;
		skip = copy;
		from += copy;
		bytes -= copy;
	}
	kunmap(page);
done:
	i->count -= wanted - bytes;
	i->nr_segs -= iov - i->iov;
	i->iov = iov;
	i->iov_offset = skip;
	return wanted - bytes;
}
EXPORT_SYMBOL(copy_page_to_iter);

static size_t __iovec_copy_from_user_inatomic(char *vaddr,
			const struct iovec *iov, size_t base, size_t bytes)
{
	size_t copied = 0, left = 0;

	while (bytes) {
		char __user *buf = iov->iov_base + base;
		int copy = min(bytes, iov->iov_len - base);

		base = 0;
		left = __copy_from_user_inatomic(vaddr, buf, copy);
		copied += copy;
		bytes -= copy;
		vaddr += copy;
		iov++;

		if (unlikely(left))
			break;
	}
	return copied - left;
}

/*
 * Copy as much as we can into the page and return the number of bytes which
 * were successfully copied.  If a fault is encountered then return the number of
 * bytes which were copied.
 */
size_t iov_iter_copy_from_user_atomic(struct page *page,
		struct iov_iter *i, unsigned long offset, size_t bytes)
{
	char *kaddr;
	size_t copied;

	kaddr = kmap_atomic(page);
	if (likely(i->nr_segs == 1)) {
		int left;
		char __user *buf = i->iov->iov_base + i->iov_offset;
		left = __copy_from_user_inatomic(kaddr + offset, buf, bytes);
		copied = bytes - left;
	} else {
		copied = __iovec_copy_from_user_inatomic(kaddr + offset,
						i->iov, i->iov_offset, bytes);
	}
	kunmap_atomic(kaddr);

	return copied;
}
EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);

void iov_iter_advance(struct iov_iter *i, size_t bytes)
{
	BUG_ON(i->count < bytes);

	if (likely(i->nr_segs == 1)) {
		i->iov_offset += bytes;
		i->count -= bytes;
	} else {
		const struct iovec *iov = i->iov;
		size_t base = i->iov_offset;
		unsigned long nr_segs = i->nr_segs;

		/*
		 * The !iov->iov_len check ensures we skip over unlikely
		 * zero-length segments (without overruning the iovec).
		 */
		while (bytes || unlikely(i->count && !iov->iov_len)) {
			int copy;

			copy = min(bytes, iov->iov_len - base);
			BUG_ON(!i->count || i->count < copy);
			i->count -= copy;
			bytes -= copy;
			base += copy;
			if (iov->iov_len == base) {
				iov++;
				nr_segs--;
				base = 0;
			}
		}
		i->iov = iov;
		i->iov_offset = base;
		i->nr_segs = nr_segs;
	}
}
EXPORT_SYMBOL(iov_iter_advance);

/*
 * Fault in the first iovec of the given iov_iter, to a maximum length
 * of bytes. Returns 0 on success, or non-zero if the memory could not be
 * accessed (ie. because it is an invalid address).
 *
 * writev-intensive code may want this to prefault several iovecs -- that
 * would be possible (callers must not rely on the fact that _only_ the
 * first iovec will be faulted with the current implementation).
 */
int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
{
	char __user *buf = i->iov->iov_base + i->iov_offset;
	bytes = min(bytes, i->iov->iov_len - i->iov_offset);
	return fault_in_pages_readable(buf, bytes);
}
EXPORT_SYMBOL(iov_iter_fault_in_readable);

/*
 * Return the count of just the current iov_iter segment.
 */
size_t iov_iter_single_seg_count(const struct iov_iter *i)
{
	const struct iovec *iov = i->iov;
	if (i->nr_segs == 1)
		return i->count;
	else
		return min(i->count, iov->iov_len - i->iov_offset);
}
EXPORT_SYMBOL(iov_iter_single_seg_count);

unsigned long iov_iter_alignment(const struct iov_iter *i)
{
	const struct iovec *iov = i->iov;
	unsigned long res;
	size_t size = i->count;
	size_t n;

	if (!size)
		return 0;

	res = (unsigned long)iov->iov_base + i->iov_offset;
	n = iov->iov_len - i->iov_offset;
	if (n >= size)
		return res | size;
	size -= n;
	res |= n;
	while (size > (++iov)->iov_len) {
		res |= (unsigned long)iov->iov_base | iov->iov_len;
		size -= iov->iov_len;
	}
	res |= (unsigned long)iov->iov_base | size;
	return res;
}
EXPORT_SYMBOL(iov_iter_alignment);

void iov_iter_init(struct iov_iter *i, int direction,
			const struct iovec *iov, unsigned long nr_segs,
			size_t count)
{
	/* It will get better.  Eventually... */
	if (segment_eq(get_fs(), KERNEL_DS))
		direction |= REQ_KERNEL;
	i->type = direction;
	i->iov = iov;
	i->nr_segs = nr_segs;
	i->iov_offset = 0;
	i->count = count;
}
EXPORT_SYMBOL(iov_iter_init);

ssize_t iov_iter_get_pages(struct iov_iter *i,
		   struct page **pages, size_t maxsize,
		   size_t *start)
{
	size_t offset = i->iov_offset;
	const struct iovec *iov = i->iov;
	size_t len;
	unsigned long addr;
	int n;
	int res;

	len = iov->iov_len - offset;
	if (len > i->count)
		len = i->count;
	if (len > maxsize)
		len = maxsize;
	addr = (unsigned long)iov->iov_base + offset;
	len += *start = addr & (PAGE_SIZE - 1);
	addr &= ~(PAGE_SIZE - 1);
	n = (len + PAGE_SIZE - 1) / PAGE_SIZE;
	res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
	if (unlikely(res < 0))
		return res;
	return (res == n ? len : res * PAGE_SIZE) - *start;
}
EXPORT_SYMBOL(iov_iter_get_pages);

ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
		   struct page ***pages, size_t maxsize,
		   size_t *start)
{
	size_t offset = i->iov_offset;
	const struct iovec *iov = i->iov;
	size_t len;
	unsigned long addr;
	void *p;
	int n;
	int res;

	len = iov->iov_len - offset;
	if (len > i->count)
		len = i->count;
	if (len > maxsize)
		len = maxsize;
	addr = (unsigned long)iov->iov_base + offset;
	len += *start = addr & (PAGE_SIZE - 1);
	addr &= ~(PAGE_SIZE - 1);
	n = (len + PAGE_SIZE - 1) / PAGE_SIZE;
	
	p = kmalloc(n * sizeof(struct page *), GFP_KERNEL);
	if (!p)
		p = vmalloc(n * sizeof(struct page *));
	if (!p)
		return -ENOMEM;

	res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);
	if (unlikely(res < 0)) {
		kvfree(p);
		return res;
	}
	*pages = p;
	return (res == n ? len : res * PAGE_SIZE) - *start;
}
EXPORT_SYMBOL(iov_iter_get_pages_alloc);

int iov_iter_npages(const struct iov_iter *i, int maxpages)
{
	size_t offset = i->iov_offset;
	size_t size = i->count;
	const struct iovec *iov = i->iov;
	int npages = 0;
	int n;

	for (n = 0; size && n < i->nr_segs; n++, iov++) {
		unsigned long addr = (unsigned long)iov->iov_base + offset;
		size_t len = iov->iov_len - offset;
		offset = 0;
		if (unlikely(!len))	/* empty segment */
			continue;
		if (len > size)
			len = size;
		npages += (addr + len + PAGE_SIZE - 1) / PAGE_SIZE
			  - addr / PAGE_SIZE;
		if (npages >= maxpages)	/* don't bother going further */
			return maxpages;
		size -= len;
		offset = 0;
	}
	return min(npages, maxpages);
}
EXPORT_SYMBOL(iov_iter_npages);
Commit	Line	Data
4f18cd31 AV	1	#include <linux/export.h>
	2	#include <linux/uio.h>
	3	#include <linux/pagemap.h>
91f79c43 AV	4	#include <linux/slab.h>
91f79c43 AV	5	#include <linux/vmalloc.h>
4f18cd31 AV	6
	7	size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
	8	struct iov_iter *i)
	9	{
	10	size_t skip, copy, left, wanted;
	11	const struct iovec *iov;
	12	char __user *buf;
	13	void kaddr, from;
	14
	15	if (unlikely(bytes > i->count))
	16	bytes = i->count;
	17
	18	if (unlikely(!bytes))
	19	return 0;
	20
	21	wanted = bytes;
	22	iov = i->iov;
	23	skip = i->iov_offset;
	24	buf = iov->iov_base + skip;
	25	copy = min(bytes, iov->iov_len - skip);
	26
	27	if (!fault_in_pages_writeable(buf, copy)) {
	28	kaddr = kmap_atomic(page);
	29	from = kaddr + offset;
	30
	31	/* first chunk, usually the only one */
	32	left = __copy_to_user_inatomic(buf, from, copy);
	33	copy -= left;
	34	skip += copy;
	35	from += copy;
	36	bytes -= copy;
	37
	38	while (unlikely(!left && bytes)) {
	39	iov++;
	40	buf = iov->iov_base;
	41	copy = min(bytes, iov->iov_len);
	42	left = __copy_to_user_inatomic(buf, from, copy);
	43	copy -= left;
	44	skip = copy;
	45	from += copy;
	46	bytes -= copy;
	47	}
	48	if (likely(!bytes)) {
	49	kunmap_atomic(kaddr);
	50	goto done;
	51	}
	52	offset = from - kaddr;
	53	buf += copy;
	54	kunmap_atomic(kaddr);
	55	copy = min(bytes, iov->iov_len - skip);
	56	}
	57	/* Too bad - revert to non-atomic kmap */
	58	kaddr = kmap(page);
	59	from = kaddr + offset;
	60	left = __copy_to_user(buf, from, copy);
	61	copy -= left;
	62	skip += copy;
	63	from += copy;
	64	bytes -= copy;
	65	while (unlikely(!left && bytes)) {
	66	iov++;
	67	buf = iov->iov_base;
	68	copy = min(bytes, iov->iov_len);
	69	left = __copy_to_user(buf, from, copy);
70	copy -= left;
71	skip = copy;
72	from += copy;
73	bytes -= copy;
74	}
75	kunmap(page);
76	done:
77	i->count -= wanted - bytes;
78	i->nr_segs -= iov - i->iov;
79	i->iov = iov;
80	i->iov_offset = skip;
81	return wanted - bytes;
82	}
83	EXPORT_SYMBOL(copy_page_to_iter);
84
85	static size_t __iovec_copy_from_user_inatomic(char *vaddr,
86	const struct iovec *iov, size_t base, size_t bytes)
87	{
88	size_t copied = 0, left = 0;
89
90	while (bytes) {
91	char __user *buf = iov->iov_base + base;
92	int copy = min(bytes, iov->iov_len - base);
93
94	base = 0;
95	left = __copy_from_user_inatomic(vaddr, buf, copy);
96	copied += copy;
97	bytes -= copy;
98	vaddr += copy;
99	iov++;
100
101	if (unlikely(left))
102	break;
103	}
104	return copied - left;
105	}
106
107	/*
108	* Copy as much as we can into the page and return the number of bytes which
109	* were successfully copied. If a fault is encountered then return the number of
110	* bytes which were copied.
111	*/
112	size_t iov_iter_copy_from_user_atomic(struct page *page,
113	struct iov_iter *i, unsigned long offset, size_t bytes)
114	{
115	char *kaddr;
116	size_t copied;
117
118	kaddr = kmap_atomic(page);
119	if (likely(i->nr_segs == 1)) {
120	int left;
121	char __user *buf = i->iov->iov_base + i->iov_offset;
122	left = __copy_from_user_inatomic(kaddr + offset, buf, bytes);
123	copied = bytes - left;
124	} else {
125	copied = __iovec_copy_from_user_inatomic(kaddr + offset,
126	i->iov, i->iov_offset, bytes);
127	}
128	kunmap_atomic(kaddr);
129
130	return copied;
131	}
132	EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
133
4f18cd31 AV	134	void iov_iter_advance(struct iov_iter *i, size_t bytes)
	135	{
	136	BUG_ON(i->count < bytes);
	137
	138	if (likely(i->nr_segs == 1)) {
	139	i->iov_offset += bytes;
	140	i->count -= bytes;
	141	} else {
	142	const struct iovec *iov = i->iov;
	143	size_t base = i->iov_offset;
	144	unsigned long nr_segs = i->nr_segs;
	145
	146	/*
	147	* The !iov->iov_len check ensures we skip over unlikely
	148	* zero-length segments (without overruning the iovec).
	149	*/
	150	while (bytes \|\| unlikely(i->count && !iov->iov_len)) {
	151	int copy;
	152
	153	copy = min(bytes, iov->iov_len - base);
	154	BUG_ON(!i->count \|\| i->count < copy);
	155	i->count -= copy;
	156	bytes -= copy;
	157	base += copy;
	158	if (iov->iov_len == base) {
	159	iov++;
	160	nr_segs--;
	161	base = 0;
	162	}
	163	}
	164	i->iov = iov;
	165	i->iov_offset = base;
	166	i->nr_segs = nr_segs;
	167	}
	168	}
	169	EXPORT_SYMBOL(iov_iter_advance);
	170
	171	/*
	172	* Fault in the first iovec of the given iov_iter, to a maximum length
	173	* of bytes. Returns 0 on success, or non-zero if the memory could not be
	174	* accessed (ie. because it is an invalid address).
	175	*
	176	* writev-intensive code may want this to prefault several iovecs -- that
	177	* would be possible (callers must not rely on the fact that _only_ the
	178	* first iovec will be faulted with the current implementation).
	179	*/
	180	int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
	181	{
	182	char __user *buf = i->iov->iov_base + i->iov_offset;
	183	bytes = min(bytes, i->iov->iov_len - i->iov_offset);
	184	return fault_in_pages_readable(buf, bytes);
	185	}
	186	EXPORT_SYMBOL(iov_iter_fault_in_readable);
	187
	188	/*
	189	* Return the count of just the current iov_iter segment.
	190	*/
	191	size_t iov_iter_single_seg_count(const struct iov_iter *i)
	192	{
	193	const struct iovec *iov = i->iov;
	194	if (i->nr_segs == 1)
	195	return i->count;
	196	else
	197	return min(i->count, iov->iov_len - i->iov_offset);
198	}
199	EXPORT_SYMBOL(iov_iter_single_seg_count);
886a3911 AV	200
	201	unsigned long iov_iter_alignment(const struct iov_iter *i)
	202	{
	203	const struct iovec *iov = i->iov;
	204	unsigned long res;
	205	size_t size = i->count;
	206	size_t n;
	207
	208	if (!size)
	209	return 0;
	210
	211	res = (unsigned long)iov->iov_base + i->iov_offset;
	212	n = iov->iov_len - i->iov_offset;
	213	if (n >= size)
	214	return res \| size;
	215	size -= n;
	216	res \|= n;
	217	while (size > (++iov)->iov_len) {
	218	res \|= (unsigned long)iov->iov_base \| iov->iov_len;
	219	size -= iov->iov_len;
	220	}
	221	res \|= (unsigned long)iov->iov_base \| size;
	222	return res;
	223	}
	224	EXPORT_SYMBOL(iov_iter_alignment);
71d8e532 AV	225
	226	void iov_iter_init(struct iov_iter *i, int direction,
	227	const struct iovec *iov, unsigned long nr_segs,
	228	size_t count)
	229	{
	230	/* It will get better. Eventually... */
	231	if (segment_eq(get_fs(), KERNEL_DS))
	232	direction \|= REQ_KERNEL;
	233	i->type = direction;
	234	i->iov = iov;
	235	i->nr_segs = nr_segs;
	236	i->iov_offset = 0;
	237	i->count = count;
	238	}
	239	EXPORT_SYMBOL(iov_iter_init);
7b2c99d1 AV	240
	241	ssize_t iov_iter_get_pages(struct iov_iter *i,
	242	struct page **pages, size_t maxsize,
	243	size_t *start)
	244	{
	245	size_t offset = i->iov_offset;
	246	const struct iovec *iov = i->iov;
	247	size_t len;
	248	unsigned long addr;
	249	int n;
	250	int res;
	251
	252	len = iov->iov_len - offset;
	253	if (len > i->count)
	254	len = i->count;
	255	if (len > maxsize)
	256	len = maxsize;
	257	addr = (unsigned long)iov->iov_base + offset;
	258	len += *start = addr & (PAGE_SIZE - 1);
	259	addr &= ~(PAGE_SIZE - 1);
	260	n = (len + PAGE_SIZE - 1) / PAGE_SIZE;
	261	res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
	262	if (unlikely(res < 0))
	263	return res;
	264	return (res == n ? len : res * PAGE_SIZE) - *start;
	265	}
	266	EXPORT_SYMBOL(iov_iter_get_pages);
f67da30c	267
91f79c43 AV	268	ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
	269	struct page ***pages, size_t maxsize,
	270	size_t *start)
	271	{
	272	size_t offset = i->iov_offset;
	273	const struct iovec *iov = i->iov;
	274	size_t len;
	275	unsigned long addr;
	276	void *p;
	277	int n;
	278	int res;
	279
	280	len = iov->iov_len - offset;
	281	if (len > i->count)
	282	len = i->count;
	283	if (len > maxsize)
	284	len = maxsize;
	285	addr = (unsigned long)iov->iov_base + offset;
	286	len += *start = addr & (PAGE_SIZE - 1);
	287	addr &= ~(PAGE_SIZE - 1);
	288	n = (len + PAGE_SIZE - 1) / PAGE_SIZE;
	289
	290	p = kmalloc(n * sizeof(struct page *), GFP_KERNEL);
	291	if (!p)
	292	p = vmalloc(n * sizeof(struct page *));
	293	if (!p)
	294	return -ENOMEM;
	295
	296	res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);
	297	if (unlikely(res < 0)) {
	298	kvfree(p);
	299	return res;
	300	}
	301	*pages = p;
	302	return (res == n ? len : res * PAGE_SIZE) - *start;
	303	}
	304	EXPORT_SYMBOL(iov_iter_get_pages_alloc);
	305
f67da30c AV	306	int iov_iter_npages(const struct iov_iter *i, int maxpages)
	307	{
	308	size_t offset = i->iov_offset;
	309	size_t size = i->count;
	310	const struct iovec *iov = i->iov;
	311	int npages = 0;
	312	int n;
	313
	314	for (n = 0; size && n < i->nr_segs; n++, iov++) {
	315	unsigned long addr = (unsigned long)iov->iov_base + offset;
	316	size_t len = iov->iov_len - offset;
	317	offset = 0;
	318	if (unlikely(!len)) /* empty segment */
	319	continue;
	320	if (len > size)
	321	len = size;
	322	npages += (addr + len + PAGE_SIZE - 1) / PAGE_SIZE
	323	- addr / PAGE_SIZE;
	324	if (npages >= maxpages) /* don't bother going further */
	325	return maxpages;
	326	size -= len;
	327	offset = 0;
	328	}
	329	return min(npages, maxpages);
	330	}
	331	EXPORT_SYMBOL(iov_iter_npages);