[mirror_ubuntu-zesty-kernel.git] / mm / process_vm_access.c

/*
 * linux/mm/process_vm_access.c
 *
 * Copyright (C) 2010-2011 Christopher Yeoh <cyeoh@au1.ibm.com>, IBM Corp.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */

#include <linux/mm.h>
#include <linux/uio.h>
#include <linux/sched.h>
#include <linux/highmem.h>
#include <linux/ptrace.h>
#include <linux/slab.h>
#include <linux/syscalls.h>

#ifdef CONFIG_COMPAT
#include <linux/compat.h>
#endif

/**
 * process_vm_rw_pages - read/write pages from task specified
 * @task: task to read/write from
 * @mm: mm for task
 * @process_pages: struct pages area that can store at least
 *  nr_pages_to_copy struct page pointers
 * @pa: address of page in task to start copying from/to
 * @start_offset: offset in page to start copying from/to
 * @len: number of bytes to copy
 * @lvec: iovec array specifying where to copy to/from
 * @lvec_cnt: number of elements in iovec array
 * @lvec_current: index in iovec array we are up to
 * @lvec_offset: offset in bytes from current iovec iov_base we are up to
 * @vm_write: 0 means copy from, 1 means copy to
 * @nr_pages_to_copy: number of pages to copy
 * @bytes_copied: returns number of bytes successfully copied
 * Returns 0 on success, error code otherwise
 */
static int process_vm_rw_pages(struct page **pages,
			       unsigned offset,
			       size_t len,
			       struct iov_iter *iter,
			       int vm_write,
			       ssize_t *bytes_copied)
{
	*bytes_copied = 0;

	/* Do the copy for each page */
	while (iov_iter_count(iter) && len) {
		struct page *page = *pages++;
		size_t copy = PAGE_SIZE - offset;
		size_t copied;

		if (copy > len)
			copy = len;

		if (vm_write) {
			if (copy > iov_iter_count(iter))
				copy = iov_iter_count(iter);
			copied = iov_iter_copy_from_user(page, iter,
					offset, copy);
			iov_iter_advance(iter, copied);
			set_page_dirty_lock(page);
		} else {
			copied = copy_page_to_iter(page, offset, copy, iter);
		}
		*bytes_copied += copied;
		len -= copied;
		if (copied < copy && iov_iter_count(iter))
			return -EFAULT;
		offset = 0;
	}
	return 0;
}

/* Maximum number of pages kmalloc'd to hold struct page's during copy */
#define PVM_MAX_KMALLOC_PAGES (PAGE_SIZE * 2)

/**
 * process_vm_rw_single_vec - read/write pages from task specified
 * @addr: start memory address of target process
 * @len: size of area to copy to/from
 * @lvec: iovec array specifying where to copy to/from locally
 * @lvec_cnt: number of elements in iovec array
 * @lvec_current: index in iovec array we are up to
 * @lvec_offset: offset in bytes from current iovec iov_base we are up to
 * @process_pages: struct pages area that can store at least
 *  nr_pages_to_copy struct page pointers
 * @mm: mm for task
 * @task: task to read/write from
 * @vm_write: 0 means copy from, 1 means copy to
 * @bytes_copied: returns number of bytes successfully copied
 * Returns 0 on success or on failure error code
 */
static int process_vm_rw_single_vec(unsigned long addr,
				    unsigned long len,
				    struct iov_iter *iter,
				    struct page **process_pages,
				    struct mm_struct *mm,
				    struct task_struct *task,
				    int vm_write,
				    ssize_t *bytes_copied)
{
	unsigned long pa = addr & PAGE_MASK;
	unsigned long start_offset = addr - pa;
	unsigned long nr_pages;
	ssize_t bytes_copied_loop;
	ssize_t rc = 0;
	unsigned long nr_pages_copied = 0;
	unsigned long max_pages_per_loop = PVM_MAX_KMALLOC_PAGES
		/ sizeof(struct pages *);

	*bytes_copied = 0;

	/* Work out address and page range required */
	if (len == 0)
		return 0;
	nr_pages = (addr + len - 1) / PAGE_SIZE - addr / PAGE_SIZE + 1;

	while ((nr_pages_copied < nr_pages) && iov_iter_count(iter)) {
		int nr_pages_to_copy;
		int pages_pinned;
		size_t n;
		nr_pages_to_copy = min(nr_pages - nr_pages_copied,
				       max_pages_per_loop);

		/* Get the pages we're interested in */
		down_read(&mm->mmap_sem);
		pages_pinned = get_user_pages(task, mm, pa,
					      nr_pages_to_copy,
					      vm_write, 0, process_pages, NULL);
		up_read(&mm->mmap_sem);

		if (pages_pinned <= 0)
			return -EFAULT;

		n = pages_pinned * PAGE_SIZE - start_offset;
		if (n > len)
			n = len;

		rc = process_vm_rw_pages(process_pages,
					 start_offset, n, iter,
					 vm_write,
					 &bytes_copied_loop);
		len -= n;
		start_offset = 0;
		*bytes_copied += bytes_copied_loop;
		nr_pages_copied += pages_pinned;
		pa += pages_pinned * PAGE_SIZE;
		while (pages_pinned)
			put_page(process_pages[--pages_pinned]);
		if (rc < 0)
			break;
	}

	return rc;
}

/* Maximum number of entries for process pages array
   which lives on stack */
#define PVM_MAX_PP_ARRAY_COUNT 16

/**
 * process_vm_rw_core - core of reading/writing pages from task specified
 * @pid: PID of process to read/write from/to
 * @lvec: iovec array specifying where to copy to/from locally
 * @liovcnt: size of lvec array
 * @rvec: iovec array specifying where to copy to/from in the other process
 * @riovcnt: size of rvec array
 * @flags: currently unused
 * @vm_write: 0 if reading from other process, 1 if writing to other process
 * Returns the number of bytes read/written or error code. May
 *  return less bytes than expected if an error occurs during the copying
 *  process.
 */
static ssize_t process_vm_rw_core(pid_t pid, struct iov_iter *iter,
				  const struct iovec *rvec,
				  unsigned long riovcnt,
				  unsigned long flags, int vm_write)
{
	struct task_struct *task;
	struct page *pp_stack[PVM_MAX_PP_ARRAY_COUNT];
	struct page **process_pages = pp_stack;
	struct mm_struct *mm;
	unsigned long i;
	ssize_t rc = 0;
	ssize_t bytes_copied_loop;
	ssize_t bytes_copied = 0;
	unsigned long nr_pages = 0;
	unsigned long nr_pages_iov;
	ssize_t iov_len;

	/*
	 * Work out how many pages of struct pages we're going to need
	 * when eventually calling get_user_pages
	 */
	for (i = 0; i < riovcnt; i++) {
		iov_len = rvec[i].iov_len;
		if (iov_len > 0) {
			nr_pages_iov = ((unsigned long)rvec[i].iov_base
					+ iov_len)
				/ PAGE_SIZE - (unsigned long)rvec[i].iov_base
				/ PAGE_SIZE + 1;
			nr_pages = max(nr_pages, nr_pages_iov);
		}
	}

	if (nr_pages == 0)
		return 0;

	if (nr_pages > PVM_MAX_PP_ARRAY_COUNT) {
		/* For reliability don't try to kmalloc more than
		   2 pages worth */
		process_pages = kmalloc(min_t(size_t, PVM_MAX_KMALLOC_PAGES,
					      sizeof(struct pages *)*nr_pages),
					GFP_KERNEL);

		if (!process_pages)
			return -ENOMEM;
	}

	/* Get process information */
	rcu_read_lock();
	task = find_task_by_vpid(pid);
	if (task)
		get_task_struct(task);
	rcu_read_unlock();
	if (!task) {
		rc = -ESRCH;
		goto free_proc_pages;
	}

	mm = mm_access(task, PTRACE_MODE_ATTACH);
	if (!mm || IS_ERR(mm)) {
		rc = IS_ERR(mm) ? PTR_ERR(mm) : -ESRCH;
		/*
		 * Explicitly map EACCES to EPERM as EPERM is a more a
		 * appropriate error code for process_vw_readv/writev
		 */
		if (rc == -EACCES)
			rc = -EPERM;
		goto put_task_struct;
	}

	for (i = 0; i < riovcnt && iov_iter_count(iter); i++) {
		rc = process_vm_rw_single_vec(
			(unsigned long)rvec[i].iov_base, rvec[i].iov_len,
			iter, process_pages, mm, task, vm_write,
			&bytes_copied_loop);
		bytes_copied += bytes_copied_loop;
		if (rc != 0) {
			/* If we have managed to copy any data at all then
			   we return the number of bytes copied. Otherwise
			   we return the error code */
			if (bytes_copied)
				rc = bytes_copied;
			goto put_mm;
		}
	}

	rc = bytes_copied;
put_mm:
	mmput(mm);

put_task_struct:
	put_task_struct(task);

free_proc_pages:
	if (process_pages != pp_stack)
		kfree(process_pages);
	return rc;
}

/**
 * process_vm_rw - check iovecs before calling core routine
 * @pid: PID of process to read/write from/to
 * @lvec: iovec array specifying where to copy to/from locally
 * @liovcnt: size of lvec array
 * @rvec: iovec array specifying where to copy to/from in the other process
 * @riovcnt: size of rvec array
 * @flags: currently unused
 * @vm_write: 0 if reading from other process, 1 if writing to other process
 * Returns the number of bytes read/written or error code. May
 *  return less bytes than expected if an error occurs during the copying
 *  process.
 */
static ssize_t process_vm_rw(pid_t pid,
			     const struct iovec __user *lvec,
			     unsigned long liovcnt,
			     const struct iovec __user *rvec,
			     unsigned long riovcnt,
			     unsigned long flags, int vm_write)
{
	struct iovec iovstack_l[UIO_FASTIOV];
	struct iovec iovstack_r[UIO_FASTIOV];
	struct iovec *iov_l = iovstack_l;
	struct iovec *iov_r = iovstack_r;
	struct iov_iter iter;
	ssize_t rc;

	if (flags != 0)
		return -EINVAL;

	/* Check iovecs */
	if (vm_write)
		rc = rw_copy_check_uvector(WRITE, lvec, liovcnt, UIO_FASTIOV,
					   iovstack_l, &iov_l);
	else
		rc = rw_copy_check_uvector(READ, lvec, liovcnt, UIO_FASTIOV,
					   iovstack_l, &iov_l);
	if (rc <= 0)
		goto free_iovecs;

	iov_iter_init(&iter, iov_l, liovcnt, rc, 0);

	rc = rw_copy_check_uvector(CHECK_IOVEC_ONLY, rvec, riovcnt, UIO_FASTIOV,
				   iovstack_r, &iov_r);
	if (rc <= 0)
		goto free_iovecs;

	rc = process_vm_rw_core(pid, &iter, iov_r, riovcnt, flags, vm_write);

free_iovecs:
	if (iov_r != iovstack_r)
		kfree(iov_r);
	if (iov_l != iovstack_l)
		kfree(iov_l);

	return rc;
}

SYSCALL_DEFINE6(process_vm_readv, pid_t, pid, const struct iovec __user *, lvec,
		unsigned long, liovcnt, const struct iovec __user *, rvec,
		unsigned long, riovcnt,	unsigned long, flags)
{
	return process_vm_rw(pid, lvec, liovcnt, rvec, riovcnt, flags, 0);
}

SYSCALL_DEFINE6(process_vm_writev, pid_t, pid,
		const struct iovec __user *, lvec,
		unsigned long, liovcnt, const struct iovec __user *, rvec,
		unsigned long, riovcnt,	unsigned long, flags)
{
	return process_vm_rw(pid, lvec, liovcnt, rvec, riovcnt, flags, 1);
}

#ifdef CONFIG_COMPAT

asmlinkage ssize_t
compat_process_vm_rw(compat_pid_t pid,
		     const struct compat_iovec __user *lvec,
		     unsigned long liovcnt,
		     const struct compat_iovec __user *rvec,
		     unsigned long riovcnt,
		     unsigned long flags, int vm_write)
{
	struct iovec iovstack_l[UIO_FASTIOV];
	struct iovec iovstack_r[UIO_FASTIOV];
	struct iovec *iov_l = iovstack_l;
	struct iovec *iov_r = iovstack_r;
	struct iov_iter iter;
	ssize_t rc = -EFAULT;

	if (flags != 0)
		return -EINVAL;

	if (vm_write)
		rc = compat_rw_copy_check_uvector(WRITE, lvec, liovcnt,
						  UIO_FASTIOV, iovstack_l,
						  &iov_l);
	else
		rc = compat_rw_copy_check_uvector(READ, lvec, liovcnt,
						  UIO_FASTIOV, iovstack_l,
						  &iov_l);
	if (rc <= 0)
		goto free_iovecs;
	iov_iter_init(&iter, iov_l, liovcnt, rc, 0);
	rc = compat_rw_copy_check_uvector(CHECK_IOVEC_ONLY, rvec, riovcnt,
					  UIO_FASTIOV, iovstack_r,
					  &iov_r);
	if (rc <= 0)
		goto free_iovecs;

	rc = process_vm_rw_core(pid, &iter, iov_r, riovcnt, flags, vm_write);

free_iovecs:
	if (iov_r != iovstack_r)
		kfree(iov_r);
	if (iov_l != iovstack_l)
		kfree(iov_l);
	return rc;
}

asmlinkage ssize_t
compat_sys_process_vm_readv(compat_pid_t pid,
			    const struct compat_iovec __user *lvec,
			    unsigned long liovcnt,
			    const struct compat_iovec __user *rvec,
			    unsigned long riovcnt,
			    unsigned long flags)
{
	return compat_process_vm_rw(pid, lvec, liovcnt, rvec,
				    riovcnt, flags, 0);
}

asmlinkage ssize_t
compat_sys_process_vm_writev(compat_pid_t pid,
			     const struct compat_iovec __user *lvec,
			     unsigned long liovcnt,
			     const struct compat_iovec __user *rvec,
			     unsigned long riovcnt,
			     unsigned long flags)
{
	return compat_process_vm_rw(pid, lvec, liovcnt, rvec,
				    riovcnt, flags, 1);
}

#endif
Commit	Line	Data
fcf63409 CY	1	/*
	2	* linux/mm/process_vm_access.c
	3	*
	4	* Copyright (C) 2010-2011 Christopher Yeoh <cyeoh@au1.ibm.com>, IBM Corp.
	5	*
	6	* This program is free software; you can redistribute it and/or
	7	* modify it under the terms of the GNU General Public License
	8	* as published by the Free Software Foundation; either version
	9	* 2 of the License, or (at your option) any later version.
	10	*/
	11
	12	#include <linux/mm.h>
	13	#include <linux/uio.h>
	14	#include <linux/sched.h>
	15	#include <linux/highmem.h>
	16	#include <linux/ptrace.h>
	17	#include <linux/slab.h>
	18	#include <linux/syscalls.h>
	19
	20	#ifdef CONFIG_COMPAT
	21	#include <linux/compat.h>
	22	#endif
	23
	24	/**
	25	* process_vm_rw_pages - read/write pages from task specified
	26	* @task: task to read/write from
	27	* @mm: mm for task
	28	* @process_pages: struct pages area that can store at least
	29	* nr_pages_to_copy struct page pointers
	30	* @pa: address of page in task to start copying from/to
	31	* @start_offset: offset in page to start copying from/to
	32	* @len: number of bytes to copy
	33	* @lvec: iovec array specifying where to copy to/from
	34	* @lvec_cnt: number of elements in iovec array
	35	* @lvec_current: index in iovec array we are up to
	36	* @lvec_offset: offset in bytes from current iovec iov_base we are up to
	37	* @vm_write: 0 means copy from, 1 means copy to
	38	* @nr_pages_to_copy: number of pages to copy
	39	* @bytes_copied: returns number of bytes successfully copied
	40	* Returns 0 on success, error code otherwise
	41	*/
70eca12d AV	42	static int process_vm_rw_pages(struct page **pages,
70eca12d AV	43	unsigned offset,
e21345f9	44	size_t len,
9f78bdfa	45	struct iov_iter *iter,
fcf63409	46	int vm_write,
fcf63409 CY	47	ssize_t *bytes_copied)
fcf63409 CY	48	{
fcf63409 CY	49	*bytes_copied = 0;
fcf63409 CY	50
fcf63409	51	/* Do the copy for each page */
e21345f9	52	while (iov_iter_count(iter) && len) {
70eca12d	53	struct page page = pages++;
e21345f9	54	size_t copy = PAGE_SIZE - offset;
70eca12d	55	size_t copied;
240f3905	56
e21345f9 AV	57	if (copy > len)
	58	copy = len;
	59
240f3905	60	if (vm_write) {
70eca12d AV	61	if (copy > iov_iter_count(iter))
	62	copy = iov_iter_count(iter);
	63	copied = iov_iter_copy_from_user(page, iter,
	64	offset, copy);
	65	iov_iter_advance(iter, copied);
240f3905 AV	66	set_page_dirty_lock(page);
240f3905 AV	67	} else {
70eca12d	68	copied = copy_page_to_iter(page, offset, copy, iter);
fcf63409	69	}
70eca12d AV	70	*bytes_copied += copied;
	71	len -= copied;
	72	if (copied < copy && iov_iter_count(iter))
	73	return -EFAULT;
	74	offset = 0;
fcf63409	75	}
70eca12d	76	return 0;
fcf63409 CY	77	}
	78
	79	/* Maximum number of pages kmalloc'd to hold struct page's during copy */
	80	#define PVM_MAX_KMALLOC_PAGES (PAGE_SIZE * 2)
	81
	82	/**
	83	* process_vm_rw_single_vec - read/write pages from task specified
	84	* @addr: start memory address of target process
	85	* @len: size of area to copy to/from
	86	* @lvec: iovec array specifying where to copy to/from locally
	87	* @lvec_cnt: number of elements in iovec array
	88	* @lvec_current: index in iovec array we are up to
	89	* @lvec_offset: offset in bytes from current iovec iov_base we are up to
	90	* @process_pages: struct pages area that can store at least
	91	* nr_pages_to_copy struct page pointers
	92	* @mm: mm for task
	93	* @task: task to read/write from
	94	* @vm_write: 0 means copy from, 1 means copy to
	95	* @bytes_copied: returns number of bytes successfully copied
	96	* Returns 0 on success or on failure error code
	97	*/
	98	static int process_vm_rw_single_vec(unsigned long addr,
	99	unsigned long len,
9f78bdfa	100	struct iov_iter *iter,
fcf63409 CY	101	struct page **process_pages,
	102	struct mm_struct *mm,
	103	struct task_struct *task,
	104	int vm_write,
	105	ssize_t *bytes_copied)
	106	{
	107	unsigned long pa = addr & PAGE_MASK;
	108	unsigned long start_offset = addr - pa;
	109	unsigned long nr_pages;
	110	ssize_t bytes_copied_loop;
	111	ssize_t rc = 0;
	112	unsigned long nr_pages_copied = 0;
fcf63409 CY	113	unsigned long max_pages_per_loop = PVM_MAX_KMALLOC_PAGES
	114	/ sizeof(struct pages *);
	115
	116	*bytes_copied = 0;
	117
	118	/* Work out address and page range required */
	119	if (len == 0)
	120	return 0;
	121	nr_pages = (addr + len - 1) / PAGE_SIZE - addr / PAGE_SIZE + 1;
	122
240f3905	123	while ((nr_pages_copied < nr_pages) && iov_iter_count(iter)) {
70eca12d AV	124	int nr_pages_to_copy;
70eca12d AV	125	int pages_pinned;
e21345f9	126	size_t n;
fcf63409 CY	127	nr_pages_to_copy = min(nr_pages - nr_pages_copied,
	128	max_pages_per_loop);
	129
70eca12d AV	130	/* Get the pages we're interested in */
	131	down_read(&mm->mmap_sem);
	132	pages_pinned = get_user_pages(task, mm, pa,
	133	nr_pages_to_copy,
	134	vm_write, 0, process_pages, NULL);
	135	up_read(&mm->mmap_sem);
	136
	137	if (pages_pinned <= 0)
	138	return -EFAULT;
	139
e21345f9 AV	140	n = pages_pinned * PAGE_SIZE - start_offset;
	141	if (n > len)
	142	n = len;
	143
70eca12d	144	rc = process_vm_rw_pages(process_pages,
e21345f9 AV	145	start_offset, n, iter,
e21345f9 AV	146	vm_write,
fcf63409	147	&bytes_copied_loop);
e21345f9	148	len -= n;
fcf63409 CY	149	start_offset = 0;
fcf63409 CY	150	*bytes_copied += bytes_copied_loop;
70eca12d AV	151	nr_pages_copied += pages_pinned;
	152	pa += pages_pinned * PAGE_SIZE;
	153	while (pages_pinned)
	154	put_page(process_pages[--pages_pinned]);
70eca12d AV	155	if (rc < 0)
70eca12d AV	156	break;
fcf63409 CY	157	}
	158
	159	return rc;
	160	}
	161
	162	/* Maximum number of entries for process pages array
	163	which lives on stack */
	164	#define PVM_MAX_PP_ARRAY_COUNT 16
	165
	166	/**
	167	* process_vm_rw_core - core of reading/writing pages from task specified
	168	* @pid: PID of process to read/write from/to
	169	* @lvec: iovec array specifying where to copy to/from locally
	170	* @liovcnt: size of lvec array
	171	* @rvec: iovec array specifying where to copy to/from in the other process
	172	* @riovcnt: size of rvec array
	173	* @flags: currently unused
	174	* @vm_write: 0 if reading from other process, 1 if writing to other process
	175	* Returns the number of bytes read/written or error code. May
	176	* return less bytes than expected if an error occurs during the copying
	177	* process.
	178	*/
9f78bdfa	179	static ssize_t process_vm_rw_core(pid_t pid, struct iov_iter *iter,
fcf63409 CY	180	const struct iovec *rvec,
	181	unsigned long riovcnt,
	182	unsigned long flags, int vm_write)
	183	{
	184	struct task_struct *task;
	185	struct page *pp_stack[PVM_MAX_PP_ARRAY_COUNT];
	186	struct page **process_pages = pp_stack;
	187	struct mm_struct *mm;
	188	unsigned long i;
	189	ssize_t rc = 0;
	190	ssize_t bytes_copied_loop;
	191	ssize_t bytes_copied = 0;
	192	unsigned long nr_pages = 0;
	193	unsigned long nr_pages_iov;
fcf63409 CY	194	ssize_t iov_len;
	195
	196	/*
	197	* Work out how many pages of struct pages we're going to need
	198	* when eventually calling get_user_pages
	199	*/
	200	for (i = 0; i < riovcnt; i++) {
	201	iov_len = rvec[i].iov_len;
	202	if (iov_len > 0) {
	203	nr_pages_iov = ((unsigned long)rvec[i].iov_base
	204	+ iov_len)
	205	/ PAGE_SIZE - (unsigned long)rvec[i].iov_base
	206	/ PAGE_SIZE + 1;
	207	nr_pages = max(nr_pages, nr_pages_iov);
	208	}
	209	}
	210
	211	if (nr_pages == 0)
	212	return 0;
	213
	214	if (nr_pages > PVM_MAX_PP_ARRAY_COUNT) {
	215	/* For reliability don't try to kmalloc more than
	216	2 pages worth */
	217	process_pages = kmalloc(min_t(size_t, PVM_MAX_KMALLOC_PAGES,
	218	sizeof(struct pages )nr_pages),
	219	GFP_KERNEL);
	220
	221	if (!process_pages)
	222	return -ENOMEM;
	223	}
	224
	225	/* Get process information */
	226	rcu_read_lock();
	227	task = find_task_by_vpid(pid);
	228	if (task)
	229	get_task_struct(task);
	230	rcu_read_unlock();
	231	if (!task) {
	232	rc = -ESRCH;
	233	goto free_proc_pages;
	234	}
	235
8cdb878d CY	236	mm = mm_access(task, PTRACE_MODE_ATTACH);
	237	if (!mm \|\| IS_ERR(mm)) {
	238	rc = IS_ERR(mm) ? PTR_ERR(mm) : -ESRCH;
	239	/*
	240	* Explicitly map EACCES to EPERM as EPERM is a more a
	241	* appropriate error code for process_vw_readv/writev
	242	*/
	243	if (rc == -EACCES)
	244	rc = -EPERM;
fcf63409 CY	245	goto put_task_struct;
	246	}
	247
240f3905	248	for (i = 0; i < riovcnt && iov_iter_count(iter); i++) {
fcf63409 CY	249	rc = process_vm_rw_single_vec(
fcf63409 CY	250	(unsigned long)rvec[i].iov_base, rvec[i].iov_len,
9f78bdfa AV	251	iter, process_pages, mm, task, vm_write,
9f78bdfa AV	252	&bytes_copied_loop);
fcf63409 CY	253	bytes_copied += bytes_copied_loop;
	254	if (rc != 0) {
	255	/* If we have managed to copy any data at all then
	256	we return the number of bytes copied. Otherwise
	257	we return the error code */
	258	if (bytes_copied)
	259	rc = bytes_copied;
	260	goto put_mm;
	261	}
	262	}
	263
	264	rc = bytes_copied;
	265	put_mm:
	266	mmput(mm);
	267
	268	put_task_struct:
	269	put_task_struct(task);
	270
	271	free_proc_pages:
	272	if (process_pages != pp_stack)
	273	kfree(process_pages);
	274	return rc;
	275	}
	276
	277	/**
	278	* process_vm_rw - check iovecs before calling core routine
	279	* @pid: PID of process to read/write from/to
	280	* @lvec: iovec array specifying where to copy to/from locally
	281	* @liovcnt: size of lvec array
	282	* @rvec: iovec array specifying where to copy to/from in the other process
	283	* @riovcnt: size of rvec array
	284	* @flags: currently unused
	285	* @vm_write: 0 if reading from other process, 1 if writing to other process
	286	* Returns the number of bytes read/written or error code. May
	287	* return less bytes than expected if an error occurs during the copying
	288	* process.
	289	*/
	290	static ssize_t process_vm_rw(pid_t pid,
	291	const struct iovec __user *lvec,
	292	unsigned long liovcnt,
	293	const struct iovec __user *rvec,
	294	unsigned long riovcnt,
	295	unsigned long flags, int vm_write)
	296	{
	297	struct iovec iovstack_l[UIO_FASTIOV];
	298	struct iovec iovstack_r[UIO_FASTIOV];
	299	struct iovec *iov_l = iovstack_l;
	300	struct iovec *iov_r = iovstack_r;
9f78bdfa	301	struct iov_iter iter;
fcf63409 CY	302	ssize_t rc;
	303
	304	if (flags != 0)
	305	return -EINVAL;
	306
	307	/* Check iovecs */
	308	if (vm_write)
	309	rc = rw_copy_check_uvector(WRITE, lvec, liovcnt, UIO_FASTIOV,
ac34ebb3	310	iovstack_l, &iov_l);
fcf63409 CY	311	else
fcf63409 CY	312	rc = rw_copy_check_uvector(READ, lvec, liovcnt, UIO_FASTIOV,
ac34ebb3	313	iovstack_l, &iov_l);
fcf63409 CY	314	if (rc <= 0)
	315	goto free_iovecs;
	316
9f78bdfa AV	317	iov_iter_init(&iter, iov_l, liovcnt, rc, 0);
9f78bdfa AV	318
ac34ebb3 CY	319	rc = rw_copy_check_uvector(CHECK_IOVEC_ONLY, rvec, riovcnt, UIO_FASTIOV,
ac34ebb3 CY	320	iovstack_r, &iov_r);
fcf63409 CY	321	if (rc <= 0)
	322	goto free_iovecs;
	323
9f78bdfa	324	rc = process_vm_rw_core(pid, &iter, iov_r, riovcnt, flags, vm_write);
fcf63409 CY	325
	326	free_iovecs:
	327	if (iov_r != iovstack_r)
	328	kfree(iov_r);
	329	if (iov_l != iovstack_l)
	330	kfree(iov_l);
	331
	332	return rc;
	333	}
	334
	335	SYSCALL_DEFINE6(process_vm_readv, pid_t, pid, const struct iovec __user *, lvec,
	336	unsigned long, liovcnt, const struct iovec __user *, rvec,
	337	unsigned long, riovcnt, unsigned long, flags)
	338	{
	339	return process_vm_rw(pid, lvec, liovcnt, rvec, riovcnt, flags, 0);
	340	}
	341
	342	SYSCALL_DEFINE6(process_vm_writev, pid_t, pid,
	343	const struct iovec __user *, lvec,
	344	unsigned long, liovcnt, const struct iovec __user *, rvec,
	345	unsigned long, riovcnt, unsigned long, flags)
	346	{
	347	return process_vm_rw(pid, lvec, liovcnt, rvec, riovcnt, flags, 1);
	348	}
	349
	350	#ifdef CONFIG_COMPAT
	351
	352	asmlinkage ssize_t
	353	compat_process_vm_rw(compat_pid_t pid,
	354	const struct compat_iovec __user *lvec,
	355	unsigned long liovcnt,
	356	const struct compat_iovec __user *rvec,
	357	unsigned long riovcnt,
	358	unsigned long flags, int vm_write)
	359	{
	360	struct iovec iovstack_l[UIO_FASTIOV];
	361	struct iovec iovstack_r[UIO_FASTIOV];
	362	struct iovec *iov_l = iovstack_l;
	363	struct iovec *iov_r = iovstack_r;
9f78bdfa	364	struct iov_iter iter;
fcf63409 CY	365	ssize_t rc = -EFAULT;
	366
	367	if (flags != 0)
	368	return -EINVAL;
	369
fcf63409 CY	370	if (vm_write)
	371	rc = compat_rw_copy_check_uvector(WRITE, lvec, liovcnt,
	372	UIO_FASTIOV, iovstack_l,
ac34ebb3	373	&iov_l);
fcf63409 CY	374	else
	375	rc = compat_rw_copy_check_uvector(READ, lvec, liovcnt,
	376	UIO_FASTIOV, iovstack_l,
ac34ebb3	377	&iov_l);
fcf63409 CY	378	if (rc <= 0)
fcf63409 CY	379	goto free_iovecs;
9f78bdfa	380	iov_iter_init(&iter, iov_l, liovcnt, rc, 0);
ac34ebb3	381	rc = compat_rw_copy_check_uvector(CHECK_IOVEC_ONLY, rvec, riovcnt,
fcf63409	382	UIO_FASTIOV, iovstack_r,
ac34ebb3	383	&iov_r);
fcf63409 CY	384	if (rc <= 0)
	385	goto free_iovecs;
	386
9f78bdfa	387	rc = process_vm_rw_core(pid, &iter, iov_r, riovcnt, flags, vm_write);
fcf63409 CY	388
	389	free_iovecs:
	390	if (iov_r != iovstack_r)
	391	kfree(iov_r);
	392	if (iov_l != iovstack_l)
	393	kfree(iov_l);
fcf63409 CY	394	return rc;
	395	}
	396
	397	asmlinkage ssize_t
	398	compat_sys_process_vm_readv(compat_pid_t pid,
	399	const struct compat_iovec __user *lvec,
	400	unsigned long liovcnt,
	401	const struct compat_iovec __user *rvec,
	402	unsigned long riovcnt,
	403	unsigned long flags)
	404	{
	405	return compat_process_vm_rw(pid, lvec, liovcnt, rvec,
	406	riovcnt, flags, 0);
	407	}
	408
	409	asmlinkage ssize_t
	410	compat_sys_process_vm_writev(compat_pid_t pid,
	411	const struct compat_iovec __user *lvec,
	412	unsigned long liovcnt,
	413	const struct compat_iovec __user *rvec,
	414	unsigned long riovcnt,
	415	unsigned long flags)
	416	{
	417	return compat_process_vm_rw(pid, lvec, liovcnt, rvec,
	418	riovcnt, flags, 1);
	419	}
	420
	421	#endif