[mirror_ubuntu-zesty-kernel.git] / mm / process_vm_access.c

/*
 * linux/mm/process_vm_access.c
 *
 * Copyright (C) 2010-2011 Christopher Yeoh <cyeoh@au1.ibm.com>, IBM Corp.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */

#include <linux/mm.h>
#include <linux/uio.h>
#include <linux/sched.h>
#include <linux/highmem.h>
#include <linux/ptrace.h>
#include <linux/slab.h>
#include <linux/syscalls.h>

#ifdef CONFIG_COMPAT
#include <linux/compat.h>
#endif

/**
 * process_vm_rw_pages - read/write pages from task specified
 * @task: task to read/write from
 * @mm: mm for task
 * @process_pages: struct pages area that can store at least
 *  nr_pages_to_copy struct page pointers
 * @pa: address of page in task to start copying from/to
 * @start_offset: offset in page to start copying from/to
 * @len: number of bytes to copy
 * @lvec: iovec array specifying where to copy to/from
 * @lvec_cnt: number of elements in iovec array
 * @lvec_current: index in iovec array we are up to
 * @lvec_offset: offset in bytes from current iovec iov_base we are up to
 * @vm_write: 0 means copy from, 1 means copy to
 * @nr_pages_to_copy: number of pages to copy
 * @bytes_copied: returns number of bytes successfully copied
 * Returns 0 on success, error code otherwise
 */
static int process_vm_rw_pages(struct task_struct *task,
			       struct mm_struct *mm,
			       struct page **process_pages,
			       unsigned long pa,
			       unsigned long start_offset,
			       unsigned long len,
			       struct iov_iter *iter,
			       int vm_write,
			       unsigned int nr_pages_to_copy,
			       ssize_t *bytes_copied)
{
	int pages_pinned;
	int pgs_copied = 0;
	int j;
	int ret;
	ssize_t bytes_to_copy;
	ssize_t rc = 0;

	*bytes_copied = 0;

	/* Get the pages we're interested in */
	down_read(&mm->mmap_sem);
	pages_pinned = get_user_pages(task, mm, pa,
				      nr_pages_to_copy,
				      vm_write, 0, process_pages, NULL);
	up_read(&mm->mmap_sem);

	if (pages_pinned != nr_pages_to_copy) {
		rc = -EFAULT;
		goto end;
	}

	/* Do the copy for each page */
	for (pgs_copied = 0;
	     (pgs_copied < nr_pages_to_copy) && iov_iter_count(iter);
	     pgs_copied++) {
		struct page *page = process_pages[pgs_copied];
		bytes_to_copy = min_t(ssize_t, PAGE_SIZE - start_offset, len);

		if (vm_write) {
			if (bytes_to_copy > iov_iter_count(iter))
				bytes_to_copy = iov_iter_count(iter);
			ret = iov_iter_copy_from_user(page,
					iter, start_offset, bytes_to_copy);
			iov_iter_advance(iter, ret);
			set_page_dirty_lock(page);
		} else {
			ret = copy_page_to_iter(page, start_offset,
					bytes_to_copy, iter);
		}
		*bytes_copied += ret;
		len -= ret;
		if (ret < bytes_to_copy && iov_iter_count(iter)) {
			rc = -EFAULT;
			break;
		}
		start_offset = 0;
	}

end:
	for (j = 0; j < pages_pinned; j++)
		put_page(process_pages[j]);
	return rc;
}

/* Maximum number of pages kmalloc'd to hold struct page's during copy */
#define PVM_MAX_KMALLOC_PAGES (PAGE_SIZE * 2)

/**
 * process_vm_rw_single_vec - read/write pages from task specified
 * @addr: start memory address of target process
 * @len: size of area to copy to/from
 * @lvec: iovec array specifying where to copy to/from locally
 * @lvec_cnt: number of elements in iovec array
 * @lvec_current: index in iovec array we are up to
 * @lvec_offset: offset in bytes from current iovec iov_base we are up to
 * @process_pages: struct pages area that can store at least
 *  nr_pages_to_copy struct page pointers
 * @mm: mm for task
 * @task: task to read/write from
 * @vm_write: 0 means copy from, 1 means copy to
 * @bytes_copied: returns number of bytes successfully copied
 * Returns 0 on success or on failure error code
 */
static int process_vm_rw_single_vec(unsigned long addr,
				    unsigned long len,
				    struct iov_iter *iter,
				    struct page **process_pages,
				    struct mm_struct *mm,
				    struct task_struct *task,
				    int vm_write,
				    ssize_t *bytes_copied)
{
	unsigned long pa = addr & PAGE_MASK;
	unsigned long start_offset = addr - pa;
	unsigned long nr_pages;
	ssize_t bytes_copied_loop;
	ssize_t rc = 0;
	unsigned long nr_pages_copied = 0;
	unsigned long nr_pages_to_copy;
	unsigned long max_pages_per_loop = PVM_MAX_KMALLOC_PAGES
		/ sizeof(struct pages *);

	*bytes_copied = 0;

	/* Work out address and page range required */
	if (len == 0)
		return 0;
	nr_pages = (addr + len - 1) / PAGE_SIZE - addr / PAGE_SIZE + 1;

	while ((nr_pages_copied < nr_pages) && iov_iter_count(iter)) {
		nr_pages_to_copy = min(nr_pages - nr_pages_copied,
				       max_pages_per_loop);

		rc = process_vm_rw_pages(task, mm, process_pages, pa,
					 start_offset, len, iter,
					 vm_write, nr_pages_to_copy,
					 &bytes_copied_loop);
		start_offset = 0;
		*bytes_copied += bytes_copied_loop;

		if (rc < 0) {
			return rc;
		} else {
			len -= bytes_copied_loop;
			nr_pages_copied += nr_pages_to_copy;
			pa += nr_pages_to_copy * PAGE_SIZE;
		}
	}

	return rc;
}

/* Maximum number of entries for process pages array
   which lives on stack */
#define PVM_MAX_PP_ARRAY_COUNT 16

/**
 * process_vm_rw_core - core of reading/writing pages from task specified
 * @pid: PID of process to read/write from/to
 * @lvec: iovec array specifying where to copy to/from locally
 * @liovcnt: size of lvec array
 * @rvec: iovec array specifying where to copy to/from in the other process
 * @riovcnt: size of rvec array
 * @flags: currently unused
 * @vm_write: 0 if reading from other process, 1 if writing to other process
 * Returns the number of bytes read/written or error code. May
 *  return less bytes than expected if an error occurs during the copying
 *  process.
 */
static ssize_t process_vm_rw_core(pid_t pid, struct iov_iter *iter,
				  const struct iovec *rvec,
				  unsigned long riovcnt,
				  unsigned long flags, int vm_write)
{
	struct task_struct *task;
	struct page *pp_stack[PVM_MAX_PP_ARRAY_COUNT];
	struct page **process_pages = pp_stack;
	struct mm_struct *mm;
	unsigned long i;
	ssize_t rc = 0;
	ssize_t bytes_copied_loop;
	ssize_t bytes_copied = 0;
	unsigned long nr_pages = 0;
	unsigned long nr_pages_iov;
	ssize_t iov_len;

	/*
	 * Work out how many pages of struct pages we're going to need
	 * when eventually calling get_user_pages
	 */
	for (i = 0; i < riovcnt; i++) {
		iov_len = rvec[i].iov_len;
		if (iov_len > 0) {
			nr_pages_iov = ((unsigned long)rvec[i].iov_base
					+ iov_len)
				/ PAGE_SIZE - (unsigned long)rvec[i].iov_base
				/ PAGE_SIZE + 1;
			nr_pages = max(nr_pages, nr_pages_iov);
		}
	}

	if (nr_pages == 0)
		return 0;

	if (nr_pages > PVM_MAX_PP_ARRAY_COUNT) {
		/* For reliability don't try to kmalloc more than
		   2 pages worth */
		process_pages = kmalloc(min_t(size_t, PVM_MAX_KMALLOC_PAGES,
					      sizeof(struct pages *)*nr_pages),
					GFP_KERNEL);

		if (!process_pages)
			return -ENOMEM;
	}

	/* Get process information */
	rcu_read_lock();
	task = find_task_by_vpid(pid);
	if (task)
		get_task_struct(task);
	rcu_read_unlock();
	if (!task) {
		rc = -ESRCH;
		goto free_proc_pages;
	}

	mm = mm_access(task, PTRACE_MODE_ATTACH);
	if (!mm || IS_ERR(mm)) {
		rc = IS_ERR(mm) ? PTR_ERR(mm) : -ESRCH;
		/*
		 * Explicitly map EACCES to EPERM as EPERM is a more a
		 * appropriate error code for process_vw_readv/writev
		 */
		if (rc == -EACCES)
			rc = -EPERM;
		goto put_task_struct;
	}

	for (i = 0; i < riovcnt && iov_iter_count(iter); i++) {
		rc = process_vm_rw_single_vec(
			(unsigned long)rvec[i].iov_base, rvec[i].iov_len,
			iter, process_pages, mm, task, vm_write,
			&bytes_copied_loop);
		bytes_copied += bytes_copied_loop;
		if (rc != 0) {
			/* If we have managed to copy any data at all then
			   we return the number of bytes copied. Otherwise
			   we return the error code */
			if (bytes_copied)
				rc = bytes_copied;
			goto put_mm;
		}
	}

	rc = bytes_copied;
put_mm:
	mmput(mm);

put_task_struct:
	put_task_struct(task);

free_proc_pages:
	if (process_pages != pp_stack)
		kfree(process_pages);
	return rc;
}

/**
 * process_vm_rw - check iovecs before calling core routine
 * @pid: PID of process to read/write from/to
 * @lvec: iovec array specifying where to copy to/from locally
 * @liovcnt: size of lvec array
 * @rvec: iovec array specifying where to copy to/from in the other process
 * @riovcnt: size of rvec array
 * @flags: currently unused
 * @vm_write: 0 if reading from other process, 1 if writing to other process
 * Returns the number of bytes read/written or error code. May
 *  return less bytes than expected if an error occurs during the copying
 *  process.
 */
static ssize_t process_vm_rw(pid_t pid,
			     const struct iovec __user *lvec,
			     unsigned long liovcnt,
			     const struct iovec __user *rvec,
			     unsigned long riovcnt,
			     unsigned long flags, int vm_write)
{
	struct iovec iovstack_l[UIO_FASTIOV];
	struct iovec iovstack_r[UIO_FASTIOV];
	struct iovec *iov_l = iovstack_l;
	struct iovec *iov_r = iovstack_r;
	struct iov_iter iter;
	ssize_t rc;

	if (flags != 0)
		return -EINVAL;

	/* Check iovecs */
	if (vm_write)
		rc = rw_copy_check_uvector(WRITE, lvec, liovcnt, UIO_FASTIOV,
					   iovstack_l, &iov_l);
	else
		rc = rw_copy_check_uvector(READ, lvec, liovcnt, UIO_FASTIOV,
					   iovstack_l, &iov_l);
	if (rc <= 0)
		goto free_iovecs;

	iov_iter_init(&iter, iov_l, liovcnt, rc, 0);

	rc = rw_copy_check_uvector(CHECK_IOVEC_ONLY, rvec, riovcnt, UIO_FASTIOV,
				   iovstack_r, &iov_r);
	if (rc <= 0)
		goto free_iovecs;

	rc = process_vm_rw_core(pid, &iter, iov_r, riovcnt, flags, vm_write);

free_iovecs:
	if (iov_r != iovstack_r)
		kfree(iov_r);
	if (iov_l != iovstack_l)
		kfree(iov_l);

	return rc;
}

SYSCALL_DEFINE6(process_vm_readv, pid_t, pid, const struct iovec __user *, lvec,
		unsigned long, liovcnt, const struct iovec __user *, rvec,
		unsigned long, riovcnt,	unsigned long, flags)
{
	return process_vm_rw(pid, lvec, liovcnt, rvec, riovcnt, flags, 0);
}

SYSCALL_DEFINE6(process_vm_writev, pid_t, pid,
		const struct iovec __user *, lvec,
		unsigned long, liovcnt, const struct iovec __user *, rvec,
		unsigned long, riovcnt,	unsigned long, flags)
{
	return process_vm_rw(pid, lvec, liovcnt, rvec, riovcnt, flags, 1);
}

#ifdef CONFIG_COMPAT

asmlinkage ssize_t
compat_process_vm_rw(compat_pid_t pid,
		     const struct compat_iovec __user *lvec,
		     unsigned long liovcnt,
		     const struct compat_iovec __user *rvec,
		     unsigned long riovcnt,
		     unsigned long flags, int vm_write)
{
	struct iovec iovstack_l[UIO_FASTIOV];
	struct iovec iovstack_r[UIO_FASTIOV];
	struct iovec *iov_l = iovstack_l;
	struct iovec *iov_r = iovstack_r;
	struct iov_iter iter;
	ssize_t rc = -EFAULT;

	if (flags != 0)
		return -EINVAL;

	if (vm_write)
		rc = compat_rw_copy_check_uvector(WRITE, lvec, liovcnt,
						  UIO_FASTIOV, iovstack_l,
						  &iov_l);
	else
		rc = compat_rw_copy_check_uvector(READ, lvec, liovcnt,
						  UIO_FASTIOV, iovstack_l,
						  &iov_l);
	if (rc <= 0)
		goto free_iovecs;
	iov_iter_init(&iter, iov_l, liovcnt, rc, 0);
	rc = compat_rw_copy_check_uvector(CHECK_IOVEC_ONLY, rvec, riovcnt,
					  UIO_FASTIOV, iovstack_r,
					  &iov_r);
	if (rc <= 0)
		goto free_iovecs;

	rc = process_vm_rw_core(pid, &iter, iov_r, riovcnt, flags, vm_write);

free_iovecs:
	if (iov_r != iovstack_r)
		kfree(iov_r);
	if (iov_l != iovstack_l)
		kfree(iov_l);
	return rc;
}

asmlinkage ssize_t
compat_sys_process_vm_readv(compat_pid_t pid,
			    const struct compat_iovec __user *lvec,
			    unsigned long liovcnt,
			    const struct compat_iovec __user *rvec,
			    unsigned long riovcnt,
			    unsigned long flags)
{
	return compat_process_vm_rw(pid, lvec, liovcnt, rvec,
				    riovcnt, flags, 0);
}

asmlinkage ssize_t
compat_sys_process_vm_writev(compat_pid_t pid,
			     const struct compat_iovec __user *lvec,
			     unsigned long liovcnt,
			     const struct compat_iovec __user *rvec,
			     unsigned long riovcnt,
			     unsigned long flags)
{
	return compat_process_vm_rw(pid, lvec, liovcnt, rvec,
				    riovcnt, flags, 1);
}

#endif
Commit	Line	Data
fcf63409 CY	1	/*
	2	* linux/mm/process_vm_access.c
	3	*
	4	* Copyright (C) 2010-2011 Christopher Yeoh <cyeoh@au1.ibm.com>, IBM Corp.
	5	*
	6	* This program is free software; you can redistribute it and/or
	7	* modify it under the terms of the GNU General Public License
	8	* as published by the Free Software Foundation; either version
	9	* 2 of the License, or (at your option) any later version.
	10	*/
	11
	12	#include <linux/mm.h>
	13	#include <linux/uio.h>
	14	#include <linux/sched.h>
	15	#include <linux/highmem.h>
	16	#include <linux/ptrace.h>
	17	#include <linux/slab.h>
	18	#include <linux/syscalls.h>
	19
	20	#ifdef CONFIG_COMPAT
	21	#include <linux/compat.h>
	22	#endif
	23
	24	/**
	25	* process_vm_rw_pages - read/write pages from task specified
	26	* @task: task to read/write from
	27	* @mm: mm for task
	28	* @process_pages: struct pages area that can store at least
	29	* nr_pages_to_copy struct page pointers
	30	* @pa: address of page in task to start copying from/to
	31	* @start_offset: offset in page to start copying from/to
	32	* @len: number of bytes to copy
	33	* @lvec: iovec array specifying where to copy to/from
	34	* @lvec_cnt: number of elements in iovec array
	35	* @lvec_current: index in iovec array we are up to
	36	* @lvec_offset: offset in bytes from current iovec iov_base we are up to
	37	* @vm_write: 0 means copy from, 1 means copy to
	38	* @nr_pages_to_copy: number of pages to copy
	39	* @bytes_copied: returns number of bytes successfully copied
	40	* Returns 0 on success, error code otherwise
	41	*/
	42	static int process_vm_rw_pages(struct task_struct *task,
	43	struct mm_struct *mm,
	44	struct page **process_pages,
	45	unsigned long pa,
	46	unsigned long start_offset,
	47	unsigned long len,
9f78bdfa	48	struct iov_iter *iter,
fcf63409 CY	49	int vm_write,
	50	unsigned int nr_pages_to_copy,
	51	ssize_t *bytes_copied)
	52	{
	53	int pages_pinned;
fcf63409 CY	54	int pgs_copied = 0;
	55	int j;
	56	int ret;
	57	ssize_t bytes_to_copy;
	58	ssize_t rc = 0;
	59
	60	*bytes_copied = 0;
	61
	62	/* Get the pages we're interested in */
	63	down_read(&mm->mmap_sem);
	64	pages_pinned = get_user_pages(task, mm, pa,
	65	nr_pages_to_copy,
	66	vm_write, 0, process_pages, NULL);
	67	up_read(&mm->mmap_sem);
	68
	69	if (pages_pinned != nr_pages_to_copy) {
	70	rc = -EFAULT;
	71	goto end;
	72	}
	73
	74	/* Do the copy for each page */
	75	for (pgs_copied = 0;
240f3905	76	(pgs_copied < nr_pages_to_copy) && iov_iter_count(iter);
fcf63409	77	pgs_copied++) {
240f3905 AV	78	struct page *page = process_pages[pgs_copied];
	79	bytes_to_copy = min_t(ssize_t, PAGE_SIZE - start_offset, len);
	80
	81	if (vm_write) {
	82	if (bytes_to_copy > iov_iter_count(iter))
	83	bytes_to_copy = iov_iter_count(iter);
	84	ret = iov_iter_copy_from_user(page,
	85	iter, start_offset, bytes_to_copy);
	86	iov_iter_advance(iter, ret);
	87	set_page_dirty_lock(page);
	88	} else {
	89	ret = copy_page_to_iter(page, start_offset,
	90	bytes_to_copy, iter);
480402e1	91	}
240f3905 AV	92	*bytes_copied += ret;
	93	len -= ret;
	94	if (ret < bytes_to_copy && iov_iter_count(iter)) {
fcf63409	95	rc = -EFAULT;
240f3905	96	break;
fcf63409	97	}
240f3905	98	start_offset = 0;
fcf63409 CY	99	}
	100
	101	end:
240f3905 AV	102	for (j = 0; j < pages_pinned; j++)
240f3905 AV	103	put_page(process_pages[j]);
fcf63409 CY	104	return rc;
	105	}
	106
	107	/* Maximum number of pages kmalloc'd to hold struct page's during copy */
	108	#define PVM_MAX_KMALLOC_PAGES (PAGE_SIZE * 2)
	109
	110	/**
	111	* process_vm_rw_single_vec - read/write pages from task specified
	112	* @addr: start memory address of target process
	113	* @len: size of area to copy to/from
	114	* @lvec: iovec array specifying where to copy to/from locally
	115	* @lvec_cnt: number of elements in iovec array
	116	* @lvec_current: index in iovec array we are up to
	117	* @lvec_offset: offset in bytes from current iovec iov_base we are up to
	118	* @process_pages: struct pages area that can store at least
	119	* nr_pages_to_copy struct page pointers
	120	* @mm: mm for task
	121	* @task: task to read/write from
	122	* @vm_write: 0 means copy from, 1 means copy to
	123	* @bytes_copied: returns number of bytes successfully copied
	124	* Returns 0 on success or on failure error code
	125	*/
	126	static int process_vm_rw_single_vec(unsigned long addr,
	127	unsigned long len,
9f78bdfa	128	struct iov_iter *iter,
fcf63409 CY	129	struct page **process_pages,
	130	struct mm_struct *mm,
	131	struct task_struct *task,
	132	int vm_write,
	133	ssize_t *bytes_copied)
	134	{
	135	unsigned long pa = addr & PAGE_MASK;
	136	unsigned long start_offset = addr - pa;
	137	unsigned long nr_pages;
	138	ssize_t bytes_copied_loop;
	139	ssize_t rc = 0;
	140	unsigned long nr_pages_copied = 0;
	141	unsigned long nr_pages_to_copy;
	142	unsigned long max_pages_per_loop = PVM_MAX_KMALLOC_PAGES
	143	/ sizeof(struct pages *);
	144
	145	*bytes_copied = 0;
	146
	147	/* Work out address and page range required */
	148	if (len == 0)
	149	return 0;
	150	nr_pages = (addr + len - 1) / PAGE_SIZE - addr / PAGE_SIZE + 1;
	151
240f3905	152	while ((nr_pages_copied < nr_pages) && iov_iter_count(iter)) {
fcf63409 CY	153	nr_pages_to_copy = min(nr_pages - nr_pages_copied,
	154	max_pages_per_loop);
	155
	156	rc = process_vm_rw_pages(task, mm, process_pages, pa,
9f78bdfa	157	start_offset, len, iter,
fcf63409 CY	158	vm_write, nr_pages_to_copy,
	159	&bytes_copied_loop);
	160	start_offset = 0;
	161	*bytes_copied += bytes_copied_loop;
	162
	163	if (rc < 0) {
	164	return rc;
	165	} else {
	166	len -= bytes_copied_loop;
	167	nr_pages_copied += nr_pages_to_copy;
	168	pa += nr_pages_to_copy * PAGE_SIZE;
	169	}
	170	}
	171
	172	return rc;
	173	}
	174
	175	/* Maximum number of entries for process pages array
	176	which lives on stack */
	177	#define PVM_MAX_PP_ARRAY_COUNT 16
	178
	179	/**
	180	* process_vm_rw_core - core of reading/writing pages from task specified
	181	* @pid: PID of process to read/write from/to
	182	* @lvec: iovec array specifying where to copy to/from locally
	183	* @liovcnt: size of lvec array
	184	* @rvec: iovec array specifying where to copy to/from in the other process
	185	* @riovcnt: size of rvec array
	186	* @flags: currently unused
	187	* @vm_write: 0 if reading from other process, 1 if writing to other process
	188	* Returns the number of bytes read/written or error code. May
	189	* return less bytes than expected if an error occurs during the copying
	190	* process.
	191	*/
9f78bdfa	192	static ssize_t process_vm_rw_core(pid_t pid, struct iov_iter *iter,
fcf63409 CY	193	const struct iovec *rvec,
	194	unsigned long riovcnt,
	195	unsigned long flags, int vm_write)
	196	{
	197	struct task_struct *task;
	198	struct page *pp_stack[PVM_MAX_PP_ARRAY_COUNT];
	199	struct page **process_pages = pp_stack;
	200	struct mm_struct *mm;
	201	unsigned long i;
	202	ssize_t rc = 0;
	203	ssize_t bytes_copied_loop;
	204	ssize_t bytes_copied = 0;
	205	unsigned long nr_pages = 0;
	206	unsigned long nr_pages_iov;
fcf63409 CY	207	ssize_t iov_len;
	208
	209	/*
	210	* Work out how many pages of struct pages we're going to need
	211	* when eventually calling get_user_pages
	212	*/
	213	for (i = 0; i < riovcnt; i++) {
	214	iov_len = rvec[i].iov_len;
	215	if (iov_len > 0) {
	216	nr_pages_iov = ((unsigned long)rvec[i].iov_base
	217	+ iov_len)
	218	/ PAGE_SIZE - (unsigned long)rvec[i].iov_base
	219	/ PAGE_SIZE + 1;
	220	nr_pages = max(nr_pages, nr_pages_iov);
	221	}
	222	}
	223
	224	if (nr_pages == 0)
	225	return 0;
	226
	227	if (nr_pages > PVM_MAX_PP_ARRAY_COUNT) {
	228	/* For reliability don't try to kmalloc more than
	229	2 pages worth */
	230	process_pages = kmalloc(min_t(size_t, PVM_MAX_KMALLOC_PAGES,
	231	sizeof(struct pages )nr_pages),
	232	GFP_KERNEL);
	233
	234	if (!process_pages)
	235	return -ENOMEM;
	236	}
	237
	238	/* Get process information */
	239	rcu_read_lock();
	240	task = find_task_by_vpid(pid);
	241	if (task)
	242	get_task_struct(task);
	243	rcu_read_unlock();
	244	if (!task) {
	245	rc = -ESRCH;
	246	goto free_proc_pages;
	247	}
	248
8cdb878d CY	249	mm = mm_access(task, PTRACE_MODE_ATTACH);
	250	if (!mm \|\| IS_ERR(mm)) {
	251	rc = IS_ERR(mm) ? PTR_ERR(mm) : -ESRCH;
	252	/*
	253	* Explicitly map EACCES to EPERM as EPERM is a more a
	254	* appropriate error code for process_vw_readv/writev
	255	*/
	256	if (rc == -EACCES)
	257	rc = -EPERM;
fcf63409 CY	258	goto put_task_struct;
	259	}
	260
240f3905	261	for (i = 0; i < riovcnt && iov_iter_count(iter); i++) {
fcf63409 CY	262	rc = process_vm_rw_single_vec(
fcf63409 CY	263	(unsigned long)rvec[i].iov_base, rvec[i].iov_len,
9f78bdfa AV	264	iter, process_pages, mm, task, vm_write,
9f78bdfa AV	265	&bytes_copied_loop);
fcf63409 CY	266	bytes_copied += bytes_copied_loop;
	267	if (rc != 0) {
	268	/* If we have managed to copy any data at all then
	269	we return the number of bytes copied. Otherwise
	270	we return the error code */
	271	if (bytes_copied)
	272	rc = bytes_copied;
	273	goto put_mm;
	274	}
	275	}
	276
	277	rc = bytes_copied;
	278	put_mm:
	279	mmput(mm);
	280
	281	put_task_struct:
	282	put_task_struct(task);
	283
	284	free_proc_pages:
	285	if (process_pages != pp_stack)
	286	kfree(process_pages);
	287	return rc;
	288	}
	289
	290	/**
	291	* process_vm_rw - check iovecs before calling core routine
	292	* @pid: PID of process to read/write from/to
	293	* @lvec: iovec array specifying where to copy to/from locally
	294	* @liovcnt: size of lvec array
	295	* @rvec: iovec array specifying where to copy to/from in the other process
	296	* @riovcnt: size of rvec array
	297	* @flags: currently unused
	298	* @vm_write: 0 if reading from other process, 1 if writing to other process
	299	* Returns the number of bytes read/written or error code. May
	300	* return less bytes than expected if an error occurs during the copying
	301	* process.
	302	*/
	303	static ssize_t process_vm_rw(pid_t pid,
	304	const struct iovec __user *lvec,
	305	unsigned long liovcnt,
	306	const struct iovec __user *rvec,
	307	unsigned long riovcnt,
	308	unsigned long flags, int vm_write)
	309	{
	310	struct iovec iovstack_l[UIO_FASTIOV];
	311	struct iovec iovstack_r[UIO_FASTIOV];
	312	struct iovec *iov_l = iovstack_l;
	313	struct iovec *iov_r = iovstack_r;
9f78bdfa	314	struct iov_iter iter;
fcf63409 CY	315	ssize_t rc;
	316
	317	if (flags != 0)
	318	return -EINVAL;
	319
	320	/* Check iovecs */
	321	if (vm_write)
	322	rc = rw_copy_check_uvector(WRITE, lvec, liovcnt, UIO_FASTIOV,
ac34ebb3	323	iovstack_l, &iov_l);
fcf63409 CY	324	else
fcf63409 CY	325	rc = rw_copy_check_uvector(READ, lvec, liovcnt, UIO_FASTIOV,
ac34ebb3	326	iovstack_l, &iov_l);
fcf63409 CY	327	if (rc <= 0)
	328	goto free_iovecs;
	329
9f78bdfa AV	330	iov_iter_init(&iter, iov_l, liovcnt, rc, 0);
9f78bdfa AV	331
ac34ebb3 CY	332	rc = rw_copy_check_uvector(CHECK_IOVEC_ONLY, rvec, riovcnt, UIO_FASTIOV,
ac34ebb3 CY	333	iovstack_r, &iov_r);
fcf63409 CY	334	if (rc <= 0)
	335	goto free_iovecs;
	336
9f78bdfa	337	rc = process_vm_rw_core(pid, &iter, iov_r, riovcnt, flags, vm_write);
fcf63409 CY	338
	339	free_iovecs:
	340	if (iov_r != iovstack_r)
	341	kfree(iov_r);
	342	if (iov_l != iovstack_l)
	343	kfree(iov_l);
	344
	345	return rc;
	346	}
	347
	348	SYSCALL_DEFINE6(process_vm_readv, pid_t, pid, const struct iovec __user *, lvec,
	349	unsigned long, liovcnt, const struct iovec __user *, rvec,
	350	unsigned long, riovcnt, unsigned long, flags)
	351	{
	352	return process_vm_rw(pid, lvec, liovcnt, rvec, riovcnt, flags, 0);
	353	}
	354
	355	SYSCALL_DEFINE6(process_vm_writev, pid_t, pid,
	356	const struct iovec __user *, lvec,
	357	unsigned long, liovcnt, const struct iovec __user *, rvec,
	358	unsigned long, riovcnt, unsigned long, flags)
	359	{
	360	return process_vm_rw(pid, lvec, liovcnt, rvec, riovcnt, flags, 1);
	361	}
	362
	363	#ifdef CONFIG_COMPAT
	364
	365	asmlinkage ssize_t
	366	compat_process_vm_rw(compat_pid_t pid,
	367	const struct compat_iovec __user *lvec,
	368	unsigned long liovcnt,
	369	const struct compat_iovec __user *rvec,
	370	unsigned long riovcnt,
	371	unsigned long flags, int vm_write)
	372	{
	373	struct iovec iovstack_l[UIO_FASTIOV];
	374	struct iovec iovstack_r[UIO_FASTIOV];
	375	struct iovec *iov_l = iovstack_l;
	376	struct iovec *iov_r = iovstack_r;
9f78bdfa	377	struct iov_iter iter;
fcf63409 CY	378	ssize_t rc = -EFAULT;
	379
	380	if (flags != 0)
	381	return -EINVAL;
	382
fcf63409 CY	383	if (vm_write)
	384	rc = compat_rw_copy_check_uvector(WRITE, lvec, liovcnt,
	385	UIO_FASTIOV, iovstack_l,
ac34ebb3	386	&iov_l);
fcf63409 CY	387	else
	388	rc = compat_rw_copy_check_uvector(READ, lvec, liovcnt,
	389	UIO_FASTIOV, iovstack_l,
ac34ebb3	390	&iov_l);
fcf63409 CY	391	if (rc <= 0)
fcf63409 CY	392	goto free_iovecs;
9f78bdfa	393	iov_iter_init(&iter, iov_l, liovcnt, rc, 0);
ac34ebb3	394	rc = compat_rw_copy_check_uvector(CHECK_IOVEC_ONLY, rvec, riovcnt,
fcf63409	395	UIO_FASTIOV, iovstack_r,
ac34ebb3	396	&iov_r);
fcf63409 CY	397	if (rc <= 0)
	398	goto free_iovecs;
	399
9f78bdfa	400	rc = process_vm_rw_core(pid, &iter, iov_r, riovcnt, flags, vm_write);
fcf63409 CY	401
	402	free_iovecs:
	403	if (iov_r != iovstack_r)
	404	kfree(iov_r);
	405	if (iov_l != iovstack_l)
	406	kfree(iov_l);
fcf63409 CY	407	return rc;
	408	}
	409
	410	asmlinkage ssize_t
	411	compat_sys_process_vm_readv(compat_pid_t pid,
	412	const struct compat_iovec __user *lvec,
	413	unsigned long liovcnt,
	414	const struct compat_iovec __user *rvec,
	415	unsigned long riovcnt,
	416	unsigned long flags)
	417	{
	418	return compat_process_vm_rw(pid, lvec, liovcnt, rvec,
	419	riovcnt, flags, 0);
	420	}
	421
	422	asmlinkage ssize_t
	423	compat_sys_process_vm_writev(compat_pid_t pid,
	424	const struct compat_iovec __user *lvec,
	425	unsigned long liovcnt,
	426	const struct compat_iovec __user *rvec,
	427	unsigned long riovcnt,
	428	unsigned long flags)
	429	{
	430	return compat_process_vm_rw(pid, lvec, liovcnt, rvec,
	431	riovcnt, flags, 1);
	432	}
	433
	434	#endif