[mirror_zfs.git] / module / zcommon / zfs_uio.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
/*	  All Rights Reserved	*/

/*
 * University Copyright- Copyright (c) 1982, 1986, 1988
 * The Regents of the University of California
 * All Rights Reserved
 *
 * University Acknowledgment- Portions of this document are derived from
 * software developed by the University of California, Berkeley, and its
 * contributors.
 */
/*
 * Copyright (c) 2015 by Chunwei Chen. All rights reserved.
 */

/*
 * The uio support from OpenSolaris has been added as a short term
 * work around.  The hope is to adopt native Linux type and drop the
 * use of uio's entirely.  Under Linux they only add overhead and
 * when possible we want to use native APIs for the ZPL layer.
 */
#ifdef _KERNEL

#include <sys/types.h>
#include <sys/uio_impl.h>
#include <linux/kmap_compat.h>
#include <linux/uaccess.h>

/*
 * Move "n" bytes at byte address "p"; "rw" indicates the direction
 * of the move, and the I/O parameters are provided in "uio", which is
 * update to reflect the data which was moved.  Returns 0 on success or
 * a non-zero errno on failure.
 */
static int
uiomove_iov(void *p, size_t n, enum uio_rw rw, struct uio *uio)
{
	const struct iovec *iov = uio->uio_iov;
	size_t skip = uio->uio_skip;
	ulong_t cnt;

	while (n && uio->uio_resid) {
		cnt = MIN(iov->iov_len - skip, n);
		switch (uio->uio_segflg) {
		case UIO_USERSPACE:
		case UIO_USERISPACE:
			/*
			 * p = kernel data pointer
			 * iov->iov_base = user data pointer
			 */
			if (rw == UIO_READ) {
				if (copy_to_user(iov->iov_base+skip, p, cnt))
					return (EFAULT);
			} else {
				if (uio->uio_fault_disable) {
					if (!access_ok(VERIFY_READ,
					    (iov->iov_base + skip), cnt)) {
						return (EFAULT);
					}

					pagefault_disable();
					if (__copy_from_user_inatomic(p,
					    (iov->iov_base + skip), cnt)) {
						pagefault_enable();
						return (EFAULT);
					}
					pagefault_enable();
				} else {
					if (copy_from_user(p,
					    (iov->iov_base + skip), cnt))
						return (EFAULT);
				}
			}
			break;
		case UIO_SYSSPACE:
			if (rw == UIO_READ)
				bcopy(p, iov->iov_base + skip, cnt);
			else
				bcopy(iov->iov_base + skip, p, cnt);
			break;
		default:
			ASSERT(0);
		}
		skip += cnt;
		if (skip == iov->iov_len) {
			skip = 0;
			uio->uio_iov = (++iov);
			uio->uio_iovcnt--;
		}
		uio->uio_skip = skip;
		uio->uio_resid -= cnt;
		uio->uio_loffset += cnt;
		p = (caddr_t)p + cnt;
		n -= cnt;
	}
	return (0);
}

static int
uiomove_bvec(void *p, size_t n, enum uio_rw rw, struct uio *uio)
{
	const struct bio_vec *bv = uio->uio_bvec;
	size_t skip = uio->uio_skip;
	ulong_t cnt;

	while (n && uio->uio_resid) {
		void *paddr;
		cnt = MIN(bv->bv_len - skip, n);

		paddr = zfs_kmap_atomic(bv->bv_page, KM_USER1);
		if (rw == UIO_READ)
			bcopy(p, paddr + bv->bv_offset + skip, cnt);
		else
			bcopy(paddr + bv->bv_offset + skip, p, cnt);
		zfs_kunmap_atomic(paddr, KM_USER1);

		skip += cnt;
		if (skip == bv->bv_len) {
			skip = 0;
			uio->uio_bvec = (++bv);
			uio->uio_iovcnt--;
		}
		uio->uio_skip = skip;
		uio->uio_resid -= cnt;
		uio->uio_loffset += cnt;
		p = (caddr_t)p + cnt;
		n -= cnt;
	}
	return (0);
}

int
uiomove(void *p, size_t n, enum uio_rw rw, struct uio *uio)
{
	if (uio->uio_segflg != UIO_BVEC)
		return (uiomove_iov(p, n, rw, uio));
	else
		return (uiomove_bvec(p, n, rw, uio));
}
EXPORT_SYMBOL(uiomove);

#define	fuword8(uptr, vptr)	get_user((*vptr), (uptr))

/*
 * Fault in the pages of the first n bytes specified by the uio structure.
 * 1 byte in each page is touched and the uio struct is unmodified. Any
 * error will terminate the process as this is only a best attempt to get
 * the pages resident.
 */
int
uio_prefaultpages(ssize_t n, struct uio *uio)
{
	const struct iovec *iov;
	ulong_t cnt, incr;
	caddr_t p;
	uint8_t tmp;
	int iovcnt;
	size_t skip;

	/* no need to fault in kernel pages */
	switch (uio->uio_segflg) {
		case UIO_SYSSPACE:
		case UIO_BVEC:
			return (0);
		case UIO_USERSPACE:
		case UIO_USERISPACE:
			break;
		default:
			ASSERT(0);
	}

	iov = uio->uio_iov;
	iovcnt = uio->uio_iovcnt;
	skip = uio->uio_skip;

	for (; n > 0 && iovcnt > 0; iov++, iovcnt--, skip = 0) {
		cnt = MIN(iov->iov_len - skip, n);
		/* empty iov */
		if (cnt == 0)
			continue;
		n -= cnt;
		/*
		 * touch each page in this segment.
		 */
		p = iov->iov_base + skip;
		while (cnt) {
			if (fuword8((uint8_t *)p, &tmp))
				return (EFAULT);
			incr = MIN(cnt, PAGESIZE);
			p += incr;
			cnt -= incr;
		}
		/*
		 * touch the last byte in case it straddles a page.
		 */
		p--;
		if (fuword8((uint8_t *)p, &tmp))
			return (EFAULT);
	}

	return (0);
}
EXPORT_SYMBOL(uio_prefaultpages);

/*
 * same as uiomove() but doesn't modify uio structure.
 * return in cbytes how many bytes were copied.
 */
int
uiocopy(void *p, size_t n, enum uio_rw rw, struct uio *uio, size_t *cbytes)
{
	struct uio uio_copy;
	int ret;

	bcopy(uio, &uio_copy, sizeof (struct uio));
	ret = uiomove(p, n, rw, &uio_copy);
	*cbytes = uio->uio_resid - uio_copy.uio_resid;
	return (ret);
}
EXPORT_SYMBOL(uiocopy);

/*
 * Drop the next n chars out of *uiop.
 */
void
uioskip(uio_t *uiop, size_t n)
{
	if (n > uiop->uio_resid)
		return;

	uiop->uio_skip += n;
	if (uiop->uio_segflg != UIO_BVEC) {
		while (uiop->uio_iovcnt &&
		    uiop->uio_skip >= uiop->uio_iov->iov_len) {
			uiop->uio_skip -= uiop->uio_iov->iov_len;
			uiop->uio_iov++;
			uiop->uio_iovcnt--;
		}
	} else {
		while (uiop->uio_iovcnt &&
		    uiop->uio_skip >= uiop->uio_bvec->bv_len) {
			uiop->uio_skip -= uiop->uio_bvec->bv_len;
			uiop->uio_bvec++;
			uiop->uio_iovcnt--;
		}
	}
	uiop->uio_loffset += n;
	uiop->uio_resid -= n;
}
EXPORT_SYMBOL(uioskip);
#endif /* _KERNEL */
Commit	Line	Data
590329b5 BB	1	/*
	2	* CDDL HEADER START
	3	*
	4	* The contents of this file are subject to the terms of the
	5	* Common Development and Distribution License (the "License").
	6	* You may not use this file except in compliance with the License.
	7	*
	8	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	9	* or http://www.opensolaris.org/os/licensing.
	10	* See the License for the specific language governing permissions
	11	* and limitations under the License.
	12	*
	13	* When distributing Covered Code, include this CDDL HEADER in each
	14	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	15	* If applicable, add the following below this CDDL HEADER, with the
	16	* fields enclosed by brackets "[]" replaced with your own identifying
	17	* information: Portions Copyright [yyyy] [name of copyright owner]
	18	*
	19	* CDDL HEADER END
	20	*/
	21	/*
	22	* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
	23	* Use is subject to license terms.
	24	*/
	25
	26	/* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
	27	/* All Rights Reserved */
	28
	29	/*
	30	* University Copyright- Copyright (c) 1982, 1986, 1988
	31	* The Regents of the University of California
	32	* All Rights Reserved
	33	*
	34	* University Acknowledgment- Portions of this document are derived from
	35	* software developed by the University of California, Berkeley, and its
	36	* contributors.
	37	*/
5475aada CC	38	/*
	39	* Copyright (c) 2015 by Chunwei Chen. All rights reserved.
	40	*/
590329b5 BB	41
	42	/*
	43	* The uio support from OpenSolaris has been added as a short term
	44	* work around. The hope is to adopt native Linux type and drop the
	45	* use of uio's entirely. Under Linux they only add overhead and
	46	* when possible we want to use native APIs for the ZPL layer.
	47	*/
	48	#ifdef _KERNEL
	49
	50	#include <sys/types.h>
	51	#include <sys/uio_impl.h>
5475aada	52	#include <linux/kmap_compat.h>
98bb45e2	53	#include <linux/uaccess.h>
590329b5 BB	54
	55	/*
	56	* Move "n" bytes at byte address "p"; "rw" indicates the direction
	57	* of the move, and the I/O parameters are provided in "uio", which is
	58	* update to reflect the data which was moved. Returns 0 on success or
	59	* a non-zero errno on failure.
	60	*/
5475aada CC	61	static int
5475aada CC	62	uiomove_iov(void p, size_t n, enum uio_rw rw, struct uio uio)
590329b5	63	{
5475aada CC	64	const struct iovec *iov = uio->uio_iov;
5475aada CC	65	size_t skip = uio->uio_skip;
590329b5 BB	66	ulong_t cnt;
	67
	68	while (n && uio->uio_resid) {
5475aada	69	cnt = MIN(iov->iov_len - skip, n);
590329b5 BB	70	switch (uio->uio_segflg) {
	71	case UIO_USERSPACE:
	72	case UIO_USERISPACE:
d1d7e268 MK	73	/*
	74	* p = kernel data pointer
	75	* iov->iov_base = user data pointer
	76	*/
590329b5	77	if (rw == UIO_READ) {
5475aada	78	if (copy_to_user(iov->iov_base+skip, p, cnt))
d1d7e268	79	return (EFAULT);
590329b5	80	} else {
98bb45e2	81	if (uio->uio_fault_disable) {
	82	if (!access_ok(VERIFY_READ,
	83	(iov->iov_base + skip), cnt)) {
	84	return (EFAULT);
	85	}
	86
	87	pagefault_disable();
	88	if (__copy_from_user_inatomic(p,
	89	(iov->iov_base + skip), cnt)) {
	90	pagefault_enable();
	91	return (EFAULT);
	92	}
	93	pagefault_enable();
	94	} else {
	95	if (copy_from_user(p,
	96	(iov->iov_base + skip), cnt))
	97	return (EFAULT);
	98	}
590329b5 BB	99	}
	100	break;
	101	case UIO_SYSSPACE:
	102	if (rw == UIO_READ)
5475aada	103	bcopy(p, iov->iov_base + skip, cnt);
590329b5	104	else
5475aada	105	bcopy(iov->iov_base + skip, p, cnt);
590329b5	106	break;
5475aada CC	107	default:
	108	ASSERT(0);
	109	}
	110	skip += cnt;
	111	if (skip == iov->iov_len) {
	112	skip = 0;
	113	uio->uio_iov = (++iov);
	114	uio->uio_iovcnt--;
590329b5	115	}
5475aada	116	uio->uio_skip = skip;
590329b5 BB	117	uio->uio_resid -= cnt;
	118	uio->uio_loffset += cnt;
	119	p = (caddr_t)p + cnt;
	120	n -= cnt;
	121	}
	122	return (0);
	123	}
5475aada CC	124
	125	static int
	126	uiomove_bvec(void p, size_t n, enum uio_rw rw, struct uio uio)
	127	{
	128	const struct bio_vec *bv = uio->uio_bvec;
	129	size_t skip = uio->uio_skip;
	130	ulong_t cnt;
	131
5475aada CC	132	while (n && uio->uio_resid) {
	133	void *paddr;
	134	cnt = MIN(bv->bv_len - skip, n);
	135
	136	paddr = zfs_kmap_atomic(bv->bv_page, KM_USER1);
	137	if (rw == UIO_READ)
	138	bcopy(p, paddr + bv->bv_offset + skip, cnt);
	139	else
	140	bcopy(paddr + bv->bv_offset + skip, p, cnt);
	141	zfs_kunmap_atomic(paddr, KM_USER1);
	142
	143	skip += cnt;
	144	if (skip == bv->bv_len) {
	145	skip = 0;
	146	uio->uio_bvec = (++bv);
	147	uio->uio_iovcnt--;
	148	}
	149	uio->uio_skip = skip;
	150	uio->uio_resid -= cnt;
	151	uio->uio_loffset += cnt;
	152	p = (caddr_t)p + cnt;
	153	n -= cnt;
	154	}
	155	return (0);
	156	}
	157
	158	int
	159	uiomove(void p, size_t n, enum uio_rw rw, struct uio uio)
	160	{
	161	if (uio->uio_segflg != UIO_BVEC)
	162	return (uiomove_iov(p, n, rw, uio));
	163	else
	164	return (uiomove_bvec(p, n, rw, uio));
	165	}
590329b5 BB	166	EXPORT_SYMBOL(uiomove);
590329b5 BB	167
d1d7e268	168	#define fuword8(uptr, vptr) get_user((*vptr), (uptr))
590329b5 BB	169
	170	/*
	171	* Fault in the pages of the first n bytes specified by the uio structure.
	172	* 1 byte in each page is touched and the uio struct is unmodified. Any
	173	* error will terminate the process as this is only a best attempt to get
	174	* the pages resident.
	175	*/
98bb45e2	176	int
590329b5 BB	177	uio_prefaultpages(ssize_t n, struct uio *uio)
590329b5 BB	178	{
5475aada	179	const struct iovec *iov;
590329b5 BB	180	ulong_t cnt, incr;
	181	caddr_t p;
	182	uint8_t tmp;
	183	int iovcnt;
502923bb	184	size_t skip;
5475aada CC	185
	186	/* no need to fault in kernel pages */
	187	switch (uio->uio_segflg) {
	188	case UIO_SYSSPACE:
	189	case UIO_BVEC:
98bb45e2	190	return (0);
5475aada CC	191	case UIO_USERSPACE:
	192	case UIO_USERISPACE:
	193	break;
	194	default:
	195	ASSERT(0);
	196	}
590329b5 BB	197
	198	iov = uio->uio_iov;
	199	iovcnt = uio->uio_iovcnt;
502923bb	200	skip = uio->uio_skip;
590329b5	201
502923bb	202	for (; n > 0 && iovcnt > 0; iov++, iovcnt--, skip = 0) {
5475aada	203	cnt = MIN(iov->iov_len - skip, n);
502923bb CC	204	/* empty iov */
	205	if (cnt == 0)
	206	continue;
590329b5 BB	207	n -= cnt;
	208	/*
	209	* touch each page in this segment.
	210	*/
5475aada	211	p = iov->iov_base + skip;
590329b5	212	while (cnt) {
02730c33	213	if (fuword8((uint8_t *)p, &tmp))
98bb45e2	214	return (EFAULT);
590329b5 BB	215	incr = MIN(cnt, PAGESIZE);
	216	p += incr;
	217	cnt -= incr;
	218	}
	219	/*
	220	* touch the last byte in case it straddles a page.
	221	*/
	222	p--;
02730c33	223	if (fuword8((uint8_t *)p, &tmp))
98bb45e2	224	return (EFAULT);
590329b5	225	}
98bb45e2	226
98bb45e2	227	return (0);
590329b5 BB	228	}
	229	EXPORT_SYMBOL(uio_prefaultpages);
	230
	231	/*
	232	* same as uiomove() but doesn't modify uio structure.
	233	* return in cbytes how many bytes were copied.
	234	*/
	235	int
	236	uiocopy(void p, size_t n, enum uio_rw rw, struct uio uio, size_t *cbytes)
	237	{
5475aada CC	238	struct uio uio_copy;
5475aada CC	239	int ret;
590329b5	240
5475aada CC	241	bcopy(uio, &uio_copy, sizeof (struct uio));
	242	ret = uiomove(p, n, rw, &uio_copy);
	243	*cbytes = uio->uio_resid - uio_copy.uio_resid;
	244	return (ret);
590329b5 BB	245	}
	246	EXPORT_SYMBOL(uiocopy);
	247
	248	/*
	249	* Drop the next n chars out of *uiop.
	250	*/
	251	void
	252	uioskip(uio_t *uiop, size_t n)
	253	{
	254	if (n > uiop->uio_resid)
	255	return;
590329b5	256
5475aada CC	257	uiop->uio_skip += n;
5475aada CC	258	if (uiop->uio_segflg != UIO_BVEC) {
45838e3a CC	259	while (uiop->uio_iovcnt &&
45838e3a CC	260	uiop->uio_skip >= uiop->uio_iov->iov_len) {
5475aada	261	uiop->uio_skip -= uiop->uio_iov->iov_len;
590329b5 BB	262	uiop->uio_iov++;
590329b5 BB	263	uiop->uio_iovcnt--;
590329b5	264	}
5475aada	265	} else {
45838e3a CC	266	while (uiop->uio_iovcnt &&
45838e3a CC	267	uiop->uio_skip >= uiop->uio_bvec->bv_len) {
5475aada CC	268	uiop->uio_skip -= uiop->uio_bvec->bv_len;
	269	uiop->uio_bvec++;
	270	uiop->uio_iovcnt--;
	271	}
590329b5	272	}
5475aada CC	273	uiop->uio_loffset += n;
5475aada CC	274	uiop->uio_resid -= n;
590329b5 BB	275	}
	276	EXPORT_SYMBOL(uioskip);
	277	#endif /* _KERNEL */