[mirror_ubuntu-jammy-kernel.git] / arch / arm64 / kernel / mte.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2020 ARM Ltd.
 */

#include <linux/bitops.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/prctl.h>
#include <linux/sched.h>
#include <linux/sched/mm.h>
#include <linux/string.h>
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/thread_info.h>
#include <linux/types.h>
#include <linux/uio.h>

#include <asm/barrier.h>
#include <asm/cpufeature.h>
#include <asm/mte.h>
#include <asm/mte-kasan.h>
#include <asm/ptrace.h>
#include <asm/sysreg.h>

static void mte_sync_page_tags(struct page *page, pte_t *ptep, bool check_swap)
{
	pte_t old_pte = READ_ONCE(*ptep);

	if (check_swap && is_swap_pte(old_pte)) {
		swp_entry_t entry = pte_to_swp_entry(old_pte);

		if (!non_swap_entry(entry) && mte_restore_tags(entry, page))
			return;
	}

	page_kasan_tag_reset(page);
	/*
	 * We need smp_wmb() in between setting the flags and clearing the
	 * tags because if another thread reads page->flags and builds a
	 * tagged address out of it, there is an actual dependency to the
	 * memory access, but on the current thread we do not guarantee that
	 * the new page->flags are visible before the tags were updated.
	 */
	smp_wmb();
	mte_clear_page_tags(page_address(page));
}

void mte_sync_tags(pte_t *ptep, pte_t pte)
{
	struct page *page = pte_page(pte);
	long i, nr_pages = compound_nr(page);
	bool check_swap = nr_pages == 1;

	/* if PG_mte_tagged is set, tags have already been initialised */
	for (i = 0; i < nr_pages; i++, page++) {
		if (!test_and_set_bit(PG_mte_tagged, &page->flags))
			mte_sync_page_tags(page, ptep, check_swap);
	}
}

int memcmp_pages(struct page *page1, struct page *page2)
{
	char *addr1, *addr2;
	int ret;

	addr1 = page_address(page1);
	addr2 = page_address(page2);
	ret = memcmp(addr1, addr2, PAGE_SIZE);

	if (!system_supports_mte() || ret)
		return ret;

	/*
	 * If the page content is identical but at least one of the pages is
	 * tagged, return non-zero to avoid KSM merging. If only one of the
	 * pages is tagged, set_pte_at() may zero or change the tags of the
	 * other page via mte_sync_tags().
	 */
	if (test_bit(PG_mte_tagged, &page1->flags) ||
	    test_bit(PG_mte_tagged, &page2->flags))
		return addr1 != addr2;

	return ret;
}

u8 mte_get_mem_tag(void *addr)
{
	if (!system_supports_mte())
		return 0xFF;

	asm(__MTE_PREAMBLE "ldg %0, [%0]"
	    : "+r" (addr));

	return mte_get_ptr_tag(addr);
}

u8 mte_get_random_tag(void)
{
	void *addr;

	if (!system_supports_mte())
		return 0xFF;

	asm(__MTE_PREAMBLE "irg %0, %0"
	    : "+r" (addr));

	return mte_get_ptr_tag(addr);
}

void *mte_set_mem_tag_range(void *addr, size_t size, u8 tag)
{
	void *ptr = addr;

	if ((!system_supports_mte()) || (size == 0))
		return addr;

	/* Make sure that size is MTE granule aligned. */
	WARN_ON(size & (MTE_GRANULE_SIZE - 1));

	/* Make sure that the address is MTE granule aligned. */
	WARN_ON((u64)addr & (MTE_GRANULE_SIZE - 1));

	tag = 0xF0 | tag;
	ptr = (void *)__tag_set(ptr, tag);

	mte_assign_mem_tag_range(ptr, size);

	return ptr;
}

void mte_enable_kernel(void)
{
	/* Enable MTE Sync Mode for EL1. */
	sysreg_clear_set(sctlr_el1, SCTLR_ELx_TCF_MASK, SCTLR_ELx_TCF_SYNC);
	isb();
}

static void update_sctlr_el1_tcf0(u64 tcf0)
{
	/* ISB required for the kernel uaccess routines */
	sysreg_clear_set(sctlr_el1, SCTLR_EL1_TCF0_MASK, tcf0);
	isb();
}

static void set_sctlr_el1_tcf0(u64 tcf0)
{
	/*
	 * mte_thread_switch() checks current->thread.sctlr_tcf0 as an
	 * optimisation. Disable preemption so that it does not see
	 * the variable update before the SCTLR_EL1.TCF0 one.
	 */
	preempt_disable();
	current->thread.sctlr_tcf0 = tcf0;
	update_sctlr_el1_tcf0(tcf0);
	preempt_enable();
}

static void update_gcr_el1_excl(u64 incl)
{
	u64 excl = ~incl & SYS_GCR_EL1_EXCL_MASK;

	/*
	 * Note that 'incl' is an include mask (controlled by the user via
	 * prctl()) while GCR_EL1 accepts an exclude mask.
	 * No need for ISB since this only affects EL0 currently, implicit
	 * with ERET.
	 */
	sysreg_clear_set_s(SYS_GCR_EL1, SYS_GCR_EL1_EXCL_MASK, excl);
}

static void set_gcr_el1_excl(u64 incl)
{
	current->thread.gcr_user_incl = incl;
	update_gcr_el1_excl(incl);
}

void flush_mte_state(void)
{
	if (!system_supports_mte())
		return;

	/* clear any pending asynchronous tag fault */
	dsb(ish);
	write_sysreg_s(0, SYS_TFSRE0_EL1);
	clear_thread_flag(TIF_MTE_ASYNC_FAULT);
	/* disable tag checking */
	set_sctlr_el1_tcf0(SCTLR_EL1_TCF0_NONE);
	/* reset tag generation mask */
	set_gcr_el1_excl(0);
}

void mte_thread_switch(struct task_struct *next)
{
	if (!system_supports_mte())
		return;

	/* avoid expensive SCTLR_EL1 accesses if no change */
	if (current->thread.sctlr_tcf0 != next->thread.sctlr_tcf0)
		update_sctlr_el1_tcf0(next->thread.sctlr_tcf0);
	update_gcr_el1_excl(next->thread.gcr_user_incl);
}

void mte_suspend_exit(void)
{
	if (!system_supports_mte())
		return;

	update_gcr_el1_excl(current->thread.gcr_user_incl);
}

long set_mte_ctrl(struct task_struct *task, unsigned long arg)
{
	u64 tcf0;
	u64 gcr_incl = (arg & PR_MTE_TAG_MASK) >> PR_MTE_TAG_SHIFT;

	if (!system_supports_mte())
		return 0;

	switch (arg & PR_MTE_TCF_MASK) {
	case PR_MTE_TCF_NONE:
		tcf0 = SCTLR_EL1_TCF0_NONE;
		break;
	case PR_MTE_TCF_SYNC:
		tcf0 = SCTLR_EL1_TCF0_SYNC;
		break;
	case PR_MTE_TCF_ASYNC:
		tcf0 = SCTLR_EL1_TCF0_ASYNC;
		break;
	default:
		return -EINVAL;
	}

	if (task != current) {
		task->thread.sctlr_tcf0 = tcf0;
		task->thread.gcr_user_incl = gcr_incl;
	} else {
		set_sctlr_el1_tcf0(tcf0);
		set_gcr_el1_excl(gcr_incl);
	}

	return 0;
}

long get_mte_ctrl(struct task_struct *task)
{
	unsigned long ret;

	if (!system_supports_mte())
		return 0;

	ret = task->thread.gcr_user_incl << PR_MTE_TAG_SHIFT;

	switch (task->thread.sctlr_tcf0) {
	case SCTLR_EL1_TCF0_NONE:
		ret |= PR_MTE_TCF_NONE;
		break;
	case SCTLR_EL1_TCF0_SYNC:
		ret |= PR_MTE_TCF_SYNC;
		break;
	case SCTLR_EL1_TCF0_ASYNC:
		ret |= PR_MTE_TCF_ASYNC;
		break;
	}

	return ret;
}

/*
 * Access MTE tags in another process' address space as given in mm. Update
 * the number of tags copied. Return 0 if any tags copied, error otherwise.
 * Inspired by __access_remote_vm().
 */
static int __access_remote_tags(struct mm_struct *mm, unsigned long addr,
				struct iovec *kiov, unsigned int gup_flags)
{
	struct vm_area_struct *vma;
	void __user *buf = kiov->iov_base;
	size_t len = kiov->iov_len;
	int ret;
	int write = gup_flags & FOLL_WRITE;

	if (!access_ok(buf, len))
		return -EFAULT;

	if (mmap_read_lock_killable(mm))
		return -EIO;

	while (len) {
		unsigned long tags, offset;
		void *maddr;
		struct page *page = NULL;

		ret = get_user_pages_remote(mm, addr, 1, gup_flags, &page,
					    &vma, NULL);
		if (ret <= 0)
			break;

		/*
		 * Only copy tags if the page has been mapped as PROT_MTE
		 * (PG_mte_tagged set). Otherwise the tags are not valid and
		 * not accessible to user. Moreover, an mprotect(PROT_MTE)
		 * would cause the existing tags to be cleared if the page
		 * was never mapped with PROT_MTE.
		 */
		if (!test_bit(PG_mte_tagged, &page->flags)) {
			ret = -EOPNOTSUPP;
			put_page(page);
			break;
		}

		/* limit access to the end of the page */
		offset = offset_in_page(addr);
		tags = min(len, (PAGE_SIZE - offset) / MTE_GRANULE_SIZE);

		maddr = page_address(page);
		if (write) {
			tags = mte_copy_tags_from_user(maddr + offset, buf, tags);
			set_page_dirty_lock(page);
		} else {
			tags = mte_copy_tags_to_user(buf, maddr + offset, tags);
		}
		put_page(page);

		/* error accessing the tracer's buffer */
		if (!tags)
			break;

		len -= tags;
		buf += tags;
		addr += tags * MTE_GRANULE_SIZE;
	}
	mmap_read_unlock(mm);

	/* return an error if no tags copied */
	kiov->iov_len = buf - kiov->iov_base;
	if (!kiov->iov_len) {
		/* check for error accessing the tracee's address space */
		if (ret <= 0)
			return -EIO;
		else
			return -EFAULT;
	}

	return 0;
}

/*
 * Copy MTE tags in another process' address space at 'addr' to/from tracer's
 * iovec buffer. Return 0 on success. Inspired by ptrace_access_vm().
 */
static int access_remote_tags(struct task_struct *tsk, unsigned long addr,
			      struct iovec *kiov, unsigned int gup_flags)
{
	struct mm_struct *mm;
	int ret;

	mm = get_task_mm(tsk);
	if (!mm)
		return -EPERM;

	if (!tsk->ptrace || (current != tsk->parent) ||
	    ((get_dumpable(mm) != SUID_DUMP_USER) &&
	     !ptracer_capable(tsk, mm->user_ns))) {
		mmput(mm);
		return -EPERM;
	}

	ret = __access_remote_tags(mm, addr, kiov, gup_flags);
	mmput(mm);

	return ret;
}

int mte_ptrace_copy_tags(struct task_struct *child, long request,
			 unsigned long addr, unsigned long data)
{
	int ret;
	struct iovec kiov;
	struct iovec __user *uiov = (void __user *)data;
	unsigned int gup_flags = FOLL_FORCE;

	if (!system_supports_mte())
		return -EIO;

	if (get_user(kiov.iov_base, &uiov->iov_base) ||
	    get_user(kiov.iov_len, &uiov->iov_len))
		return -EFAULT;

	if (request == PTRACE_POKEMTETAGS)
		gup_flags |= FOLL_WRITE;

	/* align addr to the MTE tag granule */
	addr &= MTE_GRANULE_MASK;

	ret = access_remote_tags(child, addr, &kiov, gup_flags);
	if (!ret)
		ret = put_user(kiov.iov_len, &uiov->iov_len);

	return ret;
}
Commit	Line	Data
637ec831 VF	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Copyright (C) 2020 ARM Ltd.
	4	*/
	5
34bfeea4	6	#include <linux/bitops.h>
18ddbaa0	7	#include <linux/kernel.h>
34bfeea4	8	#include <linux/mm.h>
1c101da8 CM	9	#include <linux/prctl.h>
1c101da8 CM	10	#include <linux/sched.h>
18ddbaa0	11	#include <linux/sched/mm.h>
4d1a8a2d	12	#include <linux/string.h>
36943aba SP	13	#include <linux/swap.h>
36943aba SP	14	#include <linux/swapops.h>
637ec831	15	#include <linux/thread_info.h>
85f49cae	16	#include <linux/types.h>
18ddbaa0	17	#include <linux/uio.h>
637ec831	18
85f49cae	19	#include <asm/barrier.h>
637ec831 VF	20	#include <asm/cpufeature.h>
637ec831 VF	21	#include <asm/mte.h>
85f49cae	22	#include <asm/mte-kasan.h>
18ddbaa0	23	#include <asm/ptrace.h>
637ec831 VF	24	#include <asm/sysreg.h>
637ec831 VF	25
36943aba SP	26	static void mte_sync_page_tags(struct page page, pte_t ptep, bool check_swap)
	27	{
	28	pte_t old_pte = READ_ONCE(*ptep);
	29
	30	if (check_swap && is_swap_pte(old_pte)) {
	31	swp_entry_t entry = pte_to_swp_entry(old_pte);
	32
	33	if (!non_swap_entry(entry) && mte_restore_tags(entry, page))
	34	return;
	35	}
	36
e5b8d921 VF	37	page_kasan_tag_reset(page);
	38	/*
	39	* We need smp_wmb() in between setting the flags and clearing the
	40	* tags because if another thread reads page->flags and builds a
	41	* tagged address out of it, there is an actual dependency to the
	42	* memory access, but on the current thread we do not guarantee that
	43	* the new page->flags are visible before the tags were updated.
	44	*/
	45	smp_wmb();
36943aba SP	46	mte_clear_page_tags(page_address(page));
	47	}
	48
34bfeea4 CM	49	void mte_sync_tags(pte_t *ptep, pte_t pte)
	50	{
	51	struct page *page = pte_page(pte);
	52	long i, nr_pages = compound_nr(page);
36943aba	53	bool check_swap = nr_pages == 1;
34bfeea4 CM	54
	55	/* if PG_mte_tagged is set, tags have already been initialised */
	56	for (i = 0; i < nr_pages; i++, page++) {
	57	if (!test_and_set_bit(PG_mte_tagged, &page->flags))
36943aba	58	mte_sync_page_tags(page, ptep, check_swap);
34bfeea4 CM	59	}
	60	}
	61
4d1a8a2d CM	62	int memcmp_pages(struct page page1, struct page page2)
	63	{
	64	char addr1, addr2;
	65	int ret;
	66
	67	addr1 = page_address(page1);
	68	addr2 = page_address(page2);
	69	ret = memcmp(addr1, addr2, PAGE_SIZE);
	70
	71	if (!system_supports_mte() \|\| ret)
	72	return ret;
	73
	74	/*
	75	* If the page content is identical but at least one of the pages is
	76	* tagged, return non-zero to avoid KSM merging. If only one of the
	77	* pages is tagged, set_pte_at() may zero or change the tags of the
	78	* other page via mte_sync_tags().
	79	*/
	80	if (test_bit(PG_mte_tagged, &page1->flags) \|\|
	81	test_bit(PG_mte_tagged, &page2->flags))
	82	return addr1 != addr2;
	83
	84	return ret;
	85	}
	86
85f49cae VF	87	u8 mte_get_mem_tag(void *addr)
	88	{
	89	if (!system_supports_mte())
	90	return 0xFF;
	91
	92	asm(__MTE_PREAMBLE "ldg %0, [%0]"
	93	: "+r" (addr));
	94
	95	return mte_get_ptr_tag(addr);
	96	}
	97
	98	u8 mte_get_random_tag(void)
	99	{
	100	void *addr;
	101
	102	if (!system_supports_mte())
	103	return 0xFF;
	104
	105	asm(__MTE_PREAMBLE "irg %0, %0"
	106	: "+r" (addr));
	107
	108	return mte_get_ptr_tag(addr);
	109	}
	110
	111	void mte_set_mem_tag_range(void addr, size_t size, u8 tag)
	112	{
	113	void *ptr = addr;
	114
	115	if ((!system_supports_mte()) \|\| (size == 0))
	116	return addr;
	117
	118	/* Make sure that size is MTE granule aligned. */
	119	WARN_ON(size & (MTE_GRANULE_SIZE - 1));
	120
	121	/* Make sure that the address is MTE granule aligned. */
	122	WARN_ON((u64)addr & (MTE_GRANULE_SIZE - 1));
	123
	124	tag = 0xF0 \| tag;
	125	ptr = (void *)__tag_set(ptr, tag);
	126
	127	mte_assign_mem_tag_range(ptr, size);
	128
	129	return ptr;
	130	}
	131
bfc62c59 VF	132	void mte_enable_kernel(void)
	133	{
	134	/* Enable MTE Sync Mode for EL1. */
	135	sysreg_clear_set(sctlr_el1, SCTLR_ELx_TCF_MASK, SCTLR_ELx_TCF_SYNC);
	136	isb();
	137	}
	138
1c101da8 CM	139	static void update_sctlr_el1_tcf0(u64 tcf0)
	140	{
	141	/* ISB required for the kernel uaccess routines */
	142	sysreg_clear_set(sctlr_el1, SCTLR_EL1_TCF0_MASK, tcf0);
	143	isb();
	144	}
	145
	146	static void set_sctlr_el1_tcf0(u64 tcf0)
	147	{
	148	/*
	149	* mte_thread_switch() checks current->thread.sctlr_tcf0 as an
	150	* optimisation. Disable preemption so that it does not see
	151	* the variable update before the SCTLR_EL1.TCF0 one.
	152	*/
	153	preempt_disable();
	154	current->thread.sctlr_tcf0 = tcf0;
	155	update_sctlr_el1_tcf0(tcf0);
	156	preempt_enable();
	157	}
	158
af5ce952 CM	159	static void update_gcr_el1_excl(u64 incl)
	160	{
	161	u64 excl = ~incl & SYS_GCR_EL1_EXCL_MASK;
	162
	163	/*
	164	* Note that 'incl' is an include mask (controlled by the user via
	165	* prctl()) while GCR_EL1 accepts an exclude mask.
	166	* No need for ISB since this only affects EL0 currently, implicit
	167	* with ERET.
	168	*/
	169	sysreg_clear_set_s(SYS_GCR_EL1, SYS_GCR_EL1_EXCL_MASK, excl);
	170	}
	171
	172	static void set_gcr_el1_excl(u64 incl)
	173	{
	174	current->thread.gcr_user_incl = incl;
	175	update_gcr_el1_excl(incl);
	176	}
	177
637ec831 VF	178	void flush_mte_state(void)
	179	{
	180	if (!system_supports_mte())
	181	return;
	182
	183	/* clear any pending asynchronous tag fault */
	184	dsb(ish);
	185	write_sysreg_s(0, SYS_TFSRE0_EL1);
	186	clear_thread_flag(TIF_MTE_ASYNC_FAULT);
1c101da8 CM	187	/* disable tag checking */
1c101da8 CM	188	set_sctlr_el1_tcf0(SCTLR_EL1_TCF0_NONE);
af5ce952 CM	189	/* reset tag generation mask */
af5ce952 CM	190	set_gcr_el1_excl(0);
1c101da8 CM	191	}
	192
	193	void mte_thread_switch(struct task_struct *next)
	194	{
	195	if (!system_supports_mte())
	196	return;
	197
	198	/* avoid expensive SCTLR_EL1 accesses if no change */
	199	if (current->thread.sctlr_tcf0 != next->thread.sctlr_tcf0)
	200	update_sctlr_el1_tcf0(next->thread.sctlr_tcf0);
af5ce952	201	update_gcr_el1_excl(next->thread.gcr_user_incl);
1c101da8 CM	202	}
1c101da8 CM	203
39d08e83 CM	204	void mte_suspend_exit(void)
	205	{
	206	if (!system_supports_mte())
	207	return;
	208
	209	update_gcr_el1_excl(current->thread.gcr_user_incl);
	210	}
	211
93f067f6	212	long set_mte_ctrl(struct task_struct *task, unsigned long arg)
1c101da8 CM	213	{
1c101da8 CM	214	u64 tcf0;
93f067f6	215	u64 gcr_incl = (arg & PR_MTE_TAG_MASK) >> PR_MTE_TAG_SHIFT;
1c101da8 CM	216
	217	if (!system_supports_mte())
	218	return 0;
	219
	220	switch (arg & PR_MTE_TCF_MASK) {
	221	case PR_MTE_TCF_NONE:
	222	tcf0 = SCTLR_EL1_TCF0_NONE;
	223	break;
	224	case PR_MTE_TCF_SYNC:
	225	tcf0 = SCTLR_EL1_TCF0_SYNC;
	226	break;
	227	case PR_MTE_TCF_ASYNC:
	228	tcf0 = SCTLR_EL1_TCF0_ASYNC;
	229	break;
	230	default:
	231	return -EINVAL;
	232	}
	233
93f067f6 CM	234	if (task != current) {
	235	task->thread.sctlr_tcf0 = tcf0;
	236	task->thread.gcr_user_incl = gcr_incl;
	237	} else {
	238	set_sctlr_el1_tcf0(tcf0);
	239	set_gcr_el1_excl(gcr_incl);
	240	}
1c101da8 CM	241
	242	return 0;
	243	}
	244
93f067f6	245	long get_mte_ctrl(struct task_struct *task)
1c101da8	246	{
af5ce952 CM	247	unsigned long ret;
af5ce952 CM	248
1c101da8 CM	249	if (!system_supports_mte())
	250	return 0;
	251
93f067f6	252	ret = task->thread.gcr_user_incl << PR_MTE_TAG_SHIFT;
af5ce952	253
93f067f6	254	switch (task->thread.sctlr_tcf0) {
1c101da8	255	case SCTLR_EL1_TCF0_NONE:
929c1f33 PC	256	ret \|= PR_MTE_TCF_NONE;
929c1f33 PC	257	break;
1c101da8	258	case SCTLR_EL1_TCF0_SYNC:
af5ce952 CM	259	ret \|= PR_MTE_TCF_SYNC;
af5ce952 CM	260	break;
1c101da8	261	case SCTLR_EL1_TCF0_ASYNC:
af5ce952 CM	262	ret \|= PR_MTE_TCF_ASYNC;
af5ce952 CM	263	break;
1c101da8 CM	264	}
1c101da8 CM	265
af5ce952	266	return ret;
637ec831	267	}
18ddbaa0 CM	268
	269	/*
	270	* Access MTE tags in another process' address space as given in mm. Update
	271	* the number of tags copied. Return 0 if any tags copied, error otherwise.
	272	* Inspired by __access_remote_vm().
	273	*/
	274	static int __access_remote_tags(struct mm_struct *mm, unsigned long addr,
	275	struct iovec *kiov, unsigned int gup_flags)
	276	{
	277	struct vm_area_struct *vma;
	278	void __user *buf = kiov->iov_base;
	279	size_t len = kiov->iov_len;
	280	int ret;
	281	int write = gup_flags & FOLL_WRITE;
	282
	283	if (!access_ok(buf, len))
	284	return -EFAULT;
	285
	286	if (mmap_read_lock_killable(mm))
	287	return -EIO;
	288
	289	while (len) {
	290	unsigned long tags, offset;
	291	void *maddr;
	292	struct page *page = NULL;
	293
	294	ret = get_user_pages_remote(mm, addr, 1, gup_flags, &page,
	295	&vma, NULL);
	296	if (ret <= 0)
	297	break;
	298
	299	/*
	300	* Only copy tags if the page has been mapped as PROT_MTE
	301	* (PG_mte_tagged set). Otherwise the tags are not valid and
	302	* not accessible to user. Moreover, an mprotect(PROT_MTE)
	303	* would cause the existing tags to be cleared if the page
	304	* was never mapped with PROT_MTE.
	305	*/
	306	if (!test_bit(PG_mte_tagged, &page->flags)) {
	307	ret = -EOPNOTSUPP;
	308	put_page(page);
	309	break;
	310	}
	311
	312	/* limit access to the end of the page */
	313	offset = offset_in_page(addr);
	314	tags = min(len, (PAGE_SIZE - offset) / MTE_GRANULE_SIZE);
	315
	316	maddr = page_address(page);
	317	if (write) {
	318	tags = mte_copy_tags_from_user(maddr + offset, buf, tags);
	319	set_page_dirty_lock(page);
	320	} else {
	321	tags = mte_copy_tags_to_user(buf, maddr + offset, tags);
	322	}
	323	put_page(page);
	324
	325	/* error accessing the tracer's buffer */
	326	if (!tags)
	327	break;
	328
	329	len -= tags;
	330	buf += tags;
	331	addr += tags * MTE_GRANULE_SIZE;
332	}
333	mmap_read_unlock(mm);
334
335	/* return an error if no tags copied */
336	kiov->iov_len = buf - kiov->iov_base;
337	if (!kiov->iov_len) {
338	/* check for error accessing the tracee's address space */
339	if (ret <= 0)
340	return -EIO;
341	else
342	return -EFAULT;
343	}
344
345	return 0;
346	}
347
348	/*
349	* Copy MTE tags in another process' address space at 'addr' to/from tracer's
350	* iovec buffer. Return 0 on success. Inspired by ptrace_access_vm().
351	*/
352	static int access_remote_tags(struct task_struct *tsk, unsigned long addr,
353	struct iovec *kiov, unsigned int gup_flags)
354	{
355	struct mm_struct *mm;
356	int ret;
357
358	mm = get_task_mm(tsk);
359	if (!mm)
360	return -EPERM;
361
362	if (!tsk->ptrace \|\| (current != tsk->parent) \|\|
363	((get_dumpable(mm) != SUID_DUMP_USER) &&
364	!ptracer_capable(tsk, mm->user_ns))) {
365	mmput(mm);
366	return -EPERM;
367	}
368
369	ret = __access_remote_tags(mm, addr, kiov, gup_flags);
370	mmput(mm);
371
372	return ret;
373	}
374
375	int mte_ptrace_copy_tags(struct task_struct *child, long request,
376	unsigned long addr, unsigned long data)
377	{
378	int ret;
379	struct iovec kiov;
380	struct iovec __user uiov = (void __user )data;
381	unsigned int gup_flags = FOLL_FORCE;
382
383	if (!system_supports_mte())
384	return -EIO;
385
386	if (get_user(kiov.iov_base, &uiov->iov_base) \|\|
387	get_user(kiov.iov_len, &uiov->iov_len))
388	return -EFAULT;
389
390	if (request == PTRACE_POKEMTETAGS)
391	gup_flags \|= FOLL_WRITE;
392
393	/* align addr to the MTE tag granule */
394	addr &= MTE_GRANULE_MASK;
395
396	ret = access_remote_tags(child, addr, &kiov, gup_flags);
397	if (!ret)
398	ret = put_user(kiov.iov_len, &uiov->iov_len);
399
400	return ret;
401	}