[mirror_qemu.git] / target / riscv / op_helper.c

/*
 * RISC-V Emulation Helpers for QEMU.
 *
 * Copyright (c) 2016-2017 Sagar Karandikar, sagark@eecs.berkeley.edu
 * Copyright (c) 2017-2018 SiFive, Inc.
 * Copyright (c) 2022      VRULL GmbH
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2 or later, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "qemu/osdep.h"
#include "cpu.h"
#include "qemu/main-loop.h"
#include "exec/exec-all.h"
#include "exec/helper-proto.h"

/* Exceptions processing helpers */
G_NORETURN void riscv_raise_exception(CPURISCVState *env,
                                      uint32_t exception, uintptr_t pc)
{
    CPUState *cs = env_cpu(env);
    cs->exception_index = exception;
    cpu_loop_exit_restore(cs, pc);
}

void helper_raise_exception(CPURISCVState *env, uint32_t exception)
{
    riscv_raise_exception(env, exception, 0);
}

target_ulong helper_csrr(CPURISCVState *env, int csr)
{
    /*
     * The seed CSR must be accessed with a read-write instruction. A
     * read-only instruction such as CSRRS/CSRRC with rs1=x0 or CSRRSI/
     * CSRRCI with uimm=0 will raise an illegal instruction exception.
     */
    if (csr == CSR_SEED) {
        riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());
    }

    target_ulong val = 0;
    RISCVException ret = riscv_csrrw(env, csr, &val, 0, 0);

    if (ret != RISCV_EXCP_NONE) {
        riscv_raise_exception(env, ret, GETPC());
    }
    return val;
}

void helper_csrw(CPURISCVState *env, int csr, target_ulong src)
{
    target_ulong mask = env->xl == MXL_RV32 ? UINT32_MAX : (target_ulong)-1;
    RISCVException ret = riscv_csrrw(env, csr, NULL, src, mask);

    if (ret != RISCV_EXCP_NONE) {
        riscv_raise_exception(env, ret, GETPC());
    }
}

target_ulong helper_csrrw(CPURISCVState *env, int csr,
                          target_ulong src, target_ulong write_mask)
{
    target_ulong val = 0;
    RISCVException ret = riscv_csrrw(env, csr, &val, src, write_mask);

    if (ret != RISCV_EXCP_NONE) {
        riscv_raise_exception(env, ret, GETPC());
    }
    return val;
}

target_ulong helper_csrr_i128(CPURISCVState *env, int csr)
{
    Int128 rv = int128_zero();
    RISCVException ret = riscv_csrrw_i128(env, csr, &rv,
                                          int128_zero(),
                                          int128_zero());

    if (ret != RISCV_EXCP_NONE) {
        riscv_raise_exception(env, ret, GETPC());
    }

    env->retxh = int128_gethi(rv);
    return int128_getlo(rv);
}

void helper_csrw_i128(CPURISCVState *env, int csr,
                      target_ulong srcl, target_ulong srch)
{
    RISCVException ret = riscv_csrrw_i128(env, csr, NULL,
                                          int128_make128(srcl, srch),
                                          UINT128_MAX);

    if (ret != RISCV_EXCP_NONE) {
        riscv_raise_exception(env, ret, GETPC());
    }
}

target_ulong helper_csrrw_i128(CPURISCVState *env, int csr,
                       target_ulong srcl, target_ulong srch,
                       target_ulong maskl, target_ulong maskh)
{
    Int128 rv = int128_zero();
    RISCVException ret = riscv_csrrw_i128(env, csr, &rv,
                                          int128_make128(srcl, srch),
                                          int128_make128(maskl, maskh));

    if (ret != RISCV_EXCP_NONE) {
        riscv_raise_exception(env, ret, GETPC());
    }

    env->retxh = int128_gethi(rv);
    return int128_getlo(rv);
}


/*
 * check_zicbo_envcfg
 *
 * Raise virtual exceptions and illegal instruction exceptions for
 * Zicbo[mz] instructions based on the settings of [mhs]envcfg as
 * specified in section 2.5.1 of the CMO specification.
 */
static void check_zicbo_envcfg(CPURISCVState *env, target_ulong envbits,
                                uintptr_t ra)
{
#ifndef CONFIG_USER_ONLY
    if ((env->priv < PRV_M) && !get_field(env->menvcfg, envbits)) {
        riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, ra);
    }

    if (riscv_cpu_virt_enabled(env) &&
        (((env->priv < PRV_H) && !get_field(env->henvcfg, envbits)) ||
         ((env->priv < PRV_S) && !get_field(env->senvcfg, envbits)))) {
        riscv_raise_exception(env, RISCV_EXCP_VIRT_INSTRUCTION_FAULT, ra);
    }

    if ((env->priv < PRV_S) && !get_field(env->senvcfg, envbits)) {
        riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, ra);
    }
#endif
}

void helper_cbo_zero(CPURISCVState *env, target_ulong address)
{
    RISCVCPU *cpu = env_archcpu(env);
    uint16_t cbozlen = cpu->cfg.cboz_blocksize;
    int mmu_idx = cpu_mmu_index(env, false);
    uintptr_t ra = GETPC();
    void *mem;

    check_zicbo_envcfg(env, MENVCFG_CBZE, ra);

    /* Mask off low-bits to align-down to the cache-block. */
    address &= ~(cbozlen - 1);

    /*
     * cbo.zero requires MMU_DATA_STORE access. Do a probe_write()
     * to raise any exceptions, including PMP.
     */
    mem = probe_write(env, address, cbozlen, mmu_idx, ra);

    if (likely(mem)) {
        memset(mem, 0, cbozlen);
    } else {
        /*
         * This means that we're dealing with an I/O page. Section 4.2
         * of cmobase v1.0.1 says:
         *
         * "Cache-block zero instructions store zeros independently
         * of whether data from the underlying memory locations are
         * cacheable."
         *
         * Write zeros in address + cbozlen regardless of not being
         * a RAM page.
         */
        for (int i = 0; i < cbozlen; i++) {
            cpu_stb_mmuidx_ra(env, address + i, 0, mmu_idx, ra);
        }
    }
}

/*
 * check_zicbom_access
 *
 * Check access permissions (LOAD, STORE or FETCH as specified in
 * section 2.5.2 of the CMO specification) for Zicbom, raising
 * either store page-fault (non-virtualized) or store guest-page
 * fault (virtualized).
 */
static void check_zicbom_access(CPURISCVState *env,
                                target_ulong address,
                                uintptr_t ra)
{
    RISCVCPU *cpu = env_archcpu(env);
    int mmu_idx = cpu_mmu_index(env, false);
    uint16_t cbomlen = cpu->cfg.cbom_blocksize;
    void *phost;
    int ret;

    /* Mask off low-bits to align-down to the cache-block. */
    address &= ~(cbomlen - 1);

    /*
     * Section 2.5.2 of cmobase v1.0.1:
     *
     * "A cache-block management instruction is permitted to
     * access the specified cache block whenever a load instruction
     * or store instruction is permitted to access the corresponding
     * physical addresses. If neither a load instruction nor store
     * instruction is permitted to access the physical addresses,
     * but an instruction fetch is permitted to access the physical
     * addresses, whether a cache-block management instruction is
     * permitted to access the cache block is UNSPECIFIED."
     */
    ret = probe_access_flags(env, address, cbomlen, MMU_DATA_LOAD,
                             mmu_idx, true, &phost, ra);
    if (ret != TLB_INVALID_MASK) {
        /* Success: readable */
        return;
    }

    /*
     * Since not readable, must be writable. On failure, store
     * fault/store guest amo fault will be raised by
     * riscv_cpu_tlb_fill(). PMP exceptions will be caught
     * there as well.
     */
    probe_write(env, address, cbomlen, mmu_idx, ra);
}

void helper_cbo_clean_flush(CPURISCVState *env, target_ulong address)
{
    uintptr_t ra = GETPC();
    check_zicbo_envcfg(env, MENVCFG_CBCFE, ra);
    check_zicbom_access(env, address, ra);

    /* We don't emulate the cache-hierarchy, so we're done. */
}

void helper_cbo_inval(CPURISCVState *env, target_ulong address)
{
    uintptr_t ra = GETPC();
    check_zicbo_envcfg(env, MENVCFG_CBIE, ra);
    check_zicbom_access(env, address, ra);

    /* We don't emulate the cache-hierarchy, so we're done. */
}

#ifndef CONFIG_USER_ONLY

target_ulong helper_sret(CPURISCVState *env)
{
    uint64_t mstatus;
    target_ulong prev_priv, prev_virt;

    if (!(env->priv >= PRV_S)) {
        riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());
    }

    target_ulong retpc = env->sepc;
    if (!riscv_has_ext(env, RVC) && (retpc & 0x3)) {
        riscv_raise_exception(env, RISCV_EXCP_INST_ADDR_MIS, GETPC());
    }

    if (get_field(env->mstatus, MSTATUS_TSR) && !(env->priv >= PRV_M)) {
        riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());
    }

    if (riscv_has_ext(env, RVH) && riscv_cpu_virt_enabled(env) &&
        get_field(env->hstatus, HSTATUS_VTSR)) {
        riscv_raise_exception(env, RISCV_EXCP_VIRT_INSTRUCTION_FAULT, GETPC());
    }

    mstatus = env->mstatus;
    prev_priv = get_field(mstatus, MSTATUS_SPP);
    mstatus = set_field(mstatus, MSTATUS_SIE,
                        get_field(mstatus, MSTATUS_SPIE));
    mstatus = set_field(mstatus, MSTATUS_SPIE, 1);
    mstatus = set_field(mstatus, MSTATUS_SPP, PRV_U);
    if (env->priv_ver >= PRIV_VERSION_1_12_0) {
        mstatus = set_field(mstatus, MSTATUS_MPRV, 0);
    }
    env->mstatus = mstatus;

    if (riscv_has_ext(env, RVH) && !riscv_cpu_virt_enabled(env)) {
        /* We support Hypervisor extensions and virtulisation is disabled */
        target_ulong hstatus = env->hstatus;

        prev_virt = get_field(hstatus, HSTATUS_SPV);

        hstatus = set_field(hstatus, HSTATUS_SPV, 0);

        env->hstatus = hstatus;

        if (prev_virt) {
            riscv_cpu_swap_hypervisor_regs(env);
        }

        riscv_cpu_set_virt_enabled(env, prev_virt);
    }

    riscv_cpu_set_mode(env, prev_priv);

    return retpc;
}

target_ulong helper_mret(CPURISCVState *env)
{
    if (!(env->priv >= PRV_M)) {
        riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());
    }

    target_ulong retpc = env->mepc;
    if (!riscv_has_ext(env, RVC) && (retpc & 0x3)) {
        riscv_raise_exception(env, RISCV_EXCP_INST_ADDR_MIS, GETPC());
    }

    uint64_t mstatus = env->mstatus;
    target_ulong prev_priv = get_field(mstatus, MSTATUS_MPP);

    if (riscv_cpu_cfg(env)->pmp &&
        !pmp_get_num_rules(env) && (prev_priv != PRV_M)) {
        riscv_raise_exception(env, RISCV_EXCP_INST_ACCESS_FAULT, GETPC());
    }

    target_ulong prev_virt = get_field(env->mstatus, MSTATUS_MPV);
    mstatus = set_field(mstatus, MSTATUS_MIE,
                        get_field(mstatus, MSTATUS_MPIE));
    mstatus = set_field(mstatus, MSTATUS_MPIE, 1);
    mstatus = set_field(mstatus, MSTATUS_MPP, PRV_U);
    mstatus = set_field(mstatus, MSTATUS_MPV, 0);
    if ((env->priv_ver >= PRIV_VERSION_1_12_0) && (prev_priv != PRV_M)) {
        mstatus = set_field(mstatus, MSTATUS_MPRV, 0);
    }
    env->mstatus = mstatus;
    riscv_cpu_set_mode(env, prev_priv);

    if (riscv_has_ext(env, RVH)) {
        if (prev_virt) {
            riscv_cpu_swap_hypervisor_regs(env);
        }

        riscv_cpu_set_virt_enabled(env, prev_virt);
    }

    return retpc;
}

void helper_wfi(CPURISCVState *env)
{
    CPUState *cs = env_cpu(env);
    bool rvs = riscv_has_ext(env, RVS);
    bool prv_u = env->priv == PRV_U;
    bool prv_s = env->priv == PRV_S;

    if (((prv_s || (!rvs && prv_u)) && get_field(env->mstatus, MSTATUS_TW)) ||
        (rvs && prv_u && !riscv_cpu_virt_enabled(env))) {
        riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());
    } else if (riscv_cpu_virt_enabled(env) && (prv_u ||
        (prv_s && get_field(env->hstatus, HSTATUS_VTW)))) {
        riscv_raise_exception(env, RISCV_EXCP_VIRT_INSTRUCTION_FAULT, GETPC());
    } else {
        cs->halted = 1;
        cs->exception_index = EXCP_HLT;
        cpu_loop_exit(cs);
    }
}

void helper_tlb_flush(CPURISCVState *env)
{
    CPUState *cs = env_cpu(env);
    if (!(env->priv >= PRV_S) ||
        (env->priv == PRV_S &&
         get_field(env->mstatus, MSTATUS_TVM))) {
        riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());
    } else if (riscv_has_ext(env, RVH) && riscv_cpu_virt_enabled(env) &&
               get_field(env->hstatus, HSTATUS_VTVM)) {
        riscv_raise_exception(env, RISCV_EXCP_VIRT_INSTRUCTION_FAULT, GETPC());
    } else {
        tlb_flush(cs);
    }
}

void helper_tlb_flush_all(CPURISCVState *env)
{
    CPUState *cs = env_cpu(env);
    tlb_flush_all_cpus_synced(cs);
}

void helper_hyp_tlb_flush(CPURISCVState *env)
{
    CPUState *cs = env_cpu(env);

    if (env->priv == PRV_S && riscv_cpu_virt_enabled(env)) {
        riscv_raise_exception(env, RISCV_EXCP_VIRT_INSTRUCTION_FAULT, GETPC());
    }

    if (env->priv == PRV_M ||
        (env->priv == PRV_S && !riscv_cpu_virt_enabled(env))) {
        tlb_flush(cs);
        return;
    }

    riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());
}

void helper_hyp_gvma_tlb_flush(CPURISCVState *env)
{
    if (env->priv == PRV_S && !riscv_cpu_virt_enabled(env) &&
        get_field(env->mstatus, MSTATUS_TVM)) {
        riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());
    }

    helper_hyp_tlb_flush(env);
}

target_ulong helper_hyp_hlvx_hu(CPURISCVState *env, target_ulong address)
{
    int mmu_idx = cpu_mmu_index(env, true) | TB_FLAGS_PRIV_HYP_ACCESS_MASK;

    return cpu_lduw_mmuidx_ra(env, address, mmu_idx, GETPC());
}

target_ulong helper_hyp_hlvx_wu(CPURISCVState *env, target_ulong address)
{
    int mmu_idx = cpu_mmu_index(env, true) | TB_FLAGS_PRIV_HYP_ACCESS_MASK;

    return cpu_ldl_mmuidx_ra(env, address, mmu_idx, GETPC());
}

#endif /* !CONFIG_USER_ONLY */
Commit	Line	Data
0c3e702a MC	1	/*
	2	* RISC-V Emulation Helpers for QEMU.
	3	*
	4	* Copyright (c) 2016-2017 Sagar Karandikar, sagark@eecs.berkeley.edu
	5	* Copyright (c) 2017-2018 SiFive, Inc.
a939c500	6	* Copyright (c) 2022 VRULL GmbH
0c3e702a MC	7	*
	8	* This program is free software; you can redistribute it and/or modify it
	9	* under the terms and conditions of the GNU General Public License,
	10	* version 2 or later, as published by the Free Software Foundation.
	11	*
	12	* This program is distributed in the hope it will be useful, but WITHOUT
	13	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	14	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
	15	* more details.
	16	*
	17	* You should have received a copy of the GNU General Public License along with
	18	* this program. If not, see <http://www.gnu.org/licenses/>.
	19	*/
	20
	21	#include "qemu/osdep.h"
0c3e702a MC	22	#include "cpu.h"
	23	#include "qemu/main-loop.h"
	24	#include "exec/exec-all.h"
	25	#include "exec/helper-proto.h"
	26
0c3e702a	27	/* Exceptions processing helpers */
8905770b MAL	28	G_NORETURN void riscv_raise_exception(CPURISCVState *env,
8905770b MAL	29	uint32_t exception, uintptr_t pc)
0c3e702a	30	{
3109cd98	31	CPUState *cs = env_cpu(env);
0c3e702a MC	32	cs->exception_index = exception;
	33	cpu_loop_exit_restore(cs, pc);
	34	}
	35
	36	void helper_raise_exception(CPURISCVState *env, uint32_t exception)
	37	{
fb738839	38	riscv_raise_exception(env, exception, 0);
0c3e702a MC	39	}
0c3e702a MC	40
a974879b	41	target_ulong helper_csrr(CPURISCVState *env, int csr)
0c3e702a	42	{
77442380 WL	43	/*
	44	* The seed CSR must be accessed with a read-write instruction. A
	45	* read-only instruction such as CSRRS/CSRRC with rs1=x0 or CSRRSI/
	46	* CSRRCI with uimm=0 will raise an illegal instruction exception.
	47	*/
	48	if (csr == CSR_SEED) {
	49	riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());
	50	}
	51
c7b95171	52	target_ulong val = 0;
a974879b	53	RISCVException ret = riscv_csrrw(env, csr, &val, 0, 0);
57cb2083	54
533c91e8 AF	55	if (ret != RISCV_EXCP_NONE) {
533c91e8 AF	56	riscv_raise_exception(env, ret, GETPC());
c7b95171 MC	57	}
c7b95171 MC	58	return val;
0c3e702a MC	59	}
0c3e702a MC	60
a974879b	61	void helper_csrw(CPURISCVState *env, int csr, target_ulong src)
0c3e702a	62	{
83b519b8 LZ	63	target_ulong mask = env->xl == MXL_RV32 ? UINT32_MAX : (target_ulong)-1;
83b519b8 LZ	64	RISCVException ret = riscv_csrrw(env, csr, NULL, src, mask);
57cb2083	65
533c91e8 AF	66	if (ret != RISCV_EXCP_NONE) {
533c91e8 AF	67	riscv_raise_exception(env, ret, GETPC());
0c3e702a	68	}
0c3e702a MC	69	}
0c3e702a MC	70
a974879b RH	71	target_ulong helper_csrrw(CPURISCVState *env, int csr,
a974879b RH	72	target_ulong src, target_ulong write_mask)
0c3e702a	73	{
c7b95171	74	target_ulong val = 0;
a974879b	75	RISCVException ret = riscv_csrrw(env, csr, &val, src, write_mask);
57cb2083	76
533c91e8 AF	77	if (ret != RISCV_EXCP_NONE) {
533c91e8 AF	78	riscv_raise_exception(env, ret, GETPC());
0c3e702a	79	}
c7b95171	80	return val;
0c3e702a MC	81	}
0c3e702a MC	82
961738ff FP	83	target_ulong helper_csrr_i128(CPURISCVState *env, int csr)
	84	{
	85	Int128 rv = int128_zero();
	86	RISCVException ret = riscv_csrrw_i128(env, csr, &rv,
	87	int128_zero(),
	88	int128_zero());
	89
	90	if (ret != RISCV_EXCP_NONE) {
	91	riscv_raise_exception(env, ret, GETPC());
	92	}
	93
	94	env->retxh = int128_gethi(rv);
	95	return int128_getlo(rv);
	96	}
	97
	98	void helper_csrw_i128(CPURISCVState *env, int csr,
	99	target_ulong srcl, target_ulong srch)
	100	{
	101	RISCVException ret = riscv_csrrw_i128(env, csr, NULL,
	102	int128_make128(srcl, srch),
	103	UINT128_MAX);
	104
	105	if (ret != RISCV_EXCP_NONE) {
	106	riscv_raise_exception(env, ret, GETPC());
	107	}
	108	}
	109
	110	target_ulong helper_csrrw_i128(CPURISCVState *env, int csr,
	111	target_ulong srcl, target_ulong srch,
	112	target_ulong maskl, target_ulong maskh)
	113	{
	114	Int128 rv = int128_zero();
	115	RISCVException ret = riscv_csrrw_i128(env, csr, &rv,
	116	int128_make128(srcl, srch),
	117	int128_make128(maskl, maskh));
	118
	119	if (ret != RISCV_EXCP_NONE) {
	120	riscv_raise_exception(env, ret, GETPC());
	121	}
	122
	123	env->retxh = int128_gethi(rv);
	124	return int128_getlo(rv);
	125	}
	126
a939c500 CM	127
	128	/*
	129	* check_zicbo_envcfg
	130	*
	131	* Raise virtual exceptions and illegal instruction exceptions for
	132	* Zicbo[mz] instructions based on the settings of [mhs]envcfg as
	133	* specified in section 2.5.1 of the CMO specification.
	134	*/
	135	static void check_zicbo_envcfg(CPURISCVState *env, target_ulong envbits,
	136	uintptr_t ra)
	137	{
	138	#ifndef CONFIG_USER_ONLY
	139	if ((env->priv < PRV_M) && !get_field(env->menvcfg, envbits)) {
	140	riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, ra);
	141	}
	142
	143	if (riscv_cpu_virt_enabled(env) &&
	144	(((env->priv < PRV_H) && !get_field(env->henvcfg, envbits)) \|\|
	145	((env->priv < PRV_S) && !get_field(env->senvcfg, envbits)))) {
	146	riscv_raise_exception(env, RISCV_EXCP_VIRT_INSTRUCTION_FAULT, ra);
	147	}
	148
	149	if ((env->priv < PRV_S) && !get_field(env->senvcfg, envbits)) {
	150	riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, ra);
	151	}
	152	#endif
	153	}
	154
	155	void helper_cbo_zero(CPURISCVState *env, target_ulong address)
	156	{
	157	RISCVCPU *cpu = env_archcpu(env);
	158	uint16_t cbozlen = cpu->cfg.cboz_blocksize;
	159	int mmu_idx = cpu_mmu_index(env, false);
	160	uintptr_t ra = GETPC();
	161	void *mem;
	162
	163	check_zicbo_envcfg(env, MENVCFG_CBZE, ra);
	164
	165	/* Mask off low-bits to align-down to the cache-block. */
	166	address &= ~(cbozlen - 1);
	167
	168	/*
	169	* cbo.zero requires MMU_DATA_STORE access. Do a probe_write()
	170	* to raise any exceptions, including PMP.
	171	*/
	172	mem = probe_write(env, address, cbozlen, mmu_idx, ra);
	173
	174	if (likely(mem)) {
	175	memset(mem, 0, cbozlen);
	176	} else {
	177	/*
	178	* This means that we're dealing with an I/O page. Section 4.2
	179	* of cmobase v1.0.1 says:
	180	*
	181	* "Cache-block zero instructions store zeros independently
	182	* of whether data from the underlying memory locations are
	183	* cacheable."
	184	*
	185	* Write zeros in address + cbozlen regardless of not being
	186	* a RAM page.
	187	*/
	188	for (int i = 0; i < cbozlen; i++) {
	189	cpu_stb_mmuidx_ra(env, address + i, 0, mmu_idx, ra);
	190	}
191	}
192	}
193
e05da09b CM	194	/*
	195	* check_zicbom_access
	196	*
	197	* Check access permissions (LOAD, STORE or FETCH as specified in
	198	* section 2.5.2 of the CMO specification) for Zicbom, raising
	199	* either store page-fault (non-virtualized) or store guest-page
	200	* fault (virtualized).
	201	*/
	202	static void check_zicbom_access(CPURISCVState *env,
	203	target_ulong address,
	204	uintptr_t ra)
	205	{
	206	RISCVCPU *cpu = env_archcpu(env);
	207	int mmu_idx = cpu_mmu_index(env, false);
	208	uint16_t cbomlen = cpu->cfg.cbom_blocksize;
	209	void *phost;
	210	int ret;
	211
	212	/* Mask off low-bits to align-down to the cache-block. */
	213	address &= ~(cbomlen - 1);
	214
	215	/*
	216	* Section 2.5.2 of cmobase v1.0.1:
	217	*
	218	* "A cache-block management instruction is permitted to
	219	* access the specified cache block whenever a load instruction
	220	* or store instruction is permitted to access the corresponding
	221	* physical addresses. If neither a load instruction nor store
	222	* instruction is permitted to access the physical addresses,
	223	* but an instruction fetch is permitted to access the physical
	224	* addresses, whether a cache-block management instruction is
	225	* permitted to access the cache block is UNSPECIFIED."
	226	*/
	227	ret = probe_access_flags(env, address, cbomlen, MMU_DATA_LOAD,
	228	mmu_idx, true, &phost, ra);
	229	if (ret != TLB_INVALID_MASK) {
	230	/* Success: readable */
	231	return;
	232	}
	233
	234	/*
	235	* Since not readable, must be writable. On failure, store
	236	* fault/store guest amo fault will be raised by
	237	* riscv_cpu_tlb_fill(). PMP exceptions will be caught
	238	* there as well.
	239	*/
	240	probe_write(env, address, cbomlen, mmu_idx, ra);
	241	}
	242
	243	void helper_cbo_clean_flush(CPURISCVState *env, target_ulong address)
	244	{
	245	uintptr_t ra = GETPC();
	246	check_zicbo_envcfg(env, MENVCFG_CBCFE, ra);
	247	check_zicbom_access(env, address, ra);
	248
	249	/* We don't emulate the cache-hierarchy, so we're done. */
	250	}
	251
	252	void helper_cbo_inval(CPURISCVState *env, target_ulong address)
	253	{
	254	uintptr_t ra = GETPC();
	255	check_zicbo_envcfg(env, MENVCFG_CBIE, ra);
	256	check_zicbom_access(env, address, ra);
	257
258	/* We don't emulate the cache-hierarchy, so we're done. */
259	}
260
0c3e702a MC	261	#ifndef CONFIG_USER_ONLY
0c3e702a MC	262
b655dc7c	263	target_ulong helper_sret(CPURISCVState *env)
0c3e702a	264	{
284d697c YJ	265	uint64_t mstatus;
284d697c YJ	266	target_ulong prev_priv, prev_virt;
e3fba4ba	267
0c3e702a	268	if (!(env->priv >= PRV_S)) {
fb738839	269	riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());
0c3e702a MC	270	}
	271
	272	target_ulong retpc = env->sepc;
	273	if (!riscv_has_ext(env, RVC) && (retpc & 0x3)) {
fb738839	274	riscv_raise_exception(env, RISCV_EXCP_INST_ADDR_MIS, GETPC());
0c3e702a MC	275	}
0c3e702a MC	276
1a9540d1	277	if (get_field(env->mstatus, MSTATUS_TSR) && !(env->priv >= PRV_M)) {
fb738839	278	riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());
7f2b5ff1 MC	279	}
7f2b5ff1 MC	280
e39a8320 AF	281	if (riscv_has_ext(env, RVH) && riscv_cpu_virt_enabled(env) &&
	282	get_field(env->hstatus, HSTATUS_VTSR)) {
	283	riscv_raise_exception(env, RISCV_EXCP_VIRT_INSTRUCTION_FAULT, GETPC());
	284	}
	285
e3fba4ba	286	mstatus = env->mstatus;
2bfec53b BM	287	prev_priv = get_field(mstatus, MSTATUS_SPP);
	288	mstatus = set_field(mstatus, MSTATUS_SIE,
	289	get_field(mstatus, MSTATUS_SPIE));
	290	mstatus = set_field(mstatus, MSTATUS_SPIE, 1);
	291	mstatus = set_field(mstatus, MSTATUS_SPP, PRV_U);
0ff430a5 BM	292	if (env->priv_ver >= PRIV_VERSION_1_12_0) {
	293	mstatus = set_field(mstatus, MSTATUS_MPRV, 0);
	294	}
2bfec53b	295	env->mstatus = mstatus;
e3fba4ba AF	296
	297	if (riscv_has_ext(env, RVH) && !riscv_cpu_virt_enabled(env)) {
	298	/* We support Hypervisor extensions and virtulisation is disabled */
	299	target_ulong hstatus = env->hstatus;
	300
e3fba4ba AF	301	prev_virt = get_field(hstatus, HSTATUS_SPV);
e3fba4ba AF	302
f2d5850f	303	hstatus = set_field(hstatus, HSTATUS_SPV, 0);
e3fba4ba	304
e3fba4ba AF	305	env->hstatus = hstatus;
	306
	307	if (prev_virt) {
	308	riscv_cpu_swap_hypervisor_regs(env);
	309	}
	310
	311	riscv_cpu_set_virt_enabled(env, prev_virt);
e3fba4ba AF	312	}
e3fba4ba AF	313
fb738839	314	riscv_cpu_set_mode(env, prev_priv);
0c3e702a MC	315
	316	return retpc;
	317	}
	318
b655dc7c	319	target_ulong helper_mret(CPURISCVState *env)
0c3e702a MC	320	{
0c3e702a MC	321	if (!(env->priv >= PRV_M)) {
fb738839	322	riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());
0c3e702a MC	323	}
	324
	325	target_ulong retpc = env->mepc;
	326	if (!riscv_has_ext(env, RVC) && (retpc & 0x3)) {
fb738839	327	riscv_raise_exception(env, RISCV_EXCP_INST_ADDR_MIS, GETPC());
0c3e702a MC	328	}
0c3e702a MC	329
284d697c	330	uint64_t mstatus = env->mstatus;
0c3e702a	331	target_ulong prev_priv = get_field(mstatus, MSTATUS_MPP);
d102f19a	332
3fe40ef5	333	if (riscv_cpu_cfg(env)->pmp &&
0fbb5d2d	334	!pmp_get_num_rules(env) && (prev_priv != PRV_M)) {
4c48aad1	335	riscv_raise_exception(env, RISCV_EXCP_INST_ACCESS_FAULT, GETPC());
d102f19a AP	336	}
d102f19a AP	337
284d697c	338	target_ulong prev_virt = get_field(env->mstatus, MSTATUS_MPV);
1a9540d1 AF	339	mstatus = set_field(mstatus, MSTATUS_MIE,
1a9540d1 AF	340	get_field(mstatus, MSTATUS_MPIE));
a37f21c2	341	mstatus = set_field(mstatus, MSTATUS_MPIE, 1);
0c3e702a	342	mstatus = set_field(mstatus, MSTATUS_MPP, PRV_U);
e3fba4ba	343	mstatus = set_field(mstatus, MSTATUS_MPV, 0);
0ff430a5 BM	344	if ((env->priv_ver >= PRIV_VERSION_1_12_0) && (prev_priv != PRV_M)) {
	345	mstatus = set_field(mstatus, MSTATUS_MPRV, 0);
	346	}
c7b95171	347	env->mstatus = mstatus;
e3fba4ba AF	348	riscv_cpu_set_mode(env, prev_priv);
	349
	350	if (riscv_has_ext(env, RVH)) {
	351	if (prev_virt) {
	352	riscv_cpu_swap_hypervisor_regs(env);
	353	}
	354
	355	riscv_cpu_set_virt_enabled(env, prev_virt);
	356	}
0c3e702a MC	357
	358	return retpc;
	359	}
	360
0c3e702a MC	361	void helper_wfi(CPURISCVState *env)
0c3e702a MC	362	{
3109cd98	363	CPUState *cs = env_cpu(env);
719f0f60 JM	364	bool rvs = riscv_has_ext(env, RVS);
	365	bool prv_u = env->priv == PRV_U;
	366	bool prv_s = env->priv == PRV_S;
0c3e702a	367
719f0f60 JM	368	if (((prv_s \|\| (!rvs && prv_u)) && get_field(env->mstatus, MSTATUS_TW)) \|\|
	369	(rvs && prv_u && !riscv_cpu_virt_enabled(env))) {
	370	riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());
	371	} else if (riscv_cpu_virt_enabled(env) && (prv_u \|\|
	372	(prv_s && get_field(env->hstatus, HSTATUS_VTW)))) {
e39a8320	373	riscv_raise_exception(env, RISCV_EXCP_VIRT_INSTRUCTION_FAULT, GETPC());
7f2b5ff1 MC	374	} else {
	375	cs->halted = 1;
	376	cs->exception_index = EXCP_HLT;
	377	cpu_loop_exit(cs);
	378	}
0c3e702a MC	379	}
	380
	381	void helper_tlb_flush(CPURISCVState *env)
	382	{
3109cd98	383	CPUState *cs = env_cpu(env);
b86f4167 JB	384	if (!(env->priv >= PRV_S) \|\|
b86f4167 JB	385	(env->priv == PRV_S &&
b86f4167	386	get_field(env->mstatus, MSTATUS_TVM))) {
fb738839	387	riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());
e39a8320 AF	388	} else if (riscv_has_ext(env, RVH) && riscv_cpu_virt_enabled(env) &&
	389	get_field(env->hstatus, HSTATUS_VTVM)) {
	390	riscv_raise_exception(env, RISCV_EXCP_VIRT_INSTRUCTION_FAULT, GETPC());
7f2b5ff1 MC	391	} else {
	392	tlb_flush(cs);
	393	}
0c3e702a MC	394	}
0c3e702a MC	395
134c3ffa CM	396	void helper_tlb_flush_all(CPURISCVState *env)
	397	{
	398	CPUState *cs = env_cpu(env);
	399	tlb_flush_all_cpus_synced(cs);
	400	}
	401
2761db5f AF	402	void helper_hyp_tlb_flush(CPURISCVState *env)
	403	{
	404	CPUState *cs = env_cpu(env);
	405
e39a8320 AF	406	if (env->priv == PRV_S && riscv_cpu_virt_enabled(env)) {
	407	riscv_raise_exception(env, RISCV_EXCP_VIRT_INSTRUCTION_FAULT, GETPC());
	408	}
	409
2761db5f AF	410	if (env->priv == PRV_M \|\|
	411	(env->priv == PRV_S && !riscv_cpu_virt_enabled(env))) {
	412	tlb_flush(cs);
	413	return;
	414	}
	415
	416	riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());
	417	}
	418
e39a8320 AF	419	void helper_hyp_gvma_tlb_flush(CPURISCVState *env)
	420	{
	421	if (env->priv == PRV_S && !riscv_cpu_virt_enabled(env) &&
	422	get_field(env->mstatus, MSTATUS_TVM)) {
	423	riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());
	424	}
	425
	426	helper_hyp_tlb_flush(env);
	427	}
	428
7687537a	429	target_ulong helper_hyp_hlvx_hu(CPURISCVState *env, target_ulong address)
8c5362ac	430	{
7687537a	431	int mmu_idx = cpu_mmu_index(env, true) \| TB_FLAGS_PRIV_HYP_ACCESS_MASK;
8c5362ac	432
7687537a AF	433	return cpu_lduw_mmuidx_ra(env, address, mmu_idx, GETPC());
7687537a AF	434	}
8c5362ac	435
7687537a AF	436	target_ulong helper_hyp_hlvx_wu(CPURISCVState *env, target_ulong address)
	437	{
	438	int mmu_idx = cpu_mmu_index(env, true) \| TB_FLAGS_PRIV_HYP_ACCESS_MASK;
	439
	440	return cpu_ldl_mmuidx_ra(env, address, mmu_idx, GETPC());
8c5362ac AF	441	}
8c5362ac AF	442
0c3e702a	443	#endif /* !CONFIG_USER_ONLY */