target/riscv/cpu_helper.c

   1 /*
   2  * RISC-V CPU helpers for qemu.
   3  *
   4  * Copyright (c) 2016-2017 Sagar Karandikar, sagark@eecs.berkeley.edu
   5  * Copyright (c) 2017-2018 SiFive, Inc.
   6  *
   7  * This program is free software; you can redistribute it and/or modify it
   8  * under the terms and conditions of the GNU General Public License,
   9  * version 2 or later, as published by the Free Software Foundation.
  10  *
  11  * This program is distributed in the hope it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  14  * more details.
  15  *
  16  * You should have received a copy of the GNU General Public License along with
  17  * this program.  If not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 #include "qemu/osdep.h"
  21 #include "qemu/log.h"
  22 #include "qemu/main-loop.h"
  23 #include "cpu.h"
  24 #include "exec/exec-all.h"
  25 #include "tcg/tcg-op.h"
  26 #include "trace.h"
  27
  28 int riscv_cpu_mmu_index(CPURISCVState *env, bool ifetch)
  29 {
  30 #ifdef CONFIG_USER_ONLY
  31     return 0;
  32 #else
  33     return env->priv;
  34 #endif
  35 }
  36
  37 #ifndef CONFIG_USER_ONLY
  38 static int riscv_cpu_local_irq_pending(CPURISCVState *env)
  39 {
  40     target_ulong irqs;
  41
  42     target_ulong mstatus_mie = get_field(env->mstatus, MSTATUS_MIE);
  43     target_ulong mstatus_sie = get_field(env->mstatus, MSTATUS_SIE);
  44     target_ulong hs_mstatus_sie = get_field(env->mstatus_hs, MSTATUS_SIE);
  45
  46     target_ulong pending = env->mip & env->mie &
  47                                ~(MIP_VSSIP | MIP_VSTIP | MIP_VSEIP);
  48     target_ulong vspending = (env->mip & env->mie &
  49                               (MIP_VSSIP | MIP_VSTIP | MIP_VSEIP)) >> 1;
  50
  51     target_ulong mie    = env->priv < PRV_M ||
  52                           (env->priv == PRV_M && mstatus_mie);
  53     target_ulong sie    = env->priv < PRV_S ||
  54                           (env->priv == PRV_S && mstatus_sie);
  55     target_ulong hs_sie = env->priv < PRV_S ||
  56                           (env->priv == PRV_S && hs_mstatus_sie);
  57
  58     if (riscv_cpu_virt_enabled(env)) {
  59         target_ulong pending_hs_irq = pending & -hs_sie;
  60
  61         if (pending_hs_irq) {
  62             riscv_cpu_set_force_hs_excep(env, FORCE_HS_EXCEP);
  63             return ctz64(pending_hs_irq);
  64         }
  65
  66         pending = vspending;
  67     }
  68
  69     irqs = (pending & ~env->mideleg & -mie) | (pending &  env->mideleg & -sie);
  70
  71     if (irqs) {
  72         return ctz64(irqs); /* since non-zero */
  73     } else {
  74         return EXCP_NONE; /* indicates no pending interrupt */
  75     }
  76 }
  77 #endif
  78
  79 bool riscv_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
  80 {
  81 #if !defined(CONFIG_USER_ONLY)
  82     if (interrupt_request & CPU_INTERRUPT_HARD) {
  83         RISCVCPU *cpu = RISCV_CPU(cs);
  84         CPURISCVState *env = &cpu->env;
  85         int interruptno = riscv_cpu_local_irq_pending(env);
  86         if (interruptno >= 0) {
  87             cs->exception_index = RISCV_EXCP_INT_FLAG | interruptno;
  88             riscv_cpu_do_interrupt(cs);
  89             return true;
  90         }
  91     }
  92 #endif
  93     return false;
  94 }
  95
  96 #if !defined(CONFIG_USER_ONLY)
  97
  98 /* Return true is floating point support is currently enabled */
  99 bool riscv_cpu_fp_enabled(CPURISCVState *env)
 100 {
 101     if (env->mstatus & MSTATUS_FS) {
 102         return true;
 103     }
 104
 105     return false;
 106 }
 107
 108 void riscv_cpu_swap_hypervisor_regs(CPURISCVState *env)
 109 {
 110     target_ulong mstatus_mask = MSTATUS_MXR | MSTATUS_SUM | MSTATUS_FS |
 111                                 MSTATUS_SPP | MSTATUS_SPIE | MSTATUS_SIE;
 112     bool current_virt = riscv_cpu_virt_enabled(env);
 113
 114     g_assert(riscv_has_ext(env, RVH));
 115
 116 #if defined(TARGET_RISCV64)
 117     mstatus_mask |= MSTATUS64_UXL;
 118 #endif
 119
 120     if (current_virt) {
 121         /* Current V=1 and we are about to change to V=0 */
 122         env->vsstatus = env->mstatus & mstatus_mask;
 123         env->mstatus &= ~mstatus_mask;
 124         env->mstatus |= env->mstatus_hs;
 125
 126         env->vstvec = env->stvec;
 127         env->stvec = env->stvec_hs;
 128
 129         env->vsscratch = env->sscratch;
 130         env->sscratch = env->sscratch_hs;
 131
 132         env->vsepc = env->sepc;
 133         env->sepc = env->sepc_hs;
 134
 135         env->vscause = env->scause;
 136         env->scause = env->scause_hs;
 137
 138         env->vstval = env->sbadaddr;
 139         env->sbadaddr = env->stval_hs;
 140
 141         env->vsatp = env->satp;
 142         env->satp = env->satp_hs;
 143     } else {
 144         /* Current V=0 and we are about to change to V=1 */
 145         env->mstatus_hs = env->mstatus & mstatus_mask;
 146         env->mstatus &= ~mstatus_mask;
 147         env->mstatus |= env->vsstatus;
 148
 149         env->stvec_hs = env->stvec;
 150         env->stvec = env->vstvec;
 151
 152         env->sscratch_hs = env->sscratch;
 153         env->sscratch = env->vsscratch;
 154
 155         env->sepc_hs = env->sepc;
 156         env->sepc = env->vsepc;
 157
 158         env->scause_hs = env->scause;
 159         env->scause = env->vscause;
 160
 161         env->stval_hs = env->sbadaddr;
 162         env->sbadaddr = env->vstval;
 163
 164         env->satp_hs = env->satp;
 165         env->satp = env->vsatp;
 166     }
 167 }
 168
 169 bool riscv_cpu_virt_enabled(CPURISCVState *env)
 170 {
 171     if (!riscv_has_ext(env, RVH)) {
 172         return false;
 173     }
 174
 175     return get_field(env->virt, VIRT_ONOFF);
 176 }
 177
 178 void riscv_cpu_set_virt_enabled(CPURISCVState *env, bool enable)
 179 {
 180     if (!riscv_has_ext(env, RVH)) {
 181         return;
 182     }
 183
 184     /* Flush the TLB on all virt mode changes. */
 185     if (get_field(env->virt, VIRT_ONOFF) != enable) {
 186         tlb_flush(env_cpu(env));
 187     }
 188
 189     env->virt = set_field(env->virt, VIRT_ONOFF, enable);
 190 }
 191
 192 bool riscv_cpu_force_hs_excep_enabled(CPURISCVState *env)
 193 {
 194     if (!riscv_has_ext(env, RVH)) {
 195         return false;
 196     }
 197
 198     return get_field(env->virt, FORCE_HS_EXCEP);
 199 }
 200
 201 void riscv_cpu_set_force_hs_excep(CPURISCVState *env, bool enable)
 202 {
 203     if (!riscv_has_ext(env, RVH)) {
 204         return;
 205     }
 206
 207     env->virt = set_field(env->virt, FORCE_HS_EXCEP, enable);
 208 }
 209
 210 int riscv_cpu_claim_interrupts(RISCVCPU *cpu, uint32_t interrupts)
 211 {
 212     CPURISCVState *env = &cpu->env;
 213     if (env->miclaim & interrupts) {
 214         return -1;
 215     } else {
 216         env->miclaim |= interrupts;
 217         return 0;
 218     }
 219 }
 220
 221 uint32_t riscv_cpu_update_mip(RISCVCPU *cpu, uint32_t mask, uint32_t value)
 222 {
 223     CPURISCVState *env = &cpu->env;
 224     CPUState *cs = CPU(cpu);
 225     uint32_t old = env->mip;
 226     bool locked = false;
 227
 228     if (!qemu_mutex_iothread_locked()) {
 229         locked = true;
 230         qemu_mutex_lock_iothread();
 231     }
 232
 233     env->mip = (env->mip & ~mask) | (value & mask);
 234
 235     if (env->mip) {
 236         cpu_interrupt(cs, CPU_INTERRUPT_HARD);
 237     } else {
 238         cpu_reset_interrupt(cs, CPU_INTERRUPT_HARD);
 239     }
 240
 241     if (locked) {
 242         qemu_mutex_unlock_iothread();
 243     }
 244
 245     return old;
 246 }
 247
 248 void riscv_cpu_set_mode(CPURISCVState *env, target_ulong newpriv)
 249 {
 250     if (newpriv > PRV_M) {
 251         g_assert_not_reached();
 252     }
 253     if (newpriv == PRV_H) {
 254         newpriv = PRV_U;
 255     }
 256     /* tlb_flush is unnecessary as mode is contained in mmu_idx */
 257     env->priv = newpriv;
 258
 259     /*
 260      * Clear the load reservation - otherwise a reservation placed in one
 261      * context/process can be used by another, resulting in an SC succeeding
 262      * incorrectly. Version 2.2 of the ISA specification explicitly requires
 263      * this behaviour, while later revisions say that the kernel "should" use
 264      * an SC instruction to force the yielding of a load reservation on a
 265      * preemptive context switch. As a result, do both.
 266      */
 267     env->load_res = -1;
 268 }
 269
 270 /* get_physical_address - get the physical address for this virtual address
 271  *
 272  * Do a page table walk to obtain the physical address corresponding to a
 273  * virtual address. Returns 0 if the translation was successful
 274  *
 275  * Adapted from Spike's mmu_t::translate and mmu_t::walk
 276  *
 277  */
 278 static int get_physical_address(CPURISCVState *env, hwaddr *physical,
 279                                 int *prot, target_ulong addr,
 280                                 int access_type, int mmu_idx)
 281 {
 282     /* NOTE: the env->pc value visible here will not be
 283      * correct, but the value visible to the exception handler
 284      * (riscv_cpu_do_interrupt) is correct */
 285     MemTxResult res;
 286     MemTxAttrs attrs = MEMTXATTRS_UNSPECIFIED;
 287     int mode = mmu_idx;
 288
 289     if (mode == PRV_M && access_type != MMU_INST_FETCH) {
 290         if (get_field(env->mstatus, MSTATUS_MPRV)) {
 291             mode = get_field(env->mstatus, MSTATUS_MPP);
 292         }
 293     }
 294
 295     if (mode == PRV_M || !riscv_feature(env, RISCV_FEATURE_MMU)) {
 296         *physical = addr;
 297         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
 298         return TRANSLATE_SUCCESS;
 299     }
 300
 301     *prot = 0;
 302
 303     hwaddr base;
 304     int levels, ptidxbits, ptesize, vm, sum;
 305     int mxr = get_field(env->mstatus, MSTATUS_MXR);
 306
 307     if (env->priv_ver >= PRIV_VERSION_1_10_0) {
 308         base = (hwaddr)get_field(env->satp, SATP_PPN) << PGSHIFT;
 309         sum = get_field(env->mstatus, MSTATUS_SUM);
 310         vm = get_field(env->satp, SATP_MODE);
 311         switch (vm) {
 312         case VM_1_10_SV32:
 313           levels = 2; ptidxbits = 10; ptesize = 4; break;
 314         case VM_1_10_SV39:
 315           levels = 3; ptidxbits = 9; ptesize = 8; break;
 316         case VM_1_10_SV48:
 317           levels = 4; ptidxbits = 9; ptesize = 8; break;
 318         case VM_1_10_SV57:
 319           levels = 5; ptidxbits = 9; ptesize = 8; break;
 320         case VM_1_10_MBARE:
 321             *physical = addr;
 322             *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
 323             return TRANSLATE_SUCCESS;
 324         default:
 325           g_assert_not_reached();
 326         }
 327     } else {
 328         base = (hwaddr)(env->sptbr) << PGSHIFT;
 329         sum = !get_field(env->mstatus, MSTATUS_PUM);
 330         vm = get_field(env->mstatus, MSTATUS_VM);
 331         switch (vm) {
 332         case VM_1_09_SV32:
 333           levels = 2; ptidxbits = 10; ptesize = 4; break;
 334         case VM_1_09_SV39:
 335           levels = 3; ptidxbits = 9; ptesize = 8; break;
 336         case VM_1_09_SV48:
 337           levels = 4; ptidxbits = 9; ptesize = 8; break;
 338         case VM_1_09_MBARE:
 339             *physical = addr;
 340             *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
 341             return TRANSLATE_SUCCESS;
 342         default:
 343           g_assert_not_reached();
 344         }
 345     }
 346
 347     CPUState *cs = env_cpu(env);
 348     int va_bits = PGSHIFT + levels * ptidxbits;
 349     target_ulong mask = (1L << (TARGET_LONG_BITS - (va_bits - 1))) - 1;
 350     target_ulong masked_msbs = (addr >> (va_bits - 1)) & mask;
 351     if (masked_msbs != 0 && masked_msbs != mask) {
 352         return TRANSLATE_FAIL;
 353     }
 354
 355     int ptshift = (levels - 1) * ptidxbits;
 356     int i;
 357
 358 #if !TCG_OVERSIZED_GUEST
 359 restart:
 360 #endif
 361     for (i = 0; i < levels; i++, ptshift -= ptidxbits) {
 362         target_ulong idx = (addr >> (PGSHIFT + ptshift)) &
 363                            ((1 << ptidxbits) - 1);
 364
 365         /* check that physical address of PTE is legal */
 366         hwaddr pte_addr = base + idx * ptesize;
 367
 368         if (riscv_feature(env, RISCV_FEATURE_PMP) &&
 369             !pmp_hart_has_privs(env, pte_addr, sizeof(target_ulong),
 370             1 << MMU_DATA_LOAD, PRV_S)) {
 371             return TRANSLATE_PMP_FAIL;
 372         }
 373
 374 #if defined(TARGET_RISCV32)
 375         target_ulong pte = address_space_ldl(cs->as, pte_addr, attrs, &res);
 376 #elif defined(TARGET_RISCV64)
 377         target_ulong pte = address_space_ldq(cs->as, pte_addr, attrs, &res);
 378 #endif
 379         if (res != MEMTX_OK) {
 380             return TRANSLATE_FAIL;
 381         }
 382
 383         hwaddr ppn = pte >> PTE_PPN_SHIFT;
 384
 385         if (!(pte & PTE_V)) {
 386             /* Invalid PTE */
 387             return TRANSLATE_FAIL;
 388         } else if (!(pte & (PTE_R | PTE_W | PTE_X))) {
 389             /* Inner PTE, continue walking */
 390             base = ppn << PGSHIFT;
 391         } else if ((pte & (PTE_R | PTE_W | PTE_X)) == PTE_W) {
 392             /* Reserved leaf PTE flags: PTE_W */
 393             return TRANSLATE_FAIL;
 394         } else if ((pte & (PTE_R | PTE_W | PTE_X)) == (PTE_W | PTE_X)) {
 395             /* Reserved leaf PTE flags: PTE_W + PTE_X */
 396             return TRANSLATE_FAIL;
 397         } else if ((pte & PTE_U) && ((mode != PRV_U) &&
 398                    (!sum || access_type == MMU_INST_FETCH))) {
 399             /* User PTE flags when not U mode and mstatus.SUM is not set,
 400                or the access type is an instruction fetch */
 401             return TRANSLATE_FAIL;
 402         } else if (!(pte & PTE_U) && (mode != PRV_S)) {
 403             /* Supervisor PTE flags when not S mode */
 404             return TRANSLATE_FAIL;
 405         } else if (ppn & ((1ULL << ptshift) - 1)) {
 406             /* Misaligned PPN */
 407             return TRANSLATE_FAIL;
 408         } else if (access_type == MMU_DATA_LOAD && !((pte & PTE_R) ||
 409                    ((pte & PTE_X) && mxr))) {
 410             /* Read access check failed */
 411             return TRANSLATE_FAIL;
 412         } else if (access_type == MMU_DATA_STORE && !(pte & PTE_W)) {
 413             /* Write access check failed */
 414             return TRANSLATE_FAIL;
 415         } else if (access_type == MMU_INST_FETCH && !(pte & PTE_X)) {
 416             /* Fetch access check failed */
 417             return TRANSLATE_FAIL;
 418         } else {
 419             /* if necessary, set accessed and dirty bits. */
 420             target_ulong updated_pte = pte | PTE_A |
 421                 (access_type == MMU_DATA_STORE ? PTE_D : 0);
 422
 423             /* Page table updates need to be atomic with MTTCG enabled */
 424             if (updated_pte != pte) {
 425                 /*
 426                  * - if accessed or dirty bits need updating, and the PTE is
 427                  *   in RAM, then we do so atomically with a compare and swap.
 428                  * - if the PTE is in IO space or ROM, then it can't be updated
 429                  *   and we return TRANSLATE_FAIL.
 430                  * - if the PTE changed by the time we went to update it, then
 431                  *   it is no longer valid and we must re-walk the page table.
 432                  */
 433                 MemoryRegion *mr;
 434                 hwaddr l = sizeof(target_ulong), addr1;
 435                 mr = address_space_translate(cs->as, pte_addr,
 436                     &addr1, &l, false, MEMTXATTRS_UNSPECIFIED);
 437                 if (memory_region_is_ram(mr)) {
 438                     target_ulong *pte_pa =
 439                         qemu_map_ram_ptr(mr->ram_block, addr1);
 440 #if TCG_OVERSIZED_GUEST
 441                     /* MTTCG is not enabled on oversized TCG guests so
 442                      * page table updates do not need to be atomic */
 443                     *pte_pa = pte = updated_pte;
 444 #else
 445                     target_ulong old_pte =
 446                         atomic_cmpxchg(pte_pa, pte, updated_pte);
 447                     if (old_pte != pte) {
 448                         goto restart;
 449                     } else {
 450                         pte = updated_pte;
 451                     }
 452 #endif
 453                 } else {
 454                     /* misconfigured PTE in ROM (AD bits are not preset) or
 455                      * PTE is in IO space and can't be updated atomically */
 456                     return TRANSLATE_FAIL;
 457                 }
 458             }
 459
 460             /* for superpage mappings, make a fake leaf PTE for the TLB's
 461                benefit. */
 462             target_ulong vpn = addr >> PGSHIFT;
 463             *physical = (ppn | (vpn & ((1L << ptshift) - 1))) << PGSHIFT;
 464
 465             /* set permissions on the TLB entry */
 466             if ((pte & PTE_R) || ((pte & PTE_X) && mxr)) {
 467                 *prot |= PAGE_READ;
 468             }
 469             if ((pte & PTE_X)) {
 470                 *prot |= PAGE_EXEC;
 471             }
 472             /* add write permission on stores or if the page is already dirty,
 473                so that we TLB miss on later writes to update the dirty bit */
 474             if ((pte & PTE_W) &&
 475                     (access_type == MMU_DATA_STORE || (pte & PTE_D))) {
 476                 *prot |= PAGE_WRITE;
 477             }
 478             return TRANSLATE_SUCCESS;
 479         }
 480     }
 481     return TRANSLATE_FAIL;
 482 }
 483
 484 static void raise_mmu_exception(CPURISCVState *env, target_ulong address,
 485                                 MMUAccessType access_type, bool pmp_violation)
 486 {
 487     CPUState *cs = env_cpu(env);
 488     int page_fault_exceptions =
 489         (env->priv_ver >= PRIV_VERSION_1_10_0) &&
 490         get_field(env->satp, SATP_MODE) != VM_1_10_MBARE &&
 491         !pmp_violation;
 492     switch (access_type) {
 493     case MMU_INST_FETCH:
 494         cs->exception_index = page_fault_exceptions ?
 495             RISCV_EXCP_INST_PAGE_FAULT : RISCV_EXCP_INST_ACCESS_FAULT;
 496         break;
 497     case MMU_DATA_LOAD:
 498         cs->exception_index = page_fault_exceptions ?
 499             RISCV_EXCP_LOAD_PAGE_FAULT : RISCV_EXCP_LOAD_ACCESS_FAULT;
 500         break;
 501     case MMU_DATA_STORE:
 502         cs->exception_index = page_fault_exceptions ?
 503             RISCV_EXCP_STORE_PAGE_FAULT : RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
 504         break;
 505     default:
 506         g_assert_not_reached();
 507     }
 508     env->badaddr = address;
 509 }
 510
 511 hwaddr riscv_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
 512 {
 513     RISCVCPU *cpu = RISCV_CPU(cs);
 514     hwaddr phys_addr;
 515     int prot;
 516     int mmu_idx = cpu_mmu_index(&cpu->env, false);
 517
 518     if (get_physical_address(&cpu->env, &phys_addr, &prot, addr, 0, mmu_idx)) {
 519         return -1;
 520     }
 521     return phys_addr;
 522 }
 523
 524 void riscv_cpu_do_transaction_failed(CPUState *cs, hwaddr physaddr,
 525                                      vaddr addr, unsigned size,
 526                                      MMUAccessType access_type,
 527                                      int mmu_idx, MemTxAttrs attrs,
 528                                      MemTxResult response, uintptr_t retaddr)
 529 {
 530     RISCVCPU *cpu = RISCV_CPU(cs);
 531     CPURISCVState *env = &cpu->env;
 532
 533     if (access_type == MMU_DATA_STORE) {
 534         cs->exception_index = RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
 535     } else {
 536         cs->exception_index = RISCV_EXCP_LOAD_ACCESS_FAULT;
 537     }
 538
 539     env->badaddr = addr;
 540     riscv_raise_exception(&cpu->env, cs->exception_index, retaddr);
 541 }
 542
 543 void riscv_cpu_do_unaligned_access(CPUState *cs, vaddr addr,
 544                                    MMUAccessType access_type, int mmu_idx,
 545                                    uintptr_t retaddr)
 546 {
 547     RISCVCPU *cpu = RISCV_CPU(cs);
 548     CPURISCVState *env = &cpu->env;
 549     switch (access_type) {
 550     case MMU_INST_FETCH:
 551         cs->exception_index = RISCV_EXCP_INST_ADDR_MIS;
 552         break;
 553     case MMU_DATA_LOAD:
 554         cs->exception_index = RISCV_EXCP_LOAD_ADDR_MIS;
 555         break;
 556     case MMU_DATA_STORE:
 557         cs->exception_index = RISCV_EXCP_STORE_AMO_ADDR_MIS;
 558         break;
 559     default:
 560         g_assert_not_reached();
 561     }
 562     env->badaddr = addr;
 563     riscv_raise_exception(env, cs->exception_index, retaddr);
 564 }
 565 #endif
 566
 567 bool riscv_cpu_tlb_fill(CPUState *cs, vaddr address, int size,
 568                         MMUAccessType access_type, int mmu_idx,
 569                         bool probe, uintptr_t retaddr)
 570 {
 571     RISCVCPU *cpu = RISCV_CPU(cs);
 572     CPURISCVState *env = &cpu->env;
 573 #ifndef CONFIG_USER_ONLY
 574     hwaddr pa = 0;
 575     int prot;
 576     bool pmp_violation = false;
 577     int ret = TRANSLATE_FAIL;
 578     int mode = mmu_idx;
 579
 580     qemu_log_mask(CPU_LOG_MMU, "%s ad %" VADDR_PRIx " rw %d mmu_idx %d\n",
 581                   __func__, address, access_type, mmu_idx);
 582
 583     ret = get_physical_address(env, &pa, &prot, address, access_type, mmu_idx);
 584
 585     if (mode == PRV_M && access_type != MMU_INST_FETCH) {
 586         if (get_field(env->mstatus, MSTATUS_MPRV)) {
 587             mode = get_field(env->mstatus, MSTATUS_MPP);
 588         }
 589     }
 590
 591     qemu_log_mask(CPU_LOG_MMU,
 592                   "%s address=%" VADDR_PRIx " ret %d physical " TARGET_FMT_plx
 593                   " prot %d\n", __func__, address, ret, pa, prot);
 594
 595     if (riscv_feature(env, RISCV_FEATURE_PMP) &&
 596         (ret == TRANSLATE_SUCCESS) &&
 597         !pmp_hart_has_privs(env, pa, size, 1 << access_type, mode)) {
 598         ret = TRANSLATE_PMP_FAIL;
 599     }
 600     if (ret == TRANSLATE_PMP_FAIL) {
 601         pmp_violation = true;
 602     }
 603     if (ret == TRANSLATE_SUCCESS) {
 604         tlb_set_page(cs, address & TARGET_PAGE_MASK, pa & TARGET_PAGE_MASK,
 605                      prot, mmu_idx, TARGET_PAGE_SIZE);
 606         return true;
 607     } else if (probe) {
 608         return false;
 609     } else {
 610         raise_mmu_exception(env, address, access_type, pmp_violation);
 611         riscv_raise_exception(env, cs->exception_index, retaddr);
 612     }
 613 #else
 614     switch (access_type) {
 615     case MMU_INST_FETCH:
 616         cs->exception_index = RISCV_EXCP_INST_PAGE_FAULT;
 617         break;
 618     case MMU_DATA_LOAD:
 619         cs->exception_index = RISCV_EXCP_LOAD_PAGE_FAULT;
 620         break;
 621     case MMU_DATA_STORE:
 622         cs->exception_index = RISCV_EXCP_STORE_PAGE_FAULT;
 623         break;
 624     default:
 625         g_assert_not_reached();
 626     }
 627     env->badaddr = address;
 628     cpu_loop_exit_restore(cs, retaddr);
 629 #endif
 630 }
 631
 632 /*
 633  * Handle Traps
 634  *
 635  * Adapted from Spike's processor_t::take_trap.
 636  *
 637  */
 638 void riscv_cpu_do_interrupt(CPUState *cs)
 639 {
 640 #if !defined(CONFIG_USER_ONLY)
 641
 642     RISCVCPU *cpu = RISCV_CPU(cs);
 643     CPURISCVState *env = &cpu->env;
 644     bool force_hs_execp = riscv_cpu_force_hs_excep_enabled(env);
 645     target_ulong s;
 646
 647     /* cs->exception is 32-bits wide unlike mcause which is XLEN-bits wide
 648      * so we mask off the MSB and separate into trap type and cause.
 649      */
 650     bool async = !!(cs->exception_index & RISCV_EXCP_INT_FLAG);
 651     target_ulong cause = cs->exception_index & RISCV_EXCP_INT_MASK;
 652     target_ulong deleg = async ? env->mideleg : env->medeleg;
 653     target_ulong tval = 0;
 654
 655     if (!async) {
 656         /* set tval to badaddr for traps with address information */
 657         switch (cause) {
 658         case RISCV_EXCP_INST_GUEST_PAGE_FAULT:
 659         case RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT:
 660         case RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT:
 661             force_hs_execp = true;
 662             /* fallthrough */
 663         case RISCV_EXCP_INST_ADDR_MIS:
 664         case RISCV_EXCP_INST_ACCESS_FAULT:
 665         case RISCV_EXCP_LOAD_ADDR_MIS:
 666         case RISCV_EXCP_STORE_AMO_ADDR_MIS:
 667         case RISCV_EXCP_LOAD_ACCESS_FAULT:
 668         case RISCV_EXCP_STORE_AMO_ACCESS_FAULT:
 669         case RISCV_EXCP_INST_PAGE_FAULT:
 670         case RISCV_EXCP_LOAD_PAGE_FAULT:
 671         case RISCV_EXCP_STORE_PAGE_FAULT:
 672             tval = env->badaddr;
 673             break;
 674         default:
 675             break;
 676         }
 677         /* ecall is dispatched as one cause so translate based on mode */
 678         if (cause == RISCV_EXCP_U_ECALL) {
 679             assert(env->priv <= 3);
 680
 681             if (env->priv == PRV_M) {
 682                 cause = RISCV_EXCP_M_ECALL;
 683             } else if (env->priv == PRV_S && riscv_cpu_virt_enabled(env)) {
 684                 cause = RISCV_EXCP_VS_ECALL;
 685             } else if (env->priv == PRV_S && !riscv_cpu_virt_enabled(env)) {
 686                 cause = RISCV_EXCP_S_ECALL;
 687             } else if (env->priv == PRV_U) {
 688                 cause = RISCV_EXCP_U_ECALL;
 689             }
 690         }
 691     }
 692
 693     trace_riscv_trap(env->mhartid, async, cause, env->pc, tval, cause < 23 ?
 694         (async ? riscv_intr_names : riscv_excp_names)[cause] : "(unknown)");
 695
 696     if (env->priv <= PRV_S &&
 697             cause < TARGET_LONG_BITS && ((deleg >> cause) & 1)) {
 698         /* handle the trap in S-mode */
 699         if (riscv_has_ext(env, RVH)) {
 700             target_ulong hdeleg = async ? env->hideleg : env->hedeleg;
 701
 702             if (riscv_cpu_virt_enabled(env) && ((hdeleg >> cause) & 1) &&
 703                 !force_hs_execp) {
 704                 /* Trap to VS mode */
 705             } else if (riscv_cpu_virt_enabled(env)) {
 706                 /* Trap into HS mode, from virt */
 707                 riscv_cpu_swap_hypervisor_regs(env);
 708                 env->hstatus = set_field(env->hstatus, HSTATUS_SP2V,
 709                                          get_field(env->hstatus, HSTATUS_SPV));
 710                 env->hstatus = set_field(env->hstatus, HSTATUS_SP2P,
 711                                          get_field(env->mstatus, SSTATUS_SPP));
 712                 env->hstatus = set_field(env->hstatus, HSTATUS_SPV,
 713                                          riscv_cpu_virt_enabled(env));
 714
 715                 riscv_cpu_set_virt_enabled(env, 0);
 716                 riscv_cpu_set_force_hs_excep(env, 0);
 717             } else {
 718                 /* Trap into HS mode */
 719                 env->hstatus = set_field(env->hstatus, HSTATUS_SP2V,
 720                                          get_field(env->hstatus, HSTATUS_SPV));
 721                 env->hstatus = set_field(env->hstatus, HSTATUS_SP2P,
 722                                          get_field(env->mstatus, SSTATUS_SPP));
 723                 env->hstatus = set_field(env->hstatus, HSTATUS_SPV,
 724                                          riscv_cpu_virt_enabled(env));
 725             }
 726         }
 727
 728         s = env->mstatus;
 729         s = set_field(s, MSTATUS_SPIE, env->priv_ver >= PRIV_VERSION_1_10_0 ?
 730             get_field(s, MSTATUS_SIE) : get_field(s, MSTATUS_UIE << env->priv));
 731         s = set_field(s, MSTATUS_SPP, env->priv);
 732         s = set_field(s, MSTATUS_SIE, 0);
 733         env->mstatus = s;
 734         env->scause = cause | ((target_ulong)async << (TARGET_LONG_BITS - 1));
 735         env->sepc = env->pc;
 736         env->sbadaddr = tval;
 737         env->pc = (env->stvec >> 2 << 2) +
 738             ((async && (env->stvec & 3) == 1) ? cause * 4 : 0);
 739         riscv_cpu_set_mode(env, PRV_S);
 740     } else {
 741         /* handle the trap in M-mode */
 742         if (riscv_has_ext(env, RVH)) {
 743             if (riscv_cpu_virt_enabled(env)) {
 744                 riscv_cpu_swap_hypervisor_regs(env);
 745             }
 746             env->mstatus = set_field(env->mstatus, MSTATUS_MPV,
 747                                       riscv_cpu_virt_enabled(env));
 748             env->mstatus = set_field(env->mstatus, MSTATUS_MTL,
 749                                       riscv_cpu_force_hs_excep_enabled(env));
 750
 751             /* Trapping to M mode, virt is disabled */
 752             riscv_cpu_set_virt_enabled(env, 0);
 753             riscv_cpu_set_force_hs_excep(env, 0);
 754         }
 755
 756         s = env->mstatus;
 757         s = set_field(s, MSTATUS_MPIE, env->priv_ver >= PRIV_VERSION_1_10_0 ?
 758             get_field(s, MSTATUS_MIE) : get_field(s, MSTATUS_UIE << env->priv));
 759         s = set_field(s, MSTATUS_MPP, env->priv);
 760         s = set_field(s, MSTATUS_MIE, 0);
 761         env->mstatus = s;
 762         env->mcause = cause | ~(((target_ulong)-1) >> async);
 763         env->mepc = env->pc;
 764         env->mbadaddr = tval;
 765         env->pc = (env->mtvec >> 2 << 2) +
 766             ((async && (env->mtvec & 3) == 1) ? cause * 4 : 0);
 767         riscv_cpu_set_mode(env, PRV_M);
 768     }
 769
 770     /* NOTE: it is not necessary to yield load reservations here. It is only
 771      * necessary for an SC from "another hart" to cause a load reservation
 772      * to be yielded. Refer to the memory consistency model section of the
 773      * RISC-V ISA Specification.
 774      */
 775
 776 #endif
 777     cs->exception_index = EXCP_NONE; /* mark handled to qemu */
 778 }