cputlb.c

   1 /*
   2  *  Common CPU TLB handling
   3  *
   4  *  Copyright (c) 2003 Fabrice Bellard
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 #include "qemu/osdep.h"
  21 #include "qemu/main-loop.h"
  22 #include "cpu.h"
  23 #include "exec/exec-all.h"
  24 #include "exec/memory.h"
  25 #include "exec/address-spaces.h"
  26 #include "exec/cpu_ldst.h"
  27 #include "exec/cputlb.h"
  28 #include "exec/memory-internal.h"
  29 #include "exec/ram_addr.h"
  30 #include "tcg/tcg.h"
  31 #include "qemu/error-report.h"
  32 #include "exec/log.h"
  33 #include "exec/helper-proto.h"
  34 #include "qemu/atomic.h"
  35
  36 /* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */
  37 /* #define DEBUG_TLB */
  38 /* #define DEBUG_TLB_LOG */
  39
  40 #ifdef DEBUG_TLB
  41 # define DEBUG_TLB_GATE 1
  42 # ifdef DEBUG_TLB_LOG
  43 #  define DEBUG_TLB_LOG_GATE 1
  44 # else
  45 #  define DEBUG_TLB_LOG_GATE 0
  46 # endif
  47 #else
  48 # define DEBUG_TLB_GATE 0
  49 # define DEBUG_TLB_LOG_GATE 0
  50 #endif
  51
  52 #define tlb_debug(fmt, ...) do { \
  53     if (DEBUG_TLB_LOG_GATE) { \
  54         qemu_log_mask(CPU_LOG_MMU, "%s: " fmt, __func__, \
  55                       ## __VA_ARGS__); \
  56     } else if (DEBUG_TLB_GATE) { \
  57         fprintf(stderr, "%s: " fmt, __func__, ## __VA_ARGS__); \
  58     } \
  59 } while (0)
  60
  61 #define assert_cpu_is_self(this_cpu) do {                         \
  62         if (DEBUG_TLB_GATE) {                                     \
  63             g_assert(!cpu->created || qemu_cpu_is_self(cpu));     \
  64         }                                                         \
  65     } while (0)
  66
  67 /* run_on_cpu_data.target_ptr should always be big enough for a
  68  * target_ulong even on 32 bit builds */
  69 QEMU_BUILD_BUG_ON(sizeof(target_ulong) > sizeof(run_on_cpu_data));
  70
  71 /* We currently can't handle more than 16 bits in the MMUIDX bitmask.
  72  */
  73 QEMU_BUILD_BUG_ON(NB_MMU_MODES > 16);
  74 #define ALL_MMUIDX_BITS ((1 << NB_MMU_MODES) - 1)
  75
  76 /* flush_all_helper: run fn across all cpus
  77  *
  78  * If the wait flag is set then the src cpu's helper will be queued as
  79  * "safe" work and the loop exited creating a synchronisation point
  80  * where all queued work will be finished before execution starts
  81  * again.
  82  */
  83 static void flush_all_helper(CPUState *src, run_on_cpu_func fn,
  84                              run_on_cpu_data d)
  85 {
  86     CPUState *cpu;
  87
  88     CPU_FOREACH(cpu) {
  89         if (cpu != src) {
  90             async_run_on_cpu(cpu, fn, d);
  91         }
  92     }
  93 }
  94
  95 /* statistics */
  96 int tlb_flush_count;
  97
  98 /* This is OK because CPU architectures generally permit an
  99  * implementation to drop entries from the TLB at any time, so
 100  * flushing more entries than required is only an efficiency issue,
 101  * not a correctness issue.
 102  */
 103 static void tlb_flush_nocheck(CPUState *cpu)
 104 {
 105     CPUArchState *env = cpu->env_ptr;
 106
 107     /* The QOM tests will trigger tlb_flushes without setting up TCG
 108      * so we bug out here in that case.
 109      */
 110     if (!tcg_enabled()) {
 111         return;
 112     }
 113
 114     assert_cpu_is_self(cpu);
 115     tlb_debug("(count: %d)\n", tlb_flush_count++);
 116
 117     tb_lock();
 118
 119     memset(env->tlb_table, -1, sizeof(env->tlb_table));
 120     memset(env->tlb_v_table, -1, sizeof(env->tlb_v_table));
 121     memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
 122
 123     env->vtlb_index = 0;
 124     env->tlb_flush_addr = -1;
 125     env->tlb_flush_mask = 0;
 126
 127     tb_unlock();
 128
 129     atomic_mb_set(&cpu->pending_tlb_flush, 0);
 130 }
 131
 132 static void tlb_flush_global_async_work(CPUState *cpu, run_on_cpu_data data)
 133 {
 134     tlb_flush_nocheck(cpu);
 135 }
 136
 137 void tlb_flush(CPUState *cpu)
 138 {
 139     if (cpu->created && !qemu_cpu_is_self(cpu)) {
 140         if (atomic_mb_read(&cpu->pending_tlb_flush) != ALL_MMUIDX_BITS) {
 141             atomic_mb_set(&cpu->pending_tlb_flush, ALL_MMUIDX_BITS);
 142             async_run_on_cpu(cpu, tlb_flush_global_async_work,
 143                              RUN_ON_CPU_NULL);
 144         }
 145     } else {
 146         tlb_flush_nocheck(cpu);
 147     }
 148 }
 149
 150 void tlb_flush_all_cpus(CPUState *src_cpu)
 151 {
 152     const run_on_cpu_func fn = tlb_flush_global_async_work;
 153     flush_all_helper(src_cpu, fn, RUN_ON_CPU_NULL);
 154     fn(src_cpu, RUN_ON_CPU_NULL);
 155 }
 156
 157 void tlb_flush_all_cpus_synced(CPUState *src_cpu)
 158 {
 159     const run_on_cpu_func fn = tlb_flush_global_async_work;
 160     flush_all_helper(src_cpu, fn, RUN_ON_CPU_NULL);
 161     async_safe_run_on_cpu(src_cpu, fn, RUN_ON_CPU_NULL);
 162 }
 163
 164 static void tlb_flush_by_mmuidx_async_work(CPUState *cpu, run_on_cpu_data data)
 165 {
 166     CPUArchState *env = cpu->env_ptr;
 167     unsigned long mmu_idx_bitmask = data.host_int;
 168     int mmu_idx;
 169
 170     assert_cpu_is_self(cpu);
 171
 172     tb_lock();
 173
 174     tlb_debug("start: mmu_idx:0x%04lx\n", mmu_idx_bitmask);
 175
 176     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
 177
 178         if (test_bit(mmu_idx, &mmu_idx_bitmask)) {
 179             tlb_debug("%d\n", mmu_idx);
 180
 181             memset(env->tlb_table[mmu_idx], -1, sizeof(env->tlb_table[0]));
 182             memset(env->tlb_v_table[mmu_idx], -1, sizeof(env->tlb_v_table[0]));
 183         }
 184     }
 185
 186     memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
 187
 188     tlb_debug("done\n");
 189
 190     tb_unlock();
 191 }
 192
 193 void tlb_flush_by_mmuidx(CPUState *cpu, uint16_t idxmap)
 194 {
 195     tlb_debug("mmu_idx: 0x%" PRIx16 "\n", idxmap);
 196
 197     if (!qemu_cpu_is_self(cpu)) {
 198         uint16_t pending_flushes = idxmap;
 199         pending_flushes &= ~atomic_mb_read(&cpu->pending_tlb_flush);
 200
 201         if (pending_flushes) {
 202             tlb_debug("reduced mmu_idx: 0x%" PRIx16 "\n", pending_flushes);
 203
 204             atomic_or(&cpu->pending_tlb_flush, pending_flushes);
 205             async_run_on_cpu(cpu, tlb_flush_by_mmuidx_async_work,
 206                              RUN_ON_CPU_HOST_INT(pending_flushes));
 207         }
 208     } else {
 209         tlb_flush_by_mmuidx_async_work(cpu,
 210                                        RUN_ON_CPU_HOST_INT(idxmap));
 211     }
 212 }
 213
 214 void tlb_flush_by_mmuidx_all_cpus(CPUState *src_cpu, uint16_t idxmap)
 215 {
 216     const run_on_cpu_func fn = tlb_flush_by_mmuidx_async_work;
 217
 218     tlb_debug("mmu_idx: 0x%"PRIx16"\n", idxmap);
 219
 220     flush_all_helper(src_cpu, fn, RUN_ON_CPU_HOST_INT(idxmap));
 221     fn(src_cpu, RUN_ON_CPU_HOST_INT(idxmap));
 222 }
 223
 224 void tlb_flush_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
 225                                                        uint16_t idxmap)
 226 {
 227     const run_on_cpu_func fn = tlb_flush_by_mmuidx_async_work;
 228
 229     tlb_debug("mmu_idx: 0x%"PRIx16"\n", idxmap);
 230
 231     flush_all_helper(src_cpu, fn, RUN_ON_CPU_HOST_INT(idxmap));
 232     async_safe_run_on_cpu(src_cpu, fn, RUN_ON_CPU_HOST_INT(idxmap));
 233 }
 234
 235
 236
 237 static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
 238 {
 239     if (addr == (tlb_entry->addr_read &
 240                  (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
 241         addr == (tlb_entry->addr_write &
 242                  (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
 243         addr == (tlb_entry->addr_code &
 244                  (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
 245         memset(tlb_entry, -1, sizeof(*tlb_entry));
 246     }
 247 }
 248
 249 static void tlb_flush_page_async_work(CPUState *cpu, run_on_cpu_data data)
 250 {
 251     CPUArchState *env = cpu->env_ptr;
 252     target_ulong addr = (target_ulong) data.target_ptr;
 253     int i;
 254     int mmu_idx;
 255
 256     assert_cpu_is_self(cpu);
 257
 258     tlb_debug("page :" TARGET_FMT_lx "\n", addr);
 259
 260     /* Check if we need to flush due to large pages.  */
 261     if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
 262         tlb_debug("forcing full flush ("
 263                   TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
 264                   env->tlb_flush_addr, env->tlb_flush_mask);
 265
 266         tlb_flush(cpu);
 267         return;
 268     }
 269
 270     addr &= TARGET_PAGE_MASK;
 271     i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 272     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
 273         tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
 274     }
 275
 276     /* check whether there are entries that need to be flushed in the vtlb */
 277     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
 278         int k;
 279         for (k = 0; k < CPU_VTLB_SIZE; k++) {
 280             tlb_flush_entry(&env->tlb_v_table[mmu_idx][k], addr);
 281         }
 282     }
 283
 284     tb_flush_jmp_cache(cpu, addr);
 285 }
 286
 287 void tlb_flush_page(CPUState *cpu, target_ulong addr)
 288 {
 289     tlb_debug("page :" TARGET_FMT_lx "\n", addr);
 290
 291     if (!qemu_cpu_is_self(cpu)) {
 292         async_run_on_cpu(cpu, tlb_flush_page_async_work,
 293                          RUN_ON_CPU_TARGET_PTR(addr));
 294     } else {
 295         tlb_flush_page_async_work(cpu, RUN_ON_CPU_TARGET_PTR(addr));
 296     }
 297 }
 298
 299 /* As we are going to hijack the bottom bits of the page address for a
 300  * mmuidx bit mask we need to fail to build if we can't do that
 301  */
 302 QEMU_BUILD_BUG_ON(NB_MMU_MODES > TARGET_PAGE_BITS_MIN);
 303
 304 static void tlb_flush_page_by_mmuidx_async_work(CPUState *cpu,
 305                                                 run_on_cpu_data data)
 306 {
 307     CPUArchState *env = cpu->env_ptr;
 308     target_ulong addr_and_mmuidx = (target_ulong) data.target_ptr;
 309     target_ulong addr = addr_and_mmuidx & TARGET_PAGE_MASK;
 310     unsigned long mmu_idx_bitmap = addr_and_mmuidx & ALL_MMUIDX_BITS;
 311     int page = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 312     int mmu_idx;
 313     int i;
 314
 315     assert_cpu_is_self(cpu);
 316
 317     tlb_debug("page:%d addr:"TARGET_FMT_lx" mmu_idx:0x%lx\n",
 318               page, addr, mmu_idx_bitmap);
 319
 320     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
 321         if (test_bit(mmu_idx, &mmu_idx_bitmap)) {
 322             tlb_flush_entry(&env->tlb_table[mmu_idx][page], addr);
 323
 324             /* check whether there are vltb entries that need to be flushed */
 325             for (i = 0; i < CPU_VTLB_SIZE; i++) {
 326                 tlb_flush_entry(&env->tlb_v_table[mmu_idx][i], addr);
 327             }
 328         }
 329     }
 330
 331     tb_flush_jmp_cache(cpu, addr);
 332 }
 333
 334 static void tlb_check_page_and_flush_by_mmuidx_async_work(CPUState *cpu,
 335                                                           run_on_cpu_data data)
 336 {
 337     CPUArchState *env = cpu->env_ptr;
 338     target_ulong addr_and_mmuidx = (target_ulong) data.target_ptr;
 339     target_ulong addr = addr_and_mmuidx & TARGET_PAGE_MASK;
 340     unsigned long mmu_idx_bitmap = addr_and_mmuidx & ALL_MMUIDX_BITS;
 341
 342     tlb_debug("addr:"TARGET_FMT_lx" mmu_idx: %04lx\n", addr, mmu_idx_bitmap);
 343
 344     /* Check if we need to flush due to large pages.  */
 345     if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
 346         tlb_debug("forced full flush ("
 347                   TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
 348                   env->tlb_flush_addr, env->tlb_flush_mask);
 349
 350         tlb_flush_by_mmuidx_async_work(cpu,
 351                                        RUN_ON_CPU_HOST_INT(mmu_idx_bitmap));
 352     } else {
 353         tlb_flush_page_by_mmuidx_async_work(cpu, data);
 354     }
 355 }
 356
 357 void tlb_flush_page_by_mmuidx(CPUState *cpu, target_ulong addr, uint16_t idxmap)
 358 {
 359     target_ulong addr_and_mmu_idx;
 360
 361     tlb_debug("addr: "TARGET_FMT_lx" mmu_idx:%" PRIx16 "\n", addr, idxmap);
 362
 363     /* This should already be page aligned */
 364     addr_and_mmu_idx = addr & TARGET_PAGE_MASK;
 365     addr_and_mmu_idx |= idxmap;
 366
 367     if (!qemu_cpu_is_self(cpu)) {
 368         async_run_on_cpu(cpu, tlb_check_page_and_flush_by_mmuidx_async_work,
 369                          RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx));
 370     } else {
 371         tlb_check_page_and_flush_by_mmuidx_async_work(
 372             cpu, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx));
 373     }
 374 }
 375
 376 void tlb_flush_page_by_mmuidx_all_cpus(CPUState *src_cpu, target_ulong addr,
 377                                        uint16_t idxmap)
 378 {
 379     const run_on_cpu_func fn = tlb_check_page_and_flush_by_mmuidx_async_work;
 380     target_ulong addr_and_mmu_idx;
 381
 382     tlb_debug("addr: "TARGET_FMT_lx" mmu_idx:%"PRIx16"\n", addr, idxmap);
 383
 384     /* This should already be page aligned */
 385     addr_and_mmu_idx = addr & TARGET_PAGE_MASK;
 386     addr_and_mmu_idx |= idxmap;
 387
 388     flush_all_helper(src_cpu, fn, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx));
 389     fn(src_cpu, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx));
 390 }
 391
 392 void tlb_flush_page_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
 393                                                             target_ulong addr,
 394                                                             uint16_t idxmap)
 395 {
 396     const run_on_cpu_func fn = tlb_check_page_and_flush_by_mmuidx_async_work;
 397     target_ulong addr_and_mmu_idx;
 398
 399     tlb_debug("addr: "TARGET_FMT_lx" mmu_idx:%"PRIx16"\n", addr, idxmap);
 400
 401     /* This should already be page aligned */
 402     addr_and_mmu_idx = addr & TARGET_PAGE_MASK;
 403     addr_and_mmu_idx |= idxmap;
 404
 405     flush_all_helper(src_cpu, fn, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx));
 406     async_safe_run_on_cpu(src_cpu, fn, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx));
 407 }
 408
 409 void tlb_flush_page_all_cpus(CPUState *src, target_ulong addr)
 410 {
 411     const run_on_cpu_func fn = tlb_flush_page_async_work;
 412
 413     flush_all_helper(src, fn, RUN_ON_CPU_TARGET_PTR(addr));
 414     fn(src, RUN_ON_CPU_TARGET_PTR(addr));
 415 }
 416
 417 void tlb_flush_page_all_cpus_synced(CPUState *src,
 418                                                   target_ulong addr)
 419 {
 420     const run_on_cpu_func fn = tlb_flush_page_async_work;
 421
 422     flush_all_helper(src, fn, RUN_ON_CPU_TARGET_PTR(addr));
 423     async_safe_run_on_cpu(src, fn, RUN_ON_CPU_TARGET_PTR(addr));
 424 }
 425
 426 /* update the TLBs so that writes to code in the virtual page 'addr'
 427    can be detected */
 428 void tlb_protect_code(ram_addr_t ram_addr)
 429 {
 430     cpu_physical_memory_test_and_clear_dirty(ram_addr, TARGET_PAGE_SIZE,
 431                                              DIRTY_MEMORY_CODE);
 432 }
 433
 434 /* update the TLB so that writes in physical page 'phys_addr' are no longer
 435    tested for self modifying code */
 436 void tlb_unprotect_code(ram_addr_t ram_addr)
 437 {
 438     cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_CODE);
 439 }
 440
 441
 442 /*
 443  * Dirty write flag handling
 444  *
 445  * When the TCG code writes to a location it looks up the address in
 446  * the TLB and uses that data to compute the final address. If any of
 447  * the lower bits of the address are set then the slow path is forced.
 448  * There are a number of reasons to do this but for normal RAM the
 449  * most usual is detecting writes to code regions which may invalidate
 450  * generated code.
 451  *
 452  * Because we want other vCPUs to respond to changes straight away we
 453  * update the te->addr_write field atomically. If the TLB entry has
 454  * been changed by the vCPU in the mean time we skip the update.
 455  *
 456  * As this function uses atomic accesses we also need to ensure
 457  * updates to tlb_entries follow the same access rules. We don't need
 458  * to worry about this for oversized guests as MTTCG is disabled for
 459  * them.
 460  */
 461
 462 static void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry, uintptr_t start,
 463                            uintptr_t length)
 464 {
 465 #if TCG_OVERSIZED_GUEST
 466     uintptr_t addr = tlb_entry->addr_write;
 467
 468     if ((addr & (TLB_INVALID_MASK | TLB_MMIO | TLB_NOTDIRTY)) == 0) {
 469         addr &= TARGET_PAGE_MASK;
 470         addr += tlb_entry->addend;
 471         if ((addr - start) < length) {
 472             tlb_entry->addr_write |= TLB_NOTDIRTY;
 473         }
 474     }
 475 #else
 476     /* paired with atomic_mb_set in tlb_set_page_with_attrs */
 477     uintptr_t orig_addr = atomic_mb_read(&tlb_entry->addr_write);
 478     uintptr_t addr = orig_addr;
 479
 480     if ((addr & (TLB_INVALID_MASK | TLB_MMIO | TLB_NOTDIRTY)) == 0) {
 481         addr &= TARGET_PAGE_MASK;
 482         addr += atomic_read(&tlb_entry->addend);
 483         if ((addr - start) < length) {
 484             uintptr_t notdirty_addr = orig_addr | TLB_NOTDIRTY;
 485             atomic_cmpxchg(&tlb_entry->addr_write, orig_addr, notdirty_addr);
 486         }
 487     }
 488 #endif
 489 }
 490
 491 /* For atomic correctness when running MTTCG we need to use the right
 492  * primitives when copying entries */
 493 static inline void copy_tlb_helper(CPUTLBEntry *d, CPUTLBEntry *s,
 494                                    bool atomic_set)
 495 {
 496 #if TCG_OVERSIZED_GUEST
 497     *d = *s;
 498 #else
 499     if (atomic_set) {
 500         d->addr_read = s->addr_read;
 501         d->addr_code = s->addr_code;
 502         atomic_set(&d->addend, atomic_read(&s->addend));
 503         /* Pairs with flag setting in tlb_reset_dirty_range */
 504         atomic_mb_set(&d->addr_write, atomic_read(&s->addr_write));
 505     } else {
 506         d->addr_read = s->addr_read;
 507         d->addr_write = atomic_read(&s->addr_write);
 508         d->addr_code = s->addr_code;
 509         d->addend = atomic_read(&s->addend);
 510     }
 511 #endif
 512 }
 513
 514 /* This is a cross vCPU call (i.e. another vCPU resetting the flags of
 515  * the target vCPU). As such care needs to be taken that we don't
 516  * dangerously race with another vCPU update. The only thing actually
 517  * updated is the target TLB entry ->addr_write flags.
 518  */
 519 void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length)
 520 {
 521     CPUArchState *env;
 522
 523     int mmu_idx;
 524
 525     env = cpu->env_ptr;
 526     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
 527         unsigned int i;
 528
 529         for (i = 0; i < CPU_TLB_SIZE; i++) {
 530             tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
 531                                   start1, length);
 532         }
 533
 534         for (i = 0; i < CPU_VTLB_SIZE; i++) {
 535             tlb_reset_dirty_range(&env->tlb_v_table[mmu_idx][i],
 536                                   start1, length);
 537         }
 538     }
 539 }
 540
 541 static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
 542 {
 543     if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY)) {
 544         tlb_entry->addr_write = vaddr;
 545     }
 546 }
 547
 548 /* update the TLB corresponding to virtual page vaddr
 549    so that it is no longer dirty */
 550 void tlb_set_dirty(CPUState *cpu, target_ulong vaddr)
 551 {
 552     CPUArchState *env = cpu->env_ptr;
 553     int i;
 554     int mmu_idx;
 555
 556     assert_cpu_is_self(cpu);
 557
 558     vaddr &= TARGET_PAGE_MASK;
 559     i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 560     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
 561         tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
 562     }
 563
 564     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
 565         int k;
 566         for (k = 0; k < CPU_VTLB_SIZE; k++) {
 567             tlb_set_dirty1(&env->tlb_v_table[mmu_idx][k], vaddr);
 568         }
 569     }
 570 }
 571
 572 /* Our TLB does not support large pages, so remember the area covered by
 573    large pages and trigger a full TLB flush if these are invalidated.  */
 574 static void tlb_add_large_page(CPUArchState *env, target_ulong vaddr,
 575                                target_ulong size)
 576 {
 577     target_ulong mask = ~(size - 1);
 578
 579     if (env->tlb_flush_addr == (target_ulong)-1) {
 580         env->tlb_flush_addr = vaddr & mask;
 581         env->tlb_flush_mask = mask;
 582         return;
 583     }
 584     /* Extend the existing region to include the new page.
 585        This is a compromise between unnecessary flushes and the cost
 586        of maintaining a full variable size TLB.  */
 587     mask &= env->tlb_flush_mask;
 588     while (((env->tlb_flush_addr ^ vaddr) & mask) != 0) {
 589         mask <<= 1;
 590     }
 591     env->tlb_flush_addr &= mask;
 592     env->tlb_flush_mask = mask;
 593 }
 594
 595 /* Add a new TLB entry. At most one entry for a given virtual address
 596  * is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
 597  * supplied size is only used by tlb_flush_page.
 598  *
 599  * Called from TCG-generated code, which is under an RCU read-side
 600  * critical section.
 601  */
 602 void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
 603                              hwaddr paddr, MemTxAttrs attrs, int prot,
 604                              int mmu_idx, target_ulong size)
 605 {
 606     CPUArchState *env = cpu->env_ptr;
 607     MemoryRegionSection *section;
 608     unsigned int index;
 609     target_ulong address;
 610     target_ulong code_address;
 611     uintptr_t addend;
 612     CPUTLBEntry *te, *tv, tn;
 613     hwaddr iotlb, xlat, sz;
 614     unsigned vidx = env->vtlb_index++ % CPU_VTLB_SIZE;
 615     int asidx = cpu_asidx_from_attrs(cpu, attrs);
 616
 617     assert_cpu_is_self(cpu);
 618     assert(size >= TARGET_PAGE_SIZE);
 619     if (size != TARGET_PAGE_SIZE) {
 620         tlb_add_large_page(env, vaddr, size);
 621     }
 622
 623     sz = size;
 624     section = address_space_translate_for_iotlb(cpu, asidx, paddr, &xlat, &sz);
 625     assert(sz >= TARGET_PAGE_SIZE);
 626
 627     tlb_debug("vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx
 628               " prot=%x idx=%d\n",
 629               vaddr, paddr, prot, mmu_idx);
 630
 631     address = vaddr;
 632     if (!memory_region_is_ram(section->mr) && !memory_region_is_romd(section->mr)) {
 633         /* IO memory case */
 634         address |= TLB_MMIO;
 635         addend = 0;
 636     } else {
 637         /* TLB_MMIO for rom/romd handled below */
 638         addend = (uintptr_t)memory_region_get_ram_ptr(section->mr) + xlat;
 639     }
 640
 641     code_address = address;
 642     iotlb = memory_region_section_get_iotlb(cpu, section, vaddr, paddr, xlat,
 643                                             prot, &address);
 644
 645     index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 646     te = &env->tlb_table[mmu_idx][index];
 647     /* do not discard the translation in te, evict it into a victim tlb */
 648     tv = &env->tlb_v_table[mmu_idx][vidx];
 649
 650     /* addr_write can race with tlb_reset_dirty_range */
 651     copy_tlb_helper(tv, te, true);
 652
 653     env->iotlb_v[mmu_idx][vidx] = env->iotlb[mmu_idx][index];
 654
 655     /* refill the tlb */
 656     env->iotlb[mmu_idx][index].addr = iotlb - vaddr;
 657     env->iotlb[mmu_idx][index].attrs = attrs;
 658
 659     /* Now calculate the new entry */
 660     tn.addend = addend - vaddr;
 661     if (prot & PAGE_READ) {
 662         tn.addr_read = address;
 663     } else {
 664         tn.addr_read = -1;
 665     }
 666
 667     if (prot & PAGE_EXEC) {
 668         tn.addr_code = code_address;
 669     } else {
 670         tn.addr_code = -1;
 671     }
 672
 673     tn.addr_write = -1;
 674     if (prot & PAGE_WRITE) {
 675         if ((memory_region_is_ram(section->mr) && section->readonly)
 676             || memory_region_is_romd(section->mr)) {
 677             /* Write access calls the I/O callback.  */
 678             tn.addr_write = address | TLB_MMIO;
 679         } else if (memory_region_is_ram(section->mr)
 680                    && cpu_physical_memory_is_clean(
 681                         memory_region_get_ram_addr(section->mr) + xlat)) {
 682             tn.addr_write = address | TLB_NOTDIRTY;
 683         } else {
 684             tn.addr_write = address;
 685         }
 686     }
 687
 688     /* Pairs with flag setting in tlb_reset_dirty_range */
 689     copy_tlb_helper(te, &tn, true);
 690     /* atomic_mb_set(&te->addr_write, write_address); */
 691 }
 692
 693 /* Add a new TLB entry, but without specifying the memory
 694  * transaction attributes to be used.
 695  */
 696 void tlb_set_page(CPUState *cpu, target_ulong vaddr,
 697                   hwaddr paddr, int prot,
 698                   int mmu_idx, target_ulong size)
 699 {
 700     tlb_set_page_with_attrs(cpu, vaddr, paddr, MEMTXATTRS_UNSPECIFIED,
 701                             prot, mmu_idx, size);
 702 }
 703
 704 static void report_bad_exec(CPUState *cpu, target_ulong addr)
 705 {
 706     /* Accidentally executing outside RAM or ROM is quite common for
 707      * several user-error situations, so report it in a way that
 708      * makes it clear that this isn't a QEMU bug and provide suggestions
 709      * about what a user could do to fix things.
 710      */
 711     error_report("Trying to execute code outside RAM or ROM at 0x"
 712                  TARGET_FMT_lx, addr);
 713     error_printf("This usually means one of the following happened:\n\n"
 714                  "(1) You told QEMU to execute a kernel for the wrong machine "
 715                  "type, and it crashed on startup (eg trying to run a "
 716                  "raspberry pi kernel on a versatilepb QEMU machine)\n"
 717                  "(2) You didn't give QEMU a kernel or BIOS filename at all, "
 718                  "and QEMU executed a ROM full of no-op instructions until "
 719                  "it fell off the end\n"
 720                  "(3) Your guest kernel has a bug and crashed by jumping "
 721                  "off into nowhere\n\n"
 722                  "This is almost always one of the first two, so check your "
 723                  "command line and that you are using the right type of kernel "
 724                  "for this machine.\n"
 725                  "If you think option (3) is likely then you can try debugging "
 726                  "your guest with the -d debug options; in particular "
 727                  "-d guest_errors will cause the log to include a dump of the "
 728                  "guest register state at this point.\n\n"
 729                  "Execution cannot continue; stopping here.\n\n");
 730
 731     /* Report also to the logs, with more detail including register dump */
 732     qemu_log_mask(LOG_GUEST_ERROR, "qemu: fatal: Trying to execute code "
 733                   "outside RAM or ROM at 0x" TARGET_FMT_lx "\n", addr);
 734     log_cpu_state_mask(LOG_GUEST_ERROR, cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
 735 }
 736
 737 static inline ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
 738 {
 739     ram_addr_t ram_addr;
 740
 741     ram_addr = qemu_ram_addr_from_host(ptr);
 742     if (ram_addr == RAM_ADDR_INVALID) {
 743         error_report("Bad ram pointer %p", ptr);
 744         abort();
 745     }
 746     return ram_addr;
 747 }
 748
 749 /* NOTE: this function can trigger an exception */
 750 /* NOTE2: the returned address is not exactly the physical address: it
 751  * is actually a ram_addr_t (in system mode; the user mode emulation
 752  * version of this function returns a guest virtual address).
 753  */
 754 tb_page_addr_t get_page_addr_code(CPUArchState *env1, target_ulong addr)
 755 {
 756     int mmu_idx, page_index, pd;
 757     void *p;
 758     MemoryRegion *mr;
 759     CPUState *cpu = ENV_GET_CPU(env1);
 760     CPUIOTLBEntry *iotlbentry;
 761
 762     page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 763     mmu_idx = cpu_mmu_index(env1, true);
 764     if (unlikely(env1->tlb_table[mmu_idx][page_index].addr_code !=
 765                  (addr & TARGET_PAGE_MASK))) {
 766         cpu_ldub_code(env1, addr);
 767     }
 768     iotlbentry = &env1->iotlb[mmu_idx][page_index];
 769     pd = iotlbentry->addr & ~TARGET_PAGE_MASK;
 770     mr = iotlb_to_region(cpu, pd, iotlbentry->attrs);
 771     if (memory_region_is_unassigned(mr)) {
 772         cpu_unassigned_access(cpu, addr, false, true, 0, 4);
 773         /* The CPU's unassigned access hook might have longjumped out
 774          * with an exception. If it didn't (or there was no hook) then
 775          * we can't proceed further.
 776          */
 777         report_bad_exec(cpu, addr);
 778         exit(1);
 779     }
 780     p = (void *)((uintptr_t)addr + env1->tlb_table[mmu_idx][page_index].addend);
 781     return qemu_ram_addr_from_host_nofail(p);
 782 }
 783
 784 static uint64_t io_readx(CPUArchState *env, CPUIOTLBEntry *iotlbentry,
 785                          target_ulong addr, uintptr_t retaddr, int size)
 786 {
 787     CPUState *cpu = ENV_GET_CPU(env);
 788     hwaddr physaddr = iotlbentry->addr;
 789     MemoryRegion *mr = iotlb_to_region(cpu, physaddr, iotlbentry->attrs);
 790     uint64_t val;
 791     bool locked = false;
 792
 793     physaddr = (physaddr & TARGET_PAGE_MASK) + addr;
 794     cpu->mem_io_pc = retaddr;
 795     if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu->can_do_io) {
 796         cpu_io_recompile(cpu, retaddr);
 797     }
 798
 799     cpu->mem_io_vaddr = addr;
 800
 801     if (mr->global_locking) {
 802         qemu_mutex_lock_iothread();
 803         locked = true;
 804     }
 805     memory_region_dispatch_read(mr, physaddr, &val, size, iotlbentry->attrs);
 806     if (locked) {
 807         qemu_mutex_unlock_iothread();
 808     }
 809
 810     return val;
 811 }
 812
 813 static void io_writex(CPUArchState *env, CPUIOTLBEntry *iotlbentry,
 814                       uint64_t val, target_ulong addr,
 815                       uintptr_t retaddr, int size)
 816 {
 817     CPUState *cpu = ENV_GET_CPU(env);
 818     hwaddr physaddr = iotlbentry->addr;
 819     MemoryRegion *mr = iotlb_to_region(cpu, physaddr, iotlbentry->attrs);
 820     bool locked = false;
 821
 822     physaddr = (physaddr & TARGET_PAGE_MASK) + addr;
 823     if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu->can_do_io) {
 824         cpu_io_recompile(cpu, retaddr);
 825     }
 826     cpu->mem_io_vaddr = addr;
 827     cpu->mem_io_pc = retaddr;
 828
 829     if (mr->global_locking) {
 830         qemu_mutex_lock_iothread();
 831         locked = true;
 832     }
 833     memory_region_dispatch_write(mr, physaddr, val, size, iotlbentry->attrs);
 834     if (locked) {
 835         qemu_mutex_unlock_iothread();
 836     }
 837 }
 838
 839 /* Return true if ADDR is present in the victim tlb, and has been copied
 840    back to the main tlb.  */
 841 static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index,
 842                            size_t elt_ofs, target_ulong page)
 843 {
 844     size_t vidx;
 845     for (vidx = 0; vidx < CPU_VTLB_SIZE; ++vidx) {
 846         CPUTLBEntry *vtlb = &env->tlb_v_table[mmu_idx][vidx];
 847         target_ulong cmp = *(target_ulong *)((uintptr_t)vtlb + elt_ofs);
 848
 849         if (cmp == page) {
 850             /* Found entry in victim tlb, swap tlb and iotlb.  */
 851             CPUTLBEntry tmptlb, *tlb = &env->tlb_table[mmu_idx][index];
 852
 853             copy_tlb_helper(&tmptlb, tlb, false);
 854             copy_tlb_helper(tlb, vtlb, true);
 855             copy_tlb_helper(vtlb, &tmptlb, true);
 856
 857             CPUIOTLBEntry tmpio, *io = &env->iotlb[mmu_idx][index];
 858             CPUIOTLBEntry *vio = &env->iotlb_v[mmu_idx][vidx];
 859             tmpio = *io; *io = *vio; *vio = tmpio;
 860             return true;
 861         }
 862     }
 863     return false;
 864 }
 865
 866 /* Macro to call the above, with local variables from the use context.  */
 867 #define VICTIM_TLB_HIT(TY, ADDR) \
 868   victim_tlb_hit(env, mmu_idx, index, offsetof(CPUTLBEntry, TY), \
 869                  (ADDR) & TARGET_PAGE_MASK)
 870
 871 /* Probe for whether the specified guest write access is permitted.
 872  * If it is not permitted then an exception will be taken in the same
 873  * way as if this were a real write access (and we will not return).
 874  * Otherwise the function will return, and there will be a valid
 875  * entry in the TLB for this access.
 876  */
 877 void probe_write(CPUArchState *env, target_ulong addr, int mmu_idx,
 878                  uintptr_t retaddr)
 879 {
 880     int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 881     target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
 882
 883     if ((addr & TARGET_PAGE_MASK)
 884         != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
 885         /* TLB entry is for a different page */
 886         if (!VICTIM_TLB_HIT(addr_write, addr)) {
 887             tlb_fill(ENV_GET_CPU(env), addr, MMU_DATA_STORE, mmu_idx, retaddr);
 888         }
 889     }
 890 }
 891
 892 /* Probe for a read-modify-write atomic operation.  Do not allow unaligned
 893  * operations, or io operations to proceed.  Return the host address.  */
 894 static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
 895                                TCGMemOpIdx oi, uintptr_t retaddr)
 896 {
 897     size_t mmu_idx = get_mmuidx(oi);
 898     size_t index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 899     CPUTLBEntry *tlbe = &env->tlb_table[mmu_idx][index];
 900     target_ulong tlb_addr = tlbe->addr_write;
 901     TCGMemOp mop = get_memop(oi);
 902     int a_bits = get_alignment_bits(mop);
 903     int s_bits = mop & MO_SIZE;
 904
 905     /* Adjust the given return address.  */
 906     retaddr -= GETPC_ADJ;
 907
 908     /* Enforce guest required alignment.  */
 909     if (unlikely(a_bits > 0 && (addr & ((1 << a_bits) - 1)))) {
 910         /* ??? Maybe indicate atomic op to cpu_unaligned_access */
 911         cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,
 912                              mmu_idx, retaddr);
 913     }
 914
 915     /* Enforce qemu required alignment.  */
 916     if (unlikely(addr & ((1 << s_bits) - 1))) {
 917         /* We get here if guest alignment was not requested,
 918            or was not enforced by cpu_unaligned_access above.
 919            We might widen the access and emulate, but for now
 920            mark an exception and exit the cpu loop.  */
 921         goto stop_the_world;
 922     }
 923
 924     /* Check TLB entry and enforce page permissions.  */
 925     if ((addr & TARGET_PAGE_MASK)
 926         != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
 927         if (!VICTIM_TLB_HIT(addr_write, addr)) {
 928             tlb_fill(ENV_GET_CPU(env), addr, MMU_DATA_STORE, mmu_idx, retaddr);
 929         }
 930         tlb_addr = tlbe->addr_write;
 931     }
 932
 933     /* Notice an IO access, or a notdirty page.  */
 934     if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
 935         /* There's really nothing that can be done to
 936            support this apart from stop-the-world.  */
 937         goto stop_the_world;
 938     }
 939
 940     /* Let the guest notice RMW on a write-only page.  */
 941     if (unlikely(tlbe->addr_read != tlb_addr)) {
 942         tlb_fill(ENV_GET_CPU(env), addr, MMU_DATA_LOAD, mmu_idx, retaddr);
 943         /* Since we don't support reads and writes to different addresses,
 944            and we do have the proper page loaded for write, this shouldn't
 945            ever return.  But just in case, handle via stop-the-world.  */
 946         goto stop_the_world;
 947     }
 948
 949     return (void *)((uintptr_t)addr + tlbe->addend);
 950
 951  stop_the_world:
 952     cpu_loop_exit_atomic(ENV_GET_CPU(env), retaddr);
 953 }
 954
 955 #ifdef TARGET_WORDS_BIGENDIAN
 956 # define TGT_BE(X)  (X)
 957 # define TGT_LE(X)  BSWAP(X)
 958 #else
 959 # define TGT_BE(X)  BSWAP(X)
 960 # define TGT_LE(X)  (X)
 961 #endif
 962
 963 #define MMUSUFFIX _mmu
 964
 965 #define DATA_SIZE 1
 966 #include "softmmu_template.h"
 967
 968 #define DATA_SIZE 2
 969 #include "softmmu_template.h"
 970
 971 #define DATA_SIZE 4
 972 #include "softmmu_template.h"
 973
 974 #define DATA_SIZE 8
 975 #include "softmmu_template.h"
 976
 977 /* First set of helpers allows passing in of OI and RETADDR.  This makes
 978    them callable from other helpers.  */
 979
 980 #define EXTRA_ARGS     , TCGMemOpIdx oi, uintptr_t retaddr
 981 #define ATOMIC_NAME(X) \
 982     HELPER(glue(glue(glue(atomic_ ## X, SUFFIX), END), _mmu))
 983 #define ATOMIC_MMU_LOOKUP  atomic_mmu_lookup(env, addr, oi, retaddr)
 984
 985 #define DATA_SIZE 1
 986 #include "atomic_template.h"
 987
 988 #define DATA_SIZE 2
 989 #include "atomic_template.h"
 990
 991 #define DATA_SIZE 4
 992 #include "atomic_template.h"
 993
 994 #ifdef CONFIG_ATOMIC64
 995 #define DATA_SIZE 8
 996 #include "atomic_template.h"
 997 #endif
 998
 999 #ifdef CONFIG_ATOMIC128
1000 #define DATA_SIZE 16
1001 #include "atomic_template.h"
1002 #endif
1003
1004 /* Second set of helpers are directly callable from TCG as helpers.  */
1005
1006 #undef EXTRA_ARGS
1007 #undef ATOMIC_NAME
1008 #undef ATOMIC_MMU_LOOKUP
1009 #define EXTRA_ARGS         , TCGMemOpIdx oi
1010 #define ATOMIC_NAME(X)     HELPER(glue(glue(atomic_ ## X, SUFFIX), END))
1011 #define ATOMIC_MMU_LOOKUP  atomic_mmu_lookup(env, addr, oi, GETPC())
1012
1013 #define DATA_SIZE 1
1014 #include "atomic_template.h"
1015
1016 #define DATA_SIZE 2
1017 #include "atomic_template.h"
1018
1019 #define DATA_SIZE 4
1020 #include "atomic_template.h"
1021
1022 #ifdef CONFIG_ATOMIC64
1023 #define DATA_SIZE 8
1024 #include "atomic_template.h"
1025 #endif
1026
1027 /* Code access functions.  */
1028
1029 #undef MMUSUFFIX
1030 #define MMUSUFFIX _cmmu
1031 #undef GETPC
1032 #define GETPC() ((uintptr_t)0)
1033 #define SOFTMMU_CODE_ACCESS
1034
1035 #define DATA_SIZE 1
1036 #include "softmmu_template.h"
1037
1038 #define DATA_SIZE 2
1039 #include "softmmu_template.h"
1040
1041 #define DATA_SIZE 4
1042 #include "softmmu_template.h"
1043
1044 #define DATA_SIZE 8
1045 #include "softmmu_template.h"