]> git.proxmox.com Git - mirror_qemu.git/blame - cputlb.c
pc-bios/s390-ccw: Remove duplicate blk_factor adjustment
[mirror_qemu.git] / cputlb.c
CommitLineData
0cac1b66
BS
1/*
2 * Common CPU TLB handling
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19
7b31bbc2 20#include "qemu/osdep.h"
8d04fb55 21#include "qemu/main-loop.h"
0cac1b66 22#include "cpu.h"
022c62cb
PB
23#include "exec/exec-all.h"
24#include "exec/memory.h"
25#include "exec/address-spaces.h"
f08b6170 26#include "exec/cpu_ldst.h"
022c62cb 27#include "exec/cputlb.h"
022c62cb 28#include "exec/memory-internal.h"
220c3ebd 29#include "exec/ram_addr.h"
0f590e74 30#include "tcg/tcg.h"
d7f30403
PM
31#include "qemu/error-report.h"
32#include "exec/log.h"
c482cb11
RH
33#include "exec/helper-proto.h"
34#include "qemu/atomic.h"
0cac1b66 35
8526e1f4
AB
36/* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */
37/* #define DEBUG_TLB */
38/* #define DEBUG_TLB_LOG */
39
40#ifdef DEBUG_TLB
41# define DEBUG_TLB_GATE 1
42# ifdef DEBUG_TLB_LOG
43# define DEBUG_TLB_LOG_GATE 1
44# else
45# define DEBUG_TLB_LOG_GATE 0
46# endif
47#else
48# define DEBUG_TLB_GATE 0
49# define DEBUG_TLB_LOG_GATE 0
50#endif
51
52#define tlb_debug(fmt, ...) do { \
53 if (DEBUG_TLB_LOG_GATE) { \
54 qemu_log_mask(CPU_LOG_MMU, "%s: " fmt, __func__, \
55 ## __VA_ARGS__); \
56 } else if (DEBUG_TLB_GATE) { \
57 fprintf(stderr, "%s: " fmt, __func__, ## __VA_ARGS__); \
58 } \
59} while (0)
0cac1b66 60
f0aff0f1
AB
61#define assert_cpu_is_self(this_cpu) do { \
62 if (DEBUG_TLB_GATE) { \
63 g_assert(!cpu->created || qemu_cpu_is_self(cpu)); \
64 } \
65 } while (0)
66
e3b9ca81
FK
67/* run_on_cpu_data.target_ptr should always be big enough for a
68 * target_ulong even on 32 bit builds */
69QEMU_BUILD_BUG_ON(sizeof(target_ulong) > sizeof(run_on_cpu_data));
70
e7218445
AB
71/* We currently can't handle more than 16 bits in the MMUIDX bitmask.
72 */
73QEMU_BUILD_BUG_ON(NB_MMU_MODES > 16);
74#define ALL_MMUIDX_BITS ((1 << NB_MMU_MODES) - 1)
75
c3b9a07a
AB
76/* flush_all_helper: run fn across all cpus
77 *
78 * If the wait flag is set then the src cpu's helper will be queued as
79 * "safe" work and the loop exited creating a synchronisation point
80 * where all queued work will be finished before execution starts
81 * again.
82 */
83static void flush_all_helper(CPUState *src, run_on_cpu_func fn,
84 run_on_cpu_data d)
85{
86 CPUState *cpu;
87
88 CPU_FOREACH(cpu) {
89 if (cpu != src) {
90 async_run_on_cpu(cpu, fn, d);
91 }
92 }
93}
94
0cac1b66
BS
95/* statistics */
96int tlb_flush_count;
97
d10eb08f
AB
98/* This is OK because CPU architectures generally permit an
99 * implementation to drop entries from the TLB at any time, so
100 * flushing more entries than required is only an efficiency issue,
101 * not a correctness issue.
0cac1b66 102 */
e3b9ca81 103static void tlb_flush_nocheck(CPUState *cpu)
0cac1b66 104{
00c8cb0a 105 CPUArchState *env = cpu->env_ptr;
0cac1b66 106
e3b9ca81
FK
107 /* The QOM tests will trigger tlb_flushes without setting up TCG
108 * so we bug out here in that case.
109 */
110 if (!tcg_enabled()) {
111 return;
112 }
113
f0aff0f1
AB
114 assert_cpu_is_self(cpu);
115 tlb_debug("(count: %d)\n", tlb_flush_count++);
116
e3b9ca81
FK
117 tb_lock();
118
4fadb3bb 119 memset(env->tlb_table, -1, sizeof(env->tlb_table));
88e89a57 120 memset(env->tlb_v_table, -1, sizeof(env->tlb_v_table));
8cd70437 121 memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
0cac1b66 122
88e89a57 123 env->vtlb_index = 0;
0cac1b66
BS
124 env->tlb_flush_addr = -1;
125 env->tlb_flush_mask = 0;
e3b9ca81
FK
126
127 tb_unlock();
128
e7218445 129 atomic_mb_set(&cpu->pending_tlb_flush, 0);
e3b9ca81
FK
130}
131
132static void tlb_flush_global_async_work(CPUState *cpu, run_on_cpu_data data)
133{
134 tlb_flush_nocheck(cpu);
135}
136
137void tlb_flush(CPUState *cpu)
138{
139 if (cpu->created && !qemu_cpu_is_self(cpu)) {
e7218445
AB
140 if (atomic_mb_read(&cpu->pending_tlb_flush) != ALL_MMUIDX_BITS) {
141 atomic_mb_set(&cpu->pending_tlb_flush, ALL_MMUIDX_BITS);
e3b9ca81
FK
142 async_run_on_cpu(cpu, tlb_flush_global_async_work,
143 RUN_ON_CPU_NULL);
144 }
145 } else {
146 tlb_flush_nocheck(cpu);
147 }
0cac1b66
BS
148}
149
c3b9a07a
AB
150void tlb_flush_all_cpus(CPUState *src_cpu)
151{
152 const run_on_cpu_func fn = tlb_flush_global_async_work;
153 flush_all_helper(src_cpu, fn, RUN_ON_CPU_NULL);
154 fn(src_cpu, RUN_ON_CPU_NULL);
155}
156
157void tlb_flush_all_cpus_synced(CPUState *src_cpu)
158{
159 const run_on_cpu_func fn = tlb_flush_global_async_work;
160 flush_all_helper(src_cpu, fn, RUN_ON_CPU_NULL);
161 async_safe_run_on_cpu(src_cpu, fn, RUN_ON_CPU_NULL);
162}
163
e7218445 164static void tlb_flush_by_mmuidx_async_work(CPUState *cpu, run_on_cpu_data data)
d7a74a9d
PM
165{
166 CPUArchState *env = cpu->env_ptr;
e7218445 167 unsigned long mmu_idx_bitmask = data.host_int;
0336cbf8 168 int mmu_idx;
d7a74a9d 169
f0aff0f1 170 assert_cpu_is_self(cpu);
d7a74a9d 171
e3b9ca81
FK
172 tb_lock();
173
e7218445
AB
174 tlb_debug("start: mmu_idx:0x%04lx\n", mmu_idx_bitmask);
175
0336cbf8 176 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
d7a74a9d 177
0336cbf8
AB
178 if (test_bit(mmu_idx, &mmu_idx_bitmask)) {
179 tlb_debug("%d\n", mmu_idx);
d7a74a9d 180
0336cbf8
AB
181 memset(env->tlb_table[mmu_idx], -1, sizeof(env->tlb_table[0]));
182 memset(env->tlb_v_table[mmu_idx], -1, sizeof(env->tlb_v_table[0]));
183 }
d7a74a9d
PM
184 }
185
d7a74a9d 186 memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
e3b9ca81 187
e7218445
AB
188 tlb_debug("done\n");
189
e3b9ca81 190 tb_unlock();
d7a74a9d
PM
191}
192
0336cbf8 193void tlb_flush_by_mmuidx(CPUState *cpu, uint16_t idxmap)
d7a74a9d 194{
e7218445
AB
195 tlb_debug("mmu_idx: 0x%" PRIx16 "\n", idxmap);
196
197 if (!qemu_cpu_is_self(cpu)) {
198 uint16_t pending_flushes = idxmap;
199 pending_flushes &= ~atomic_mb_read(&cpu->pending_tlb_flush);
200
201 if (pending_flushes) {
202 tlb_debug("reduced mmu_idx: 0x%" PRIx16 "\n", pending_flushes);
203
204 atomic_or(&cpu->pending_tlb_flush, pending_flushes);
205 async_run_on_cpu(cpu, tlb_flush_by_mmuidx_async_work,
206 RUN_ON_CPU_HOST_INT(pending_flushes));
207 }
208 } else {
209 tlb_flush_by_mmuidx_async_work(cpu,
210 RUN_ON_CPU_HOST_INT(idxmap));
211 }
d7a74a9d
PM
212}
213
c3b9a07a
AB
214void tlb_flush_by_mmuidx_all_cpus(CPUState *src_cpu, uint16_t idxmap)
215{
216 const run_on_cpu_func fn = tlb_flush_by_mmuidx_async_work;
217
218 tlb_debug("mmu_idx: 0x%"PRIx16"\n", idxmap);
219
220 flush_all_helper(src_cpu, fn, RUN_ON_CPU_HOST_INT(idxmap));
221 fn(src_cpu, RUN_ON_CPU_HOST_INT(idxmap));
222}
223
224void tlb_flush_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
225 uint16_t idxmap)
226{
227 const run_on_cpu_func fn = tlb_flush_by_mmuidx_async_work;
228
229 tlb_debug("mmu_idx: 0x%"PRIx16"\n", idxmap);
230
231 flush_all_helper(src_cpu, fn, RUN_ON_CPU_HOST_INT(idxmap));
232 async_safe_run_on_cpu(src_cpu, fn, RUN_ON_CPU_HOST_INT(idxmap));
233}
234
235
236
0cac1b66
BS
237static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
238{
239 if (addr == (tlb_entry->addr_read &
240 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
241 addr == (tlb_entry->addr_write &
242 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
243 addr == (tlb_entry->addr_code &
244 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
4fadb3bb 245 memset(tlb_entry, -1, sizeof(*tlb_entry));
0cac1b66
BS
246 }
247}
248
e3b9ca81 249static void tlb_flush_page_async_work(CPUState *cpu, run_on_cpu_data data)
0cac1b66 250{
31b030d4 251 CPUArchState *env = cpu->env_ptr;
e3b9ca81 252 target_ulong addr = (target_ulong) data.target_ptr;
0cac1b66
BS
253 int i;
254 int mmu_idx;
255
f0aff0f1 256 assert_cpu_is_self(cpu);
e3b9ca81 257
8526e1f4
AB
258 tlb_debug("page :" TARGET_FMT_lx "\n", addr);
259
0cac1b66
BS
260 /* Check if we need to flush due to large pages. */
261 if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
8526e1f4
AB
262 tlb_debug("forcing full flush ("
263 TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
264 env->tlb_flush_addr, env->tlb_flush_mask);
265
d10eb08f 266 tlb_flush(cpu);
0cac1b66
BS
267 return;
268 }
0cac1b66
BS
269
270 addr &= TARGET_PAGE_MASK;
271 i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
272 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
273 tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
274 }
275
88e89a57
XT
276 /* check whether there are entries that need to be flushed in the vtlb */
277 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
278 int k;
279 for (k = 0; k < CPU_VTLB_SIZE; k++) {
280 tlb_flush_entry(&env->tlb_v_table[mmu_idx][k], addr);
281 }
282 }
283
611d4f99 284 tb_flush_jmp_cache(cpu, addr);
0cac1b66
BS
285}
286
e3b9ca81
FK
287void tlb_flush_page(CPUState *cpu, target_ulong addr)
288{
289 tlb_debug("page :" TARGET_FMT_lx "\n", addr);
290
291 if (!qemu_cpu_is_self(cpu)) {
292 async_run_on_cpu(cpu, tlb_flush_page_async_work,
293 RUN_ON_CPU_TARGET_PTR(addr));
294 } else {
295 tlb_flush_page_async_work(cpu, RUN_ON_CPU_TARGET_PTR(addr));
296 }
297}
298
e7218445
AB
299/* As we are going to hijack the bottom bits of the page address for a
300 * mmuidx bit mask we need to fail to build if we can't do that
301 */
302QEMU_BUILD_BUG_ON(NB_MMU_MODES > TARGET_PAGE_BITS_MIN);
303
304static void tlb_flush_page_by_mmuidx_async_work(CPUState *cpu,
305 run_on_cpu_data data)
d7a74a9d
PM
306{
307 CPUArchState *env = cpu->env_ptr;
e7218445
AB
308 target_ulong addr_and_mmuidx = (target_ulong) data.target_ptr;
309 target_ulong addr = addr_and_mmuidx & TARGET_PAGE_MASK;
310 unsigned long mmu_idx_bitmap = addr_and_mmuidx & ALL_MMUIDX_BITS;
311 int page = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
312 int mmu_idx;
313 int i;
d7a74a9d 314
f0aff0f1 315 assert_cpu_is_self(cpu);
d7a74a9d 316
e7218445
AB
317 tlb_debug("page:%d addr:"TARGET_FMT_lx" mmu_idx:0x%lx\n",
318 page, addr, mmu_idx_bitmap);
d7a74a9d 319
0336cbf8
AB
320 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
321 if (test_bit(mmu_idx, &mmu_idx_bitmap)) {
322 tlb_flush_entry(&env->tlb_table[mmu_idx][page], addr);
d7a74a9d 323
0336cbf8
AB
324 /* check whether there are vltb entries that need to be flushed */
325 for (i = 0; i < CPU_VTLB_SIZE; i++) {
326 tlb_flush_entry(&env->tlb_v_table[mmu_idx][i], addr);
327 }
d7a74a9d
PM
328 }
329 }
d7a74a9d 330
d7a74a9d
PM
331 tb_flush_jmp_cache(cpu, addr);
332}
333
e7218445
AB
334static void tlb_check_page_and_flush_by_mmuidx_async_work(CPUState *cpu,
335 run_on_cpu_data data)
336{
337 CPUArchState *env = cpu->env_ptr;
338 target_ulong addr_and_mmuidx = (target_ulong) data.target_ptr;
339 target_ulong addr = addr_and_mmuidx & TARGET_PAGE_MASK;
340 unsigned long mmu_idx_bitmap = addr_and_mmuidx & ALL_MMUIDX_BITS;
341
342 tlb_debug("addr:"TARGET_FMT_lx" mmu_idx: %04lx\n", addr, mmu_idx_bitmap);
343
344 /* Check if we need to flush due to large pages. */
345 if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
346 tlb_debug("forced full flush ("
347 TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
348 env->tlb_flush_addr, env->tlb_flush_mask);
349
350 tlb_flush_by_mmuidx_async_work(cpu,
351 RUN_ON_CPU_HOST_INT(mmu_idx_bitmap));
352 } else {
353 tlb_flush_page_by_mmuidx_async_work(cpu, data);
354 }
355}
356
357void tlb_flush_page_by_mmuidx(CPUState *cpu, target_ulong addr, uint16_t idxmap)
358{
359 target_ulong addr_and_mmu_idx;
360
361 tlb_debug("addr: "TARGET_FMT_lx" mmu_idx:%" PRIx16 "\n", addr, idxmap);
362
363 /* This should already be page aligned */
364 addr_and_mmu_idx = addr & TARGET_PAGE_MASK;
365 addr_and_mmu_idx |= idxmap;
366
367 if (!qemu_cpu_is_self(cpu)) {
368 async_run_on_cpu(cpu, tlb_check_page_and_flush_by_mmuidx_async_work,
369 RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx));
370 } else {
371 tlb_check_page_and_flush_by_mmuidx_async_work(
372 cpu, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx));
373 }
374}
375
c3b9a07a
AB
376void tlb_flush_page_by_mmuidx_all_cpus(CPUState *src_cpu, target_ulong addr,
377 uint16_t idxmap)
e3b9ca81 378{
c3b9a07a
AB
379 const run_on_cpu_func fn = tlb_check_page_and_flush_by_mmuidx_async_work;
380 target_ulong addr_and_mmu_idx;
e3b9ca81 381
c3b9a07a
AB
382 tlb_debug("addr: "TARGET_FMT_lx" mmu_idx:%"PRIx16"\n", addr, idxmap);
383
384 /* This should already be page aligned */
385 addr_and_mmu_idx = addr & TARGET_PAGE_MASK;
386 addr_and_mmu_idx |= idxmap;
387
388 flush_all_helper(src_cpu, fn, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx));
389 fn(src_cpu, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx));
390}
391
392void tlb_flush_page_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
393 target_ulong addr,
394 uint16_t idxmap)
395{
396 const run_on_cpu_func fn = tlb_check_page_and_flush_by_mmuidx_async_work;
397 target_ulong addr_and_mmu_idx;
398
399 tlb_debug("addr: "TARGET_FMT_lx" mmu_idx:%"PRIx16"\n", addr, idxmap);
400
401 /* This should already be page aligned */
402 addr_and_mmu_idx = addr & TARGET_PAGE_MASK;
403 addr_and_mmu_idx |= idxmap;
404
405 flush_all_helper(src_cpu, fn, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx));
406 async_safe_run_on_cpu(src_cpu, fn, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx));
407}
408
409void tlb_flush_page_all_cpus(CPUState *src, target_ulong addr)
410{
411 const run_on_cpu_func fn = tlb_flush_page_async_work;
412
413 flush_all_helper(src, fn, RUN_ON_CPU_TARGET_PTR(addr));
414 fn(src, RUN_ON_CPU_TARGET_PTR(addr));
415}
416
417void tlb_flush_page_all_cpus_synced(CPUState *src,
418 target_ulong addr)
419{
420 const run_on_cpu_func fn = tlb_flush_page_async_work;
421
422 flush_all_helper(src, fn, RUN_ON_CPU_TARGET_PTR(addr));
423 async_safe_run_on_cpu(src, fn, RUN_ON_CPU_TARGET_PTR(addr));
e3b9ca81
FK
424}
425
0cac1b66
BS
426/* update the TLBs so that writes to code in the virtual page 'addr'
427 can be detected */
428void tlb_protect_code(ram_addr_t ram_addr)
429{
03eebc9e
SH
430 cpu_physical_memory_test_and_clear_dirty(ram_addr, TARGET_PAGE_SIZE,
431 DIRTY_MEMORY_CODE);
0cac1b66
BS
432}
433
434/* update the TLB so that writes in physical page 'phys_addr' are no longer
435 tested for self modifying code */
9564f52d 436void tlb_unprotect_code(ram_addr_t ram_addr)
0cac1b66 437{
52159192 438 cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_CODE);
0cac1b66
BS
439}
440
0cac1b66 441
b0706b71
AB
442/*
443 * Dirty write flag handling
444 *
445 * When the TCG code writes to a location it looks up the address in
446 * the TLB and uses that data to compute the final address. If any of
447 * the lower bits of the address are set then the slow path is forced.
448 * There are a number of reasons to do this but for normal RAM the
449 * most usual is detecting writes to code regions which may invalidate
450 * generated code.
451 *
452 * Because we want other vCPUs to respond to changes straight away we
453 * update the te->addr_write field atomically. If the TLB entry has
454 * been changed by the vCPU in the mean time we skip the update.
455 *
456 * As this function uses atomic accesses we also need to ensure
457 * updates to tlb_entries follow the same access rules. We don't need
458 * to worry about this for oversized guests as MTTCG is disabled for
459 * them.
460 */
461
462static void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry, uintptr_t start,
0cac1b66
BS
463 uintptr_t length)
464{
b0706b71
AB
465#if TCG_OVERSIZED_GUEST
466 uintptr_t addr = tlb_entry->addr_write;
0cac1b66 467
b0706b71
AB
468 if ((addr & (TLB_INVALID_MASK | TLB_MMIO | TLB_NOTDIRTY)) == 0) {
469 addr &= TARGET_PAGE_MASK;
470 addr += tlb_entry->addend;
0cac1b66
BS
471 if ((addr - start) < length) {
472 tlb_entry->addr_write |= TLB_NOTDIRTY;
473 }
474 }
b0706b71
AB
475#else
476 /* paired with atomic_mb_set in tlb_set_page_with_attrs */
477 uintptr_t orig_addr = atomic_mb_read(&tlb_entry->addr_write);
478 uintptr_t addr = orig_addr;
479
480 if ((addr & (TLB_INVALID_MASK | TLB_MMIO | TLB_NOTDIRTY)) == 0) {
481 addr &= TARGET_PAGE_MASK;
482 addr += atomic_read(&tlb_entry->addend);
483 if ((addr - start) < length) {
484 uintptr_t notdirty_addr = orig_addr | TLB_NOTDIRTY;
485 atomic_cmpxchg(&tlb_entry->addr_write, orig_addr, notdirty_addr);
486 }
487 }
488#endif
489}
490
491/* For atomic correctness when running MTTCG we need to use the right
492 * primitives when copying entries */
493static inline void copy_tlb_helper(CPUTLBEntry *d, CPUTLBEntry *s,
494 bool atomic_set)
495{
496#if TCG_OVERSIZED_GUEST
497 *d = *s;
498#else
499 if (atomic_set) {
500 d->addr_read = s->addr_read;
501 d->addr_code = s->addr_code;
502 atomic_set(&d->addend, atomic_read(&s->addend));
503 /* Pairs with flag setting in tlb_reset_dirty_range */
504 atomic_mb_set(&d->addr_write, atomic_read(&s->addr_write));
505 } else {
506 d->addr_read = s->addr_read;
507 d->addr_write = atomic_read(&s->addr_write);
508 d->addr_code = s->addr_code;
509 d->addend = atomic_read(&s->addend);
510 }
511#endif
0cac1b66
BS
512}
513
b0706b71
AB
514/* This is a cross vCPU call (i.e. another vCPU resetting the flags of
515 * the target vCPU). As such care needs to be taken that we don't
516 * dangerously race with another vCPU update. The only thing actually
517 * updated is the target TLB entry ->addr_write flags.
518 */
9a13565d 519void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length)
0cac1b66
BS
520{
521 CPUArchState *env;
522
9a13565d 523 int mmu_idx;
0cac1b66 524
9a13565d
PC
525 env = cpu->env_ptr;
526 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
527 unsigned int i;
0cac1b66 528
9a13565d
PC
529 for (i = 0; i < CPU_TLB_SIZE; i++) {
530 tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
531 start1, length);
532 }
88e89a57 533
9a13565d
PC
534 for (i = 0; i < CPU_VTLB_SIZE; i++) {
535 tlb_reset_dirty_range(&env->tlb_v_table[mmu_idx][i],
536 start1, length);
0cac1b66
BS
537 }
538 }
539}
540
541static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
542{
543 if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY)) {
544 tlb_entry->addr_write = vaddr;
545 }
546}
547
548/* update the TLB corresponding to virtual page vaddr
549 so that it is no longer dirty */
bcae01e4 550void tlb_set_dirty(CPUState *cpu, target_ulong vaddr)
0cac1b66 551{
bcae01e4 552 CPUArchState *env = cpu->env_ptr;
0cac1b66
BS
553 int i;
554 int mmu_idx;
555
f0aff0f1
AB
556 assert_cpu_is_self(cpu);
557
0cac1b66
BS
558 vaddr &= TARGET_PAGE_MASK;
559 i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
560 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
561 tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
562 }
88e89a57
XT
563
564 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
565 int k;
566 for (k = 0; k < CPU_VTLB_SIZE; k++) {
567 tlb_set_dirty1(&env->tlb_v_table[mmu_idx][k], vaddr);
568 }
569 }
0cac1b66
BS
570}
571
572/* Our TLB does not support large pages, so remember the area covered by
573 large pages and trigger a full TLB flush if these are invalidated. */
574static void tlb_add_large_page(CPUArchState *env, target_ulong vaddr,
575 target_ulong size)
576{
577 target_ulong mask = ~(size - 1);
578
579 if (env->tlb_flush_addr == (target_ulong)-1) {
580 env->tlb_flush_addr = vaddr & mask;
581 env->tlb_flush_mask = mask;
582 return;
583 }
584 /* Extend the existing region to include the new page.
585 This is a compromise between unnecessary flushes and the cost
586 of maintaining a full variable size TLB. */
587 mask &= env->tlb_flush_mask;
588 while (((env->tlb_flush_addr ^ vaddr) & mask) != 0) {
589 mask <<= 1;
590 }
591 env->tlb_flush_addr &= mask;
592 env->tlb_flush_mask = mask;
593}
594
595/* Add a new TLB entry. At most one entry for a given virtual address
79e2b9ae
PB
596 * is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
597 * supplied size is only used by tlb_flush_page.
598 *
599 * Called from TCG-generated code, which is under an RCU read-side
600 * critical section.
601 */
fadc1cbe
PM
602void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
603 hwaddr paddr, MemTxAttrs attrs, int prot,
604 int mmu_idx, target_ulong size)
0cac1b66 605{
0c591eb0 606 CPUArchState *env = cpu->env_ptr;
0cac1b66
BS
607 MemoryRegionSection *section;
608 unsigned int index;
609 target_ulong address;
610 target_ulong code_address;
611 uintptr_t addend;
b0706b71 612 CPUTLBEntry *te, *tv, tn;
149f54b5 613 hwaddr iotlb, xlat, sz;
88e89a57 614 unsigned vidx = env->vtlb_index++ % CPU_VTLB_SIZE;
d7898cda 615 int asidx = cpu_asidx_from_attrs(cpu, attrs);
0cac1b66 616
f0aff0f1 617 assert_cpu_is_self(cpu);
0cac1b66
BS
618 assert(size >= TARGET_PAGE_SIZE);
619 if (size != TARGET_PAGE_SIZE) {
620 tlb_add_large_page(env, vaddr, size);
621 }
149f54b5
PB
622
623 sz = size;
d7898cda 624 section = address_space_translate_for_iotlb(cpu, asidx, paddr, &xlat, &sz);
149f54b5
PB
625 assert(sz >= TARGET_PAGE_SIZE);
626
8526e1f4
AB
627 tlb_debug("vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx
628 " prot=%x idx=%d\n",
629 vaddr, paddr, prot, mmu_idx);
0cac1b66
BS
630
631 address = vaddr;
8f3e03cb
PB
632 if (!memory_region_is_ram(section->mr) && !memory_region_is_romd(section->mr)) {
633 /* IO memory case */
0cac1b66 634 address |= TLB_MMIO;
8f3e03cb
PB
635 addend = 0;
636 } else {
637 /* TLB_MMIO for rom/romd handled below */
149f54b5 638 addend = (uintptr_t)memory_region_get_ram_ptr(section->mr) + xlat;
0cac1b66 639 }
0cac1b66
BS
640
641 code_address = address;
bb0e627a 642 iotlb = memory_region_section_get_iotlb(cpu, section, vaddr, paddr, xlat,
149f54b5 643 prot, &address);
0cac1b66
BS
644
645 index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
0cac1b66 646 te = &env->tlb_table[mmu_idx][index];
88e89a57 647 /* do not discard the translation in te, evict it into a victim tlb */
b0706b71
AB
648 tv = &env->tlb_v_table[mmu_idx][vidx];
649
650 /* addr_write can race with tlb_reset_dirty_range */
651 copy_tlb_helper(tv, te, true);
652
88e89a57
XT
653 env->iotlb_v[mmu_idx][vidx] = env->iotlb[mmu_idx][index];
654
655 /* refill the tlb */
e469b22f 656 env->iotlb[mmu_idx][index].addr = iotlb - vaddr;
fadc1cbe 657 env->iotlb[mmu_idx][index].attrs = attrs;
b0706b71
AB
658
659 /* Now calculate the new entry */
660 tn.addend = addend - vaddr;
0cac1b66 661 if (prot & PAGE_READ) {
b0706b71 662 tn.addr_read = address;
0cac1b66 663 } else {
b0706b71 664 tn.addr_read = -1;
0cac1b66
BS
665 }
666
667 if (prot & PAGE_EXEC) {
b0706b71 668 tn.addr_code = code_address;
0cac1b66 669 } else {
b0706b71 670 tn.addr_code = -1;
0cac1b66 671 }
b0706b71
AB
672
673 tn.addr_write = -1;
0cac1b66
BS
674 if (prot & PAGE_WRITE) {
675 if ((memory_region_is_ram(section->mr) && section->readonly)
cc5bea60 676 || memory_region_is_romd(section->mr)) {
0cac1b66 677 /* Write access calls the I/O callback. */
b0706b71 678 tn.addr_write = address | TLB_MMIO;
0cac1b66 679 } else if (memory_region_is_ram(section->mr)
8e41fb63
FZ
680 && cpu_physical_memory_is_clean(
681 memory_region_get_ram_addr(section->mr) + xlat)) {
b0706b71 682 tn.addr_write = address | TLB_NOTDIRTY;
0cac1b66 683 } else {
b0706b71 684 tn.addr_write = address;
0cac1b66 685 }
0cac1b66 686 }
b0706b71
AB
687
688 /* Pairs with flag setting in tlb_reset_dirty_range */
689 copy_tlb_helper(te, &tn, true);
690 /* atomic_mb_set(&te->addr_write, write_address); */
0cac1b66
BS
691}
692
fadc1cbe
PM
693/* Add a new TLB entry, but without specifying the memory
694 * transaction attributes to be used.
695 */
696void tlb_set_page(CPUState *cpu, target_ulong vaddr,
697 hwaddr paddr, int prot,
698 int mmu_idx, target_ulong size)
699{
700 tlb_set_page_with_attrs(cpu, vaddr, paddr, MEMTXATTRS_UNSPECIFIED,
701 prot, mmu_idx, size);
702}
703
d7f30403
PM
704static void report_bad_exec(CPUState *cpu, target_ulong addr)
705{
706 /* Accidentally executing outside RAM or ROM is quite common for
707 * several user-error situations, so report it in a way that
708 * makes it clear that this isn't a QEMU bug and provide suggestions
709 * about what a user could do to fix things.
710 */
711 error_report("Trying to execute code outside RAM or ROM at 0x"
712 TARGET_FMT_lx, addr);
713 error_printf("This usually means one of the following happened:\n\n"
714 "(1) You told QEMU to execute a kernel for the wrong machine "
715 "type, and it crashed on startup (eg trying to run a "
716 "raspberry pi kernel on a versatilepb QEMU machine)\n"
717 "(2) You didn't give QEMU a kernel or BIOS filename at all, "
718 "and QEMU executed a ROM full of no-op instructions until "
719 "it fell off the end\n"
720 "(3) Your guest kernel has a bug and crashed by jumping "
721 "off into nowhere\n\n"
722 "This is almost always one of the first two, so check your "
723 "command line and that you are using the right type of kernel "
724 "for this machine.\n"
725 "If you think option (3) is likely then you can try debugging "
726 "your guest with the -d debug options; in particular "
727 "-d guest_errors will cause the log to include a dump of the "
728 "guest register state at this point.\n\n"
729 "Execution cannot continue; stopping here.\n\n");
730
731 /* Report also to the logs, with more detail including register dump */
732 qemu_log_mask(LOG_GUEST_ERROR, "qemu: fatal: Trying to execute code "
733 "outside RAM or ROM at 0x" TARGET_FMT_lx "\n", addr);
734 log_cpu_state_mask(LOG_GUEST_ERROR, cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
735}
736
857baec1
AB
737static inline ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
738{
739 ram_addr_t ram_addr;
740
741 ram_addr = qemu_ram_addr_from_host(ptr);
742 if (ram_addr == RAM_ADDR_INVALID) {
743 error_report("Bad ram pointer %p", ptr);
744 abort();
745 }
746 return ram_addr;
747}
748
0cac1b66
BS
749/* NOTE: this function can trigger an exception */
750/* NOTE2: the returned address is not exactly the physical address: it
116aae36
PM
751 * is actually a ram_addr_t (in system mode; the user mode emulation
752 * version of this function returns a guest virtual address).
753 */
0cac1b66
BS
754tb_page_addr_t get_page_addr_code(CPUArchState *env1, target_ulong addr)
755{
756 int mmu_idx, page_index, pd;
757 void *p;
758 MemoryRegion *mr;
09daed84 759 CPUState *cpu = ENV_GET_CPU(env1);
a54c87b6 760 CPUIOTLBEntry *iotlbentry;
0cac1b66
BS
761
762 page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
97ed5ccd 763 mmu_idx = cpu_mmu_index(env1, true);
0cac1b66
BS
764 if (unlikely(env1->tlb_table[mmu_idx][page_index].addr_code !=
765 (addr & TARGET_PAGE_MASK))) {
0cac1b66 766 cpu_ldub_code(env1, addr);
0cac1b66 767 }
a54c87b6
PM
768 iotlbentry = &env1->iotlb[mmu_idx][page_index];
769 pd = iotlbentry->addr & ~TARGET_PAGE_MASK;
770 mr = iotlb_to_region(cpu, pd, iotlbentry->attrs);
0cac1b66 771 if (memory_region_is_unassigned(mr)) {
44d7ce0e
PM
772 cpu_unassigned_access(cpu, addr, false, true, 0, 4);
773 /* The CPU's unassigned access hook might have longjumped out
774 * with an exception. If it didn't (or there was no hook) then
775 * we can't proceed further.
776 */
777 report_bad_exec(cpu, addr);
778 exit(1);
0cac1b66
BS
779 }
780 p = (void *)((uintptr_t)addr + env1->tlb_table[mmu_idx][page_index].addend);
781 return qemu_ram_addr_from_host_nofail(p);
782}
783
82a45b96
RH
784static uint64_t io_readx(CPUArchState *env, CPUIOTLBEntry *iotlbentry,
785 target_ulong addr, uintptr_t retaddr, int size)
786{
787 CPUState *cpu = ENV_GET_CPU(env);
788 hwaddr physaddr = iotlbentry->addr;
789 MemoryRegion *mr = iotlb_to_region(cpu, physaddr, iotlbentry->attrs);
790 uint64_t val;
8d04fb55 791 bool locked = false;
82a45b96
RH
792
793 physaddr = (physaddr & TARGET_PAGE_MASK) + addr;
794 cpu->mem_io_pc = retaddr;
795 if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu->can_do_io) {
796 cpu_io_recompile(cpu, retaddr);
797 }
798
799 cpu->mem_io_vaddr = addr;
8d04fb55
JK
800
801 if (mr->global_locking) {
802 qemu_mutex_lock_iothread();
803 locked = true;
804 }
82a45b96 805 memory_region_dispatch_read(mr, physaddr, &val, size, iotlbentry->attrs);
8d04fb55
JK
806 if (locked) {
807 qemu_mutex_unlock_iothread();
808 }
809
82a45b96
RH
810 return val;
811}
812
813static void io_writex(CPUArchState *env, CPUIOTLBEntry *iotlbentry,
814 uint64_t val, target_ulong addr,
815 uintptr_t retaddr, int size)
816{
817 CPUState *cpu = ENV_GET_CPU(env);
818 hwaddr physaddr = iotlbentry->addr;
819 MemoryRegion *mr = iotlb_to_region(cpu, physaddr, iotlbentry->attrs);
8d04fb55 820 bool locked = false;
82a45b96
RH
821
822 physaddr = (physaddr & TARGET_PAGE_MASK) + addr;
823 if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu->can_do_io) {
824 cpu_io_recompile(cpu, retaddr);
825 }
82a45b96
RH
826 cpu->mem_io_vaddr = addr;
827 cpu->mem_io_pc = retaddr;
8d04fb55
JK
828
829 if (mr->global_locking) {
830 qemu_mutex_lock_iothread();
831 locked = true;
832 }
82a45b96 833 memory_region_dispatch_write(mr, physaddr, val, size, iotlbentry->attrs);
8d04fb55
JK
834 if (locked) {
835 qemu_mutex_unlock_iothread();
836 }
82a45b96
RH
837}
838
7e9a7c50
RH
839/* Return true if ADDR is present in the victim tlb, and has been copied
840 back to the main tlb. */
841static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index,
842 size_t elt_ofs, target_ulong page)
843{
844 size_t vidx;
845 for (vidx = 0; vidx < CPU_VTLB_SIZE; ++vidx) {
846 CPUTLBEntry *vtlb = &env->tlb_v_table[mmu_idx][vidx];
847 target_ulong cmp = *(target_ulong *)((uintptr_t)vtlb + elt_ofs);
848
849 if (cmp == page) {
850 /* Found entry in victim tlb, swap tlb and iotlb. */
851 CPUTLBEntry tmptlb, *tlb = &env->tlb_table[mmu_idx][index];
b0706b71
AB
852
853 copy_tlb_helper(&tmptlb, tlb, false);
854 copy_tlb_helper(tlb, vtlb, true);
855 copy_tlb_helper(vtlb, &tmptlb, true);
856
7e9a7c50
RH
857 CPUIOTLBEntry tmpio, *io = &env->iotlb[mmu_idx][index];
858 CPUIOTLBEntry *vio = &env->iotlb_v[mmu_idx][vidx];
7e9a7c50
RH
859 tmpio = *io; *io = *vio; *vio = tmpio;
860 return true;
861 }
862 }
863 return false;
864}
865
866/* Macro to call the above, with local variables from the use context. */
a390284b 867#define VICTIM_TLB_HIT(TY, ADDR) \
7e9a7c50 868 victim_tlb_hit(env, mmu_idx, index, offsetof(CPUTLBEntry, TY), \
a390284b 869 (ADDR) & TARGET_PAGE_MASK)
7e9a7c50 870
3b08f0a9
RH
871/* Probe for whether the specified guest write access is permitted.
872 * If it is not permitted then an exception will be taken in the same
873 * way as if this were a real write access (and we will not return).
874 * Otherwise the function will return, and there will be a valid
875 * entry in the TLB for this access.
876 */
877void probe_write(CPUArchState *env, target_ulong addr, int mmu_idx,
878 uintptr_t retaddr)
879{
880 int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
881 target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
882
883 if ((addr & TARGET_PAGE_MASK)
884 != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
885 /* TLB entry is for a different page */
886 if (!VICTIM_TLB_HIT(addr_write, addr)) {
887 tlb_fill(ENV_GET_CPU(env), addr, MMU_DATA_STORE, mmu_idx, retaddr);
888 }
889 }
890}
891
c482cb11
RH
892/* Probe for a read-modify-write atomic operation. Do not allow unaligned
893 * operations, or io operations to proceed. Return the host address. */
894static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
895 TCGMemOpIdx oi, uintptr_t retaddr)
896{
897 size_t mmu_idx = get_mmuidx(oi);
898 size_t index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
899 CPUTLBEntry *tlbe = &env->tlb_table[mmu_idx][index];
900 target_ulong tlb_addr = tlbe->addr_write;
901 TCGMemOp mop = get_memop(oi);
902 int a_bits = get_alignment_bits(mop);
903 int s_bits = mop & MO_SIZE;
904
905 /* Adjust the given return address. */
906 retaddr -= GETPC_ADJ;
907
908 /* Enforce guest required alignment. */
909 if (unlikely(a_bits > 0 && (addr & ((1 << a_bits) - 1)))) {
910 /* ??? Maybe indicate atomic op to cpu_unaligned_access */
911 cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,
912 mmu_idx, retaddr);
913 }
914
915 /* Enforce qemu required alignment. */
916 if (unlikely(addr & ((1 << s_bits) - 1))) {
917 /* We get here if guest alignment was not requested,
918 or was not enforced by cpu_unaligned_access above.
919 We might widen the access and emulate, but for now
920 mark an exception and exit the cpu loop. */
921 goto stop_the_world;
922 }
923
924 /* Check TLB entry and enforce page permissions. */
925 if ((addr & TARGET_PAGE_MASK)
926 != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
927 if (!VICTIM_TLB_HIT(addr_write, addr)) {
928 tlb_fill(ENV_GET_CPU(env), addr, MMU_DATA_STORE, mmu_idx, retaddr);
929 }
930 tlb_addr = tlbe->addr_write;
931 }
932
7f9af1ab
ND
933 /* Check notdirty */
934 if (unlikely(tlb_addr & TLB_NOTDIRTY)) {
935 tlb_set_dirty(ENV_GET_CPU(env), addr);
936 tlb_addr = tlb_addr & ~TLB_NOTDIRTY;
937 }
938
939 /* Notice an IO access */
c482cb11
RH
940 if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
941 /* There's really nothing that can be done to
942 support this apart from stop-the-world. */
943 goto stop_the_world;
944 }
945
946 /* Let the guest notice RMW on a write-only page. */
947 if (unlikely(tlbe->addr_read != tlb_addr)) {
948 tlb_fill(ENV_GET_CPU(env), addr, MMU_DATA_LOAD, mmu_idx, retaddr);
949 /* Since we don't support reads and writes to different addresses,
950 and we do have the proper page loaded for write, this shouldn't
951 ever return. But just in case, handle via stop-the-world. */
952 goto stop_the_world;
953 }
954
955 return (void *)((uintptr_t)addr + tlbe->addend);
956
957 stop_the_world:
958 cpu_loop_exit_atomic(ENV_GET_CPU(env), retaddr);
959}
960
c86c6e4c
RH
961#ifdef TARGET_WORDS_BIGENDIAN
962# define TGT_BE(X) (X)
963# define TGT_LE(X) BSWAP(X)
964#else
965# define TGT_BE(X) BSWAP(X)
966# define TGT_LE(X) (X)
967#endif
968
0f590e74
PB
969#define MMUSUFFIX _mmu
970
dea21982 971#define DATA_SIZE 1
58ed270d 972#include "softmmu_template.h"
0f590e74 973
dea21982 974#define DATA_SIZE 2
58ed270d 975#include "softmmu_template.h"
0f590e74 976
dea21982 977#define DATA_SIZE 4
58ed270d 978#include "softmmu_template.h"
0f590e74 979
dea21982 980#define DATA_SIZE 8
58ed270d 981#include "softmmu_template.h"
0f590e74 982
c482cb11
RH
983/* First set of helpers allows passing in of OI and RETADDR. This makes
984 them callable from other helpers. */
985
986#define EXTRA_ARGS , TCGMemOpIdx oi, uintptr_t retaddr
987#define ATOMIC_NAME(X) \
988 HELPER(glue(glue(glue(atomic_ ## X, SUFFIX), END), _mmu))
989#define ATOMIC_MMU_LOOKUP atomic_mmu_lookup(env, addr, oi, retaddr)
990
991#define DATA_SIZE 1
992#include "atomic_template.h"
993
994#define DATA_SIZE 2
995#include "atomic_template.h"
996
997#define DATA_SIZE 4
998#include "atomic_template.h"
999
df79b996 1000#ifdef CONFIG_ATOMIC64
c482cb11
RH
1001#define DATA_SIZE 8
1002#include "atomic_template.h"
df79b996 1003#endif
c482cb11 1004
7ebee43e
RH
1005#ifdef CONFIG_ATOMIC128
1006#define DATA_SIZE 16
1007#include "atomic_template.h"
1008#endif
1009
c482cb11
RH
1010/* Second set of helpers are directly callable from TCG as helpers. */
1011
1012#undef EXTRA_ARGS
1013#undef ATOMIC_NAME
1014#undef ATOMIC_MMU_LOOKUP
1015#define EXTRA_ARGS , TCGMemOpIdx oi
1016#define ATOMIC_NAME(X) HELPER(glue(glue(atomic_ ## X, SUFFIX), END))
1017#define ATOMIC_MMU_LOOKUP atomic_mmu_lookup(env, addr, oi, GETPC())
1018
1019#define DATA_SIZE 1
1020#include "atomic_template.h"
1021
1022#define DATA_SIZE 2
1023#include "atomic_template.h"
1024
1025#define DATA_SIZE 4
1026#include "atomic_template.h"
1027
df79b996 1028#ifdef CONFIG_ATOMIC64
c482cb11
RH
1029#define DATA_SIZE 8
1030#include "atomic_template.h"
df79b996 1031#endif
c482cb11
RH
1032
1033/* Code access functions. */
1034
1035#undef MMUSUFFIX
0cac1b66 1036#define MMUSUFFIX _cmmu
01ecaf43
RH
1037#undef GETPC
1038#define GETPC() ((uintptr_t)0)
0cac1b66
BS
1039#define SOFTMMU_CODE_ACCESS
1040
dea21982 1041#define DATA_SIZE 1
58ed270d 1042#include "softmmu_template.h"
0cac1b66 1043
dea21982 1044#define DATA_SIZE 2
58ed270d 1045#include "softmmu_template.h"
0cac1b66 1046
dea21982 1047#define DATA_SIZE 4
58ed270d 1048#include "softmmu_template.h"
0cac1b66 1049
dea21982 1050#define DATA_SIZE 8
58ed270d 1051#include "softmmu_template.h"