[mirror_qemu.git] / util / cacheflush.c

/*
 * Info about, and flushing the host cpu caches.
 *
 * This work is licensed under the terms of the GNU GPL, version 2 or later.
 * See the COPYING file in the top-level directory.
 */

#include "qemu/osdep.h"
#include "qemu/cacheflush.h"
#include "qemu/cacheinfo.h"
#include "qemu/bitops.h"
#include "qemu/host-utils.h"
#include "qemu/atomic.h"


int qemu_icache_linesize = 0;
int qemu_icache_linesize_log;
int qemu_dcache_linesize = 0;
int qemu_dcache_linesize_log;

/*
 * Operating system specific cache detection mechanisms.
 */

#if defined(_WIN32)

static void sys_cache_info(int *isize, int *dsize)
{
    SYSTEM_LOGICAL_PROCESSOR_INFORMATION *buf;
    DWORD size = 0;
    BOOL success;
    size_t i, n;

    /*
     * Check for the required buffer size first.  Note that if the zero
     * size we use for the probe results in success, then there is no
     * data available; fail in that case.
     */
    success = GetLogicalProcessorInformation(0, &size);
    if (success || GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
        return;
    }

    n = size / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION);
    size = n * sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION);
    buf = g_new0(SYSTEM_LOGICAL_PROCESSOR_INFORMATION, n);
    if (!GetLogicalProcessorInformation(buf, &size)) {
        goto fail;
    }

    for (i = 0; i < n; i++) {
        if (buf[i].Relationship == RelationCache
            && buf[i].Cache.Level == 1) {
            switch (buf[i].Cache.Type) {
            case CacheUnified:
                *isize = *dsize = buf[i].Cache.LineSize;
                break;
            case CacheInstruction:
                *isize = buf[i].Cache.LineSize;
                break;
            case CacheData:
                *dsize = buf[i].Cache.LineSize;
                break;
            default:
                break;
            }
        }
    }
 fail:
    g_free(buf);
}

#elif defined(CONFIG_DARWIN)
# include <sys/sysctl.h>
static void sys_cache_info(int *isize, int *dsize)
{
    /* There's only a single sysctl for both I/D cache line sizes.  */
    long size;
    size_t len = sizeof(size);
    if (!sysctlbyname("hw.cachelinesize", &size, &len, NULL, 0)) {
        *isize = *dsize = size;
    }
}
#elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
# include <sys/sysctl.h>
static void sys_cache_info(int *isize, int *dsize)
{
    /* There's only a single sysctl for both I/D cache line sizes.  */
    int size;
    size_t len = sizeof(size);
    if (!sysctlbyname("machdep.cacheline_size", &size, &len, NULL, 0)) {
        *isize = *dsize = size;
    }
}
#else
/* POSIX */

static void sys_cache_info(int *isize, int *dsize)
{
# ifdef _SC_LEVEL1_ICACHE_LINESIZE
    int tmp_isize = (int) sysconf(_SC_LEVEL1_ICACHE_LINESIZE);
    if (tmp_isize > 0) {
        *isize = tmp_isize;
    }
# endif
# ifdef _SC_LEVEL1_DCACHE_LINESIZE
    int tmp_dsize = (int) sysconf(_SC_LEVEL1_DCACHE_LINESIZE);
    if (tmp_dsize > 0) {
        *dsize = tmp_dsize;
    }
# endif
}
#endif /* sys_cache_info */


/*
 * Architecture (+ OS) specific cache detection mechanisms.
 */

#if defined(__powerpc__)
static bool have_coherent_icache;
#endif

#if defined(__aarch64__) && !defined(CONFIG_DARWIN) && !defined(CONFIG_WIN32)
/*
 * Apple does not expose CTR_EL0, so we must use system interfaces.
 * Windows neither, but we use a generic implementation of flush_idcache_range
 * in this case.
 */
static uint64_t save_ctr_el0;
static void arch_cache_info(int *isize, int *dsize)
{
    uint64_t ctr;

    /*
     * The real cache geometry is in CCSIDR_EL1/CLIDR_EL1/CSSELR_EL1,
     * but (at least under Linux) these are marked protected by the
     * kernel.  However, CTR_EL0 contains the minimum linesize in the
     * entire hierarchy, and is used by userspace cache flushing.
     *
     * We will also use this value in flush_idcache_range.
     */
    asm volatile("mrs\t%0, ctr_el0" : "=r"(ctr));
    save_ctr_el0 = ctr;

    if (*isize == 0 || *dsize == 0) {
        if (*isize == 0) {
            *isize = 4 << (ctr & 0xf);
        }
        if (*dsize == 0) {
            *dsize = 4 << ((ctr >> 16) & 0xf);
        }
    }
}

#elif defined(_ARCH_PPC) && defined(__linux__)
# include "elf.h"

static void arch_cache_info(int *isize, int *dsize)
{
    if (*isize == 0) {
        *isize = qemu_getauxval(AT_ICACHEBSIZE);
    }
    if (*dsize == 0) {
        *dsize = qemu_getauxval(AT_DCACHEBSIZE);
    }
    have_coherent_icache = qemu_getauxval(AT_HWCAP) & PPC_FEATURE_ICACHE_SNOOP;
}

#else
static void arch_cache_info(int *isize, int *dsize) { }
#endif /* arch_cache_info */

/*
 * ... and if all else fails ...
 */

static void fallback_cache_info(int *isize, int *dsize)
{
    /* If we can only find one of the two, assume they're the same.  */
    if (*isize) {
        if (*dsize) {
            /* Success! */
        } else {
            *dsize = *isize;
        }
    } else if (*dsize) {
        *isize = *dsize;
    } else {
#if defined(_ARCH_PPC)
        /*
         * For PPC, we're going to use the cache sizes computed for
         * flush_idcache_range.  Which means that we must use the
         * architecture minimum.
         */
        *isize = *dsize = 16;
#else
        /* Otherwise, 64 bytes is not uncommon.  */
        *isize = *dsize = 64;
#endif
    }
}

static void __attribute__((constructor)) init_cache_info(void)
{
    int isize = 0, dsize = 0;

    sys_cache_info(&isize, &dsize);
    arch_cache_info(&isize, &dsize);
    fallback_cache_info(&isize, &dsize);

    assert((isize & (isize - 1)) == 0);
    assert((dsize & (dsize - 1)) == 0);

    qemu_icache_linesize = isize;
    qemu_icache_linesize_log = ctz32(isize);
    qemu_dcache_linesize = dsize;
    qemu_dcache_linesize_log = ctz32(dsize);

    qatomic64_init();
}


/*
 * Architecture (+ OS) specific cache flushing mechanisms.
 */

#if defined(__i386__) || defined(__x86_64__) || defined(__s390__)

/* Caches are coherent and do not require flushing; symbol inline. */

#elif defined(__aarch64__) && !defined(CONFIG_WIN32)
/*
 * For Windows, we use generic implementation of flush_idcache_range, that
 * performs a call to FlushInstructionCache, through __builtin___clear_cache.
 */

#ifdef CONFIG_DARWIN
/* Apple does not expose CTR_EL0, so we must use system interfaces. */
#include <libkern/OSCacheControl.h>

void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
{
    if (rx == rw) {
        /*
         * sys_icache_invalidate() syncs the dcache and icache,
         * so no need to call sys_dcache_flush().
         */
    } else {
        sys_dcache_flush((void *)rw, len);
    }
    sys_icache_invalidate((void *)rx, len);
}
#else

/*
 * This is a copy of gcc's __aarch64_sync_cache_range, modified
 * to fit this three-operand interface.
 */
void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
{
    const unsigned CTR_IDC = 1u << 28;
    const unsigned CTR_DIC = 1u << 29;
    const uint64_t ctr_el0 = save_ctr_el0;
    const uintptr_t icache_lsize = qemu_icache_linesize;
    const uintptr_t dcache_lsize = qemu_dcache_linesize;
    uintptr_t p;

    /*
     * If CTR_EL0.IDC is enabled, Data cache clean to the Point of Unification
     * is not required for instruction to data coherence.
     */
    if (!(ctr_el0 & CTR_IDC)) {
        /*
         * Loop over the address range, clearing one cache line at once.
         * Data cache must be flushed to unification first to make sure
         * the instruction cache fetches the updated data.
         */
        for (p = rw & -dcache_lsize; p < rw + len; p += dcache_lsize) {
            asm volatile("dc\tcvau, %0" : : "r" (p) : "memory");
        }
        asm volatile("dsb\tish" : : : "memory");
    }

    /*
     * If CTR_EL0.DIC is enabled, Instruction cache cleaning to the Point
     * of Unification is not required for instruction to data coherence.
     */
    if (!(ctr_el0 & CTR_DIC)) {
        for (p = rx & -icache_lsize; p < rx + len; p += icache_lsize) {
            asm volatile("ic\tivau, %0" : : "r"(p) : "memory");
        }
        asm volatile ("dsb\tish" : : : "memory");
    }

    asm volatile("isb" : : : "memory");
}
#endif /* CONFIG_DARWIN */

#elif defined(__mips__)

#ifdef __OpenBSD__
#include <machine/sysarch.h>
#else
#include <sys/cachectl.h>
#endif

void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
{
    if (rx != rw) {
        cacheflush((void *)rw, len, DCACHE);
    }
    cacheflush((void *)rx, len, ICACHE);
}

#elif defined(__powerpc__)

void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
{
    uintptr_t p, b, e;
    size_t dsize, isize;

    /*
     * Some processors have coherent caches and support a simplified
     * flushing procedure.  See
     *   POWER9 UM, 4.6.2.2 Instruction Cache Block Invalidate (icbi) 
     *   https://ibm.ent.box.com/s/tmklq90ze7aj8f4n32er1mu3sy9u8k3k
     */
    if (have_coherent_icache) {
        asm volatile ("sync\n\t"
                      "icbi 0,%0\n\t"
                      "isync"
                      : : "r"(rx) : "memory");
        return;
    }

    dsize = qemu_dcache_linesize;
    isize = qemu_icache_linesize;

    b = rw & ~(dsize - 1);
    e = (rw + len + dsize - 1) & ~(dsize - 1);
    for (p = b; p < e; p += dsize) {
        asm volatile ("dcbst 0,%0" : : "r"(p) : "memory");
    }
    asm volatile ("sync" : : : "memory");

    b = rx & ~(isize - 1);
    e = (rx + len + isize - 1) & ~(isize - 1);
    for (p = b; p < e; p += isize) {
        asm volatile ("icbi 0,%0" : : "r"(p) : "memory");
    }
    asm volatile ("sync" : : : "memory");
    asm volatile ("isync" : : : "memory");
}

#elif defined(__sparc__)

void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
{
    /* No additional data flush to the RW virtual address required. */
    uintptr_t p, end = (rx + len + 7) & -8;
    for (p = rx & -8; p < end; p += 8) {
        __asm__ __volatile__("flush\t%0" : : "r" (p));
    }
}

#else

void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
{
    if (rw != rx) {
        __builtin___clear_cache((char *)rw, (char *)rw + len);
    }
    __builtin___clear_cache((char *)rx, (char *)rx + len);
}

#endif
Commit	Line	Data
084cfca1	1	/*
79713752	2	* Info about, and flushing the host cpu caches.
084cfca1 RH	3	*
	4	* This work is licensed under the terms of the GNU GPL, version 2 or later.
	5	* See the COPYING file in the top-level directory.
	6	*/
	7
	8	#include "qemu/osdep.h"
	9	#include "qemu/cacheflush.h"
ad768e6f	10	#include "qemu/cacheinfo.h"
664a7973	11	#include "qemu/bitops.h"
79713752 RH	12	#include "qemu/host-utils.h"
79713752 RH	13	#include "qemu/atomic.h"
084cfca1 RH	14
084cfca1 RH	15
79713752 RH	16	int qemu_icache_linesize = 0;
	17	int qemu_icache_linesize_log;
	18	int qemu_dcache_linesize = 0;
	19	int qemu_dcache_linesize_log;
	20
	21	/*
	22	* Operating system specific cache detection mechanisms.
	23	*/
	24
	25	#if defined(_WIN32)
	26
	27	static void sys_cache_info(int isize, int dsize)
	28	{
	29	SYSTEM_LOGICAL_PROCESSOR_INFORMATION *buf;
	30	DWORD size = 0;
	31	BOOL success;
	32	size_t i, n;
	33
	34	/*
	35	* Check for the required buffer size first. Note that if the zero
	36	* size we use for the probe results in success, then there is no
	37	* data available; fail in that case.
	38	*/
	39	success = GetLogicalProcessorInformation(0, &size);
	40	if (success \|\| GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
	41	return;
	42	}
	43
	44	n = size / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION);
	45	size = n * sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION);
	46	buf = g_new0(SYSTEM_LOGICAL_PROCESSOR_INFORMATION, n);
	47	if (!GetLogicalProcessorInformation(buf, &size)) {
	48	goto fail;
	49	}
	50
	51	for (i = 0; i < n; i++) {
	52	if (buf[i].Relationship == RelationCache
	53	&& buf[i].Cache.Level == 1) {
	54	switch (buf[i].Cache.Type) {
	55	case CacheUnified:
	56	isize = dsize = buf[i].Cache.LineSize;
	57	break;
	58	case CacheInstruction:
	59	*isize = buf[i].Cache.LineSize;
	60	break;
	61	case CacheData:
	62	*dsize = buf[i].Cache.LineSize;
	63	break;
	64	default:
	65	break;
	66	}
	67	}
	68	}
	69	fail:
	70	g_free(buf);
	71	}
	72
bdd50dc7	73	#elif defined(CONFIG_DARWIN)
79713752 RH	74	# include <sys/sysctl.h>
	75	static void sys_cache_info(int isize, int dsize)
	76	{
	77	/* There's only a single sysctl for both I/D cache line sizes. */
	78	long size;
	79	size_t len = sizeof(size);
	80	if (!sysctlbyname("hw.cachelinesize", &size, &len, NULL, 0)) {
	81	isize = dsize = size;
	82	}
	83	}
	84	#elif defined(__FreeBSD__) \|\| defined(__FreeBSD_kernel__)
	85	# include <sys/sysctl.h>
	86	static void sys_cache_info(int isize, int dsize)
	87	{
	88	/* There's only a single sysctl for both I/D cache line sizes. */
	89	int size;
	90	size_t len = sizeof(size);
	91	if (!sysctlbyname("machdep.cacheline_size", &size, &len, NULL, 0)) {
	92	isize = dsize = size;
	93	}
	94	}
	95	#else
	96	/* POSIX */
	97
	98	static void sys_cache_info(int isize, int dsize)
	99	{
	100	# ifdef _SC_LEVEL1_ICACHE_LINESIZE
	101	int tmp_isize = (int) sysconf(_SC_LEVEL1_ICACHE_LINESIZE);
	102	if (tmp_isize > 0) {
	103	*isize = tmp_isize;
	104	}
	105	# endif
	106	# ifdef _SC_LEVEL1_DCACHE_LINESIZE
	107	int tmp_dsize = (int) sysconf(_SC_LEVEL1_DCACHE_LINESIZE);
	108	if (tmp_dsize > 0) {
	109	*dsize = tmp_dsize;
	110	}
	111	# endif
	112	}
	113	#endif /* sys_cache_info */
	114
	115
	116	/*
	117	* Architecture (+ OS) specific cache detection mechanisms.
	118	*/
	119
c79a8e84 NP	120	#if defined(__powerpc__)
	121	static bool have_coherent_icache;
	122	#endif
	123
b3c32602 PB	124	#if defined(__aarch64__) && !defined(CONFIG_DARWIN) && !defined(CONFIG_WIN32)
	125	/*
	126	* Apple does not expose CTR_EL0, so we must use system interfaces.
	127	* Windows neither, but we use a generic implementation of flush_idcache_range
	128	* in this case.
	129	*/
bdd50dc7	130	static uint64_t save_ctr_el0;
79713752 RH	131	static void arch_cache_info(int isize, int dsize)
79713752 RH	132	{
bdd50dc7	133	uint64_t ctr;
79713752	134
bdd50dc7 RH	135	/*
	136	* The real cache geometry is in CCSIDR_EL1/CLIDR_EL1/CSSELR_EL1,
	137	* but (at least under Linux) these are marked protected by the
	138	* kernel. However, CTR_EL0 contains the minimum linesize in the
	139	* entire hierarchy, and is used by userspace cache flushing.
	140	*
	141	* We will also use this value in flush_idcache_range.
	142	*/
	143	asm volatile("mrs\t%0, ctr_el0" : "=r"(ctr));
	144	save_ctr_el0 = ctr;
	145
	146	if (isize == 0 \|\| dsize == 0) {
79713752 RH	147	if (*isize == 0) {
	148	*isize = 4 << (ctr & 0xf);
	149	}
	150	if (*dsize == 0) {
	151	*dsize = 4 << ((ctr >> 16) & 0xf);
	152	}
	153	}
	154	}
	155
	156	#elif defined(_ARCH_PPC) && defined(__linux__)
	157	# include "elf.h"
	158
	159	static void arch_cache_info(int isize, int dsize)
	160	{
	161	if (*isize == 0) {
	162	*isize = qemu_getauxval(AT_ICACHEBSIZE);
	163	}
	164	if (*dsize == 0) {
	165	*dsize = qemu_getauxval(AT_DCACHEBSIZE);
	166	}
c79a8e84	167	have_coherent_icache = qemu_getauxval(AT_HWCAP) & PPC_FEATURE_ICACHE_SNOOP;
79713752 RH	168	}
	169
	170	#else
	171	static void arch_cache_info(int isize, int dsize) { }
	172	#endif /* arch_cache_info */
	173
	174	/*
	175	* ... and if all else fails ...
	176	*/
	177
	178	static void fallback_cache_info(int isize, int dsize)
	179	{
	180	/* If we can only find one of the two, assume they're the same. */
	181	if (*isize) {
	182	if (*dsize) {
	183	/* Success! */
	184	} else {
	185	dsize = isize;
	186	}
	187	} else if (*dsize) {
	188	isize = dsize;
	189	} else {
	190	#if defined(_ARCH_PPC)
	191	/*
	192	* For PPC, we're going to use the cache sizes computed for
	193	* flush_idcache_range. Which means that we must use the
	194	* architecture minimum.
	195	*/
	196	isize = dsize = 16;
	197	#else
	198	/* Otherwise, 64 bytes is not uncommon. */
	199	isize = dsize = 64;
	200	#endif
	201	}
	202	}
	203
	204	static void __attribute__((constructor)) init_cache_info(void)
	205	{
	206	int isize = 0, dsize = 0;
	207
	208	sys_cache_info(&isize, &dsize);
	209	arch_cache_info(&isize, &dsize);
	210	fallback_cache_info(&isize, &dsize);
	211
	212	assert((isize & (isize - 1)) == 0);
	213	assert((dsize & (dsize - 1)) == 0);
	214
	215	qemu_icache_linesize = isize;
	216	qemu_icache_linesize_log = ctz32(isize);
	217	qemu_dcache_linesize = dsize;
	218	qemu_dcache_linesize_log = ctz32(dsize);
	219
	220	qatomic64_init();
	221	}
	222
	223
	224	/*
	225	* Architecture (+ OS) specific cache flushing mechanisms.
	226	*/
	227
084cfca1 RH	228	#if defined(__i386__) \|\| defined(__x86_64__) \|\| defined(__s390__)
	229
	230	/* Caches are coherent and do not require flushing; symbol inline. */
	231
b3c32602 PB	232	#elif defined(__aarch64__) && !defined(CONFIG_WIN32)
	233	/*
	234	* For Windows, we use generic implementation of flush_idcache_range, that
	235	* performs a call to FlushInstructionCache, through __builtin___clear_cache.
	236	*/
664a7973 RH	237
	238	#ifdef CONFIG_DARWIN
	239	/* Apple does not expose CTR_EL0, so we must use system interfaces. */
0baf54d0 PMD	240	#include <libkern/OSCacheControl.h>
0baf54d0 PMD	241
664a7973 RH	242	void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
664a7973 RH	243	{
bb6af0fa PMD	244	if (rx == rw) {
	245	/*
	246	* sys_icache_invalidate() syncs the dcache and icache,
	247	* so no need to call sys_dcache_flush().
	248	*/
	249	} else {
	250	sys_dcache_flush((void *)rw, len);
	251	}
664a7973 RH	252	sys_icache_invalidate((void *)rx, len);
	253	}
	254	#else
	255
664a7973 RH	256	/*
	257	* This is a copy of gcc's __aarch64_sync_cache_range, modified
	258	* to fit this three-operand interface.
	259	*/
	260	void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
	261	{
	262	const unsigned CTR_IDC = 1u << 28;
	263	const unsigned CTR_DIC = 1u << 29;
acd15fc2	264	const uint64_t ctr_el0 = save_ctr_el0;
bdd50dc7 RH	265	const uintptr_t icache_lsize = qemu_icache_linesize;
bdd50dc7 RH	266	const uintptr_t dcache_lsize = qemu_dcache_linesize;
664a7973 RH	267	uintptr_t p;
	268
	269	/*
	270	* If CTR_EL0.IDC is enabled, Data cache clean to the Point of Unification
	271	* is not required for instruction to data coherence.
	272	*/
	273	if (!(ctr_el0 & CTR_IDC)) {
	274	/*
	275	* Loop over the address range, clearing one cache line at once.
	276	* Data cache must be flushed to unification first to make sure
	277	* the instruction cache fetches the updated data.
	278	*/
	279	for (p = rw & -dcache_lsize; p < rw + len; p += dcache_lsize) {
	280	asm volatile("dc\tcvau, %0" : : "r" (p) : "memory");
	281	}
	282	asm volatile("dsb\tish" : : : "memory");
	283	}
	284
	285	/*
	286	* If CTR_EL0.DIC is enabled, Instruction cache cleaning to the Point
	287	* of Unification is not required for instruction to data coherence.
	288	*/
	289	if (!(ctr_el0 & CTR_DIC)) {
	290	for (p = rx & -icache_lsize; p < rx + len; p += icache_lsize) {
	291	asm volatile("ic\tivau, %0" : : "r"(p) : "memory");
	292	}
	293	asm volatile ("dsb\tish" : : : "memory");
	294	}
	295
	296	asm volatile("isb" : : : "memory");
	297	}
	298	#endif /* CONFIG_DARWIN */
	299
084cfca1 RH	300	#elif defined(__mips__)
	301
	302	#ifdef __OpenBSD__
	303	#include <machine/sysarch.h>
	304	#else
	305	#include <sys/cachectl.h>
	306	#endif
	307
1da8de39	308	void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
084cfca1	309	{
1da8de39 RH	310	if (rx != rw) {
	311	cacheflush((void *)rw, len, DCACHE);
	312	}
	313	cacheflush((void *)rx, len, ICACHE);
084cfca1 RH	314	}
	315
	316	#elif defined(__powerpc__)
	317
1da8de39	318	void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
084cfca1	319	{
1da8de39	320	uintptr_t p, b, e;
c79a8e84 NP	321	size_t dsize, isize;
	322
	323	/*
	324	* Some processors have coherent caches and support a simplified
	325	* flushing procedure. See
	326	* POWER9 UM, 4.6.2.2 Instruction Cache Block Invalidate (icbi)
	327	* https://ibm.ent.box.com/s/tmklq90ze7aj8f4n32er1mu3sy9u8k3k
	328	*/
	329	if (have_coherent_icache) {
	330	asm volatile ("sync\n\t"
	331	"icbi 0,%0\n\t"
	332	"isync"
	333	: : "r"(rx) : "memory");
	334	return;
	335	}
	336
	337	dsize = qemu_dcache_linesize;
	338	isize = qemu_icache_linesize;
084cfca1	339
1da8de39 RH	340	b = rw & ~(dsize - 1);
	341	e = (rw + len + dsize - 1) & ~(dsize - 1);
	342	for (p = b; p < e; p += dsize) {
084cfca1 RH	343	asm volatile ("dcbst 0,%0" : : "r"(p) : "memory");
	344	}
	345	asm volatile ("sync" : : : "memory");
	346
1da8de39 RH	347	b = rx & ~(isize - 1);
	348	e = (rx + len + isize - 1) & ~(isize - 1);
	349	for (p = b; p < e; p += isize) {
084cfca1 RH	350	asm volatile ("icbi 0,%0" : : "r"(p) : "memory");
	351	}
	352	asm volatile ("sync" : : : "memory");
	353	asm volatile ("isync" : : : "memory");
	354	}
	355
	356	#elif defined(__sparc__)
	357
1da8de39	358	void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
084cfca1	359	{
1da8de39 RH	360	/* No additional data flush to the RW virtual address required. */
	361	uintptr_t p, end = (rx + len + 7) & -8;
	362	for (p = rx & -8; p < end; p += 8) {
084cfca1 RH	363	__asm__ __volatile__("flush\t%0" : : "r" (p));
	364	}
	365	}
	366
	367	#else
	368
1da8de39	369	void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
084cfca1	370	{
1da8de39 RH	371	if (rw != rx) {
	372	__builtin___clear_cache((char )rw, (char )rw + len);
	373	}
	374	__builtin___clear_cache((char )rx, (char )rx + len);
084cfca1 RH	375	}
	376
	377	#endif