image: docker:stable
stage: containers
services:
- - docker:dind
+ - docker:stable-dind
before_script:
- export TAG="$CI_REGISTRY_IMAGE/qemu/$NAME:latest"
- export COMMON_TAG="$CI_REGISTRY/qemu-project/qemu/qemu/$NAME:latest"
- apk add python3
- - docker info
- docker login $CI_REGISTRY -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD"
+ - until docker info; do sleep 1; done
script:
- echo "TAG:$TAG"
- echo "COMMON_TAG:$COMMON_TAG"
variables:
GIT_DEPTH: 3
IMAGE_TAG: $CI_REGISTRY_IMAGE:opensbi-cross-build
- # We don't use TLS
- DOCKER_HOST: tcp://docker:2375
- DOCKER_TLS_CERTDIR: ""
before_script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+ - until docker info; do sleep 1; done
script:
- docker pull $IMAGE_TAG || true
- docker build --cache-from $IMAGE_TAG --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
# This file contains the set of jobs run by the QEMU project:
# https://gitlab.com/qemu-project/qemu/-/pipelines
+variables:
+ RUNNER_TAG: ""
+
+default:
+ tags:
+ - $RUNNER_TAG
+
include:
- local: '/.gitlab-ci.d/base.yml'
- local: '/.gitlab-ci.d/stages.yml'
F: include/sysemu/cpus.h
F: include/sysemu/tcg.h
F: include/hw/core/tcg-cpu-ops.h
+F: host/include/*/host/cpuinfo.h
+F: util/cpuinfo-*.c
+F: include/tcg/
FPU emulation
M: Aurelien Jarno <aurelien@aurel32.net>
Network packet abstractions
M: Dmitry Fleytman <dmitry.fleytman@gmail.com>
+R: Akihiko Odaki <akihiko.odaki@daynix.com>
S: Maintained
F: include/net/eth.h
F: net/eth.c
S: Maintained
F: docs/system/devices/igb.rst
F: hw/net/igb*
-F: tests/avocado/igb.py
+F: tests/avocado/netdev-ethtool.py
F: tests/qtest/igb-test.c
F: tests/qtest/libqos/igb.c
qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_RW);
}
-#if HAVE_ATOMIC128
-static void atomic_trace_ld_post(CPUArchState *env, uint64_t addr,
- MemOpIdx oi)
-{
- qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R);
-}
-
-static void atomic_trace_st_post(CPUArchState *env, uint64_t addr,
- MemOpIdx oi)
-{
- qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W);
-}
-#endif
-
/*
* Atomic helpers callable from TCG.
* These have a common interface and all defer to cpu_atomic_*
#undef CMPXCHG_HELPER
-Int128 HELPER(nonatomic_cmpxchgo_be)(CPUArchState *env, uint64_t addr,
- Int128 cmpv, Int128 newv, uint32_t oi)
-{
-#if TCG_TARGET_REG_BITS == 32
- uintptr_t ra = GETPC();
- Int128 oldv;
-
- oldv = cpu_ld16_be_mmu(env, addr, oi, ra);
- if (int128_eq(oldv, cmpv)) {
- cpu_st16_be_mmu(env, addr, newv, oi, ra);
- } else {
- /* Even with comparison failure, still need a write cycle. */
- probe_write(env, addr, 16, get_mmuidx(oi), ra);
- }
- return oldv;
-#else
- g_assert_not_reached();
-#endif
-}
-
-Int128 HELPER(nonatomic_cmpxchgo_le)(CPUArchState *env, uint64_t addr,
- Int128 cmpv, Int128 newv, uint32_t oi)
+Int128 HELPER(nonatomic_cmpxchgo)(CPUArchState *env, uint64_t addr,
+ Int128 cmpv, Int128 newv, uint32_t oi)
{
#if TCG_TARGET_REG_BITS == 32
uintptr_t ra = GETPC();
Int128 oldv;
- oldv = cpu_ld16_le_mmu(env, addr, oi, ra);
+ oldv = cpu_ld16_mmu(env, addr, oi, ra);
if (int128_eq(oldv, cmpv)) {
- cpu_st16_le_mmu(env, addr, newv, oi, ra);
+ cpu_st16_mmu(env, addr, newv, oi, ra);
} else {
/* Even with comparison failure, still need a write cycle. */
probe_write(env, addr, 16, get_mmuidx(oi), ra);
ABI_TYPE cmpv, ABI_TYPE newv,
MemOpIdx oi, uintptr_t retaddr)
{
- DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
- PAGE_READ | PAGE_WRITE, retaddr);
+ DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, retaddr);
DATA_TYPE ret;
#if DATA_SIZE == 16
return ret;
}
-#if DATA_SIZE >= 16
-#if HAVE_ATOMIC128
-ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, uintptr_t retaddr)
-{
- DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
- PAGE_READ, retaddr);
- DATA_TYPE val;
-
- val = atomic16_read(haddr);
- ATOMIC_MMU_CLEANUP;
- atomic_trace_ld_post(env, addr, oi);
- return val;
-}
-
-void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ABI_TYPE val,
- MemOpIdx oi, uintptr_t retaddr)
-{
- DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
- PAGE_WRITE, retaddr);
-
- atomic16_set(haddr, val);
- ATOMIC_MMU_CLEANUP;
- atomic_trace_st_post(env, addr, oi);
-}
-#endif
-#else
+#if DATA_SIZE < 16
ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val,
MemOpIdx oi, uintptr_t retaddr)
{
- DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
- PAGE_READ | PAGE_WRITE, retaddr);
+ DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, retaddr);
DATA_TYPE ret;
ret = qatomic_xchg__nocheck(haddr, val);
ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \
{ \
- DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \
- PAGE_READ | PAGE_WRITE, retaddr); \
- DATA_TYPE ret; \
+ DATA_TYPE *haddr, ret; \
+ haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, retaddr); \
ret = qatomic_##X(haddr, val); \
ATOMIC_MMU_CLEANUP; \
atomic_trace_rmw_post(env, addr, oi); \
ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \
{ \
- XDATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \
- PAGE_READ | PAGE_WRITE, retaddr); \
- XDATA_TYPE cmp, old, new, val = xval; \
+ XDATA_TYPE *haddr, cmp, old, new, val = xval; \
+ haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, retaddr); \
smp_mb(); \
cmp = qatomic_read__nocheck(haddr); \
do { \
GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new)
#undef GEN_ATOMIC_HELPER_FN
-#endif /* DATA SIZE >= 16 */
+#endif /* DATA SIZE < 16 */
#undef END
ABI_TYPE cmpv, ABI_TYPE newv,
MemOpIdx oi, uintptr_t retaddr)
{
- DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
- PAGE_READ | PAGE_WRITE, retaddr);
+ DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, retaddr);
DATA_TYPE ret;
#if DATA_SIZE == 16
return BSWAP(ret);
}
-#if DATA_SIZE >= 16
-#if HAVE_ATOMIC128
-ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, uintptr_t retaddr)
-{
- DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
- PAGE_READ, retaddr);
- DATA_TYPE val;
-
- val = atomic16_read(haddr);
- ATOMIC_MMU_CLEANUP;
- atomic_trace_ld_post(env, addr, oi);
- return BSWAP(val);
-}
-
-void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ABI_TYPE val,
- MemOpIdx oi, uintptr_t retaddr)
-{
- DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
- PAGE_WRITE, retaddr);
-
- val = BSWAP(val);
- atomic16_set(haddr, val);
- ATOMIC_MMU_CLEANUP;
- atomic_trace_st_post(env, addr, oi);
-}
-#endif
-#else
+#if DATA_SIZE < 16
ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val,
MemOpIdx oi, uintptr_t retaddr)
{
- DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
- PAGE_READ | PAGE_WRITE, retaddr);
+ DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, retaddr);
ABI_TYPE ret;
ret = qatomic_xchg__nocheck(haddr, BSWAP(val));
ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \
{ \
- DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \
- PAGE_READ | PAGE_WRITE, retaddr); \
- DATA_TYPE ret; \
+ DATA_TYPE *haddr, ret; \
+ haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, retaddr); \
ret = qatomic_##X(haddr, BSWAP(val)); \
ATOMIC_MMU_CLEANUP; \
atomic_trace_rmw_post(env, addr, oi); \
ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \
{ \
- XDATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \
- PAGE_READ | PAGE_WRITE, retaddr); \
- XDATA_TYPE ldo, ldn, old, new, val = xval; \
+ XDATA_TYPE *haddr, ldo, ldn, old, new, val = xval; \
+ haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, retaddr); \
smp_mb(); \
ldn = qatomic_read__nocheck(haddr); \
do { \
#undef ADD
#undef GEN_ATOMIC_HELPER_FN
-#endif /* DATA_SIZE >= 16 */
+#endif /* DATA_SIZE < 16 */
#undef END
#endif /* DATA_SIZE > 1 */
cpu->cpu_index, tb->tc.ptr, tb->cs_base, pc,
tb->flags, tb->cflags, lookup_symbol(pc));
-#if defined(DEBUG_DISAS)
if (qemu_loglevel_mask(CPU_LOG_TB_CPU)) {
FILE *logfile = qemu_log_trylock();
if (logfile) {
qemu_log_unlock(logfile);
}
}
-#endif /* DEBUG_DISAS */
}
}
/*
* Probe for an atomic operation. Do not allow unaligned operations,
* or io operations to proceed. Return the host address.
- *
- * @prot may be PAGE_READ, PAGE_WRITE, or PAGE_READ|PAGE_WRITE.
*/
static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, int size, int prot,
- uintptr_t retaddr)
+ MemOpIdx oi, int size, uintptr_t retaddr)
{
uintptr_t mmu_idx = get_mmuidx(oi);
MemOp mop = get_memop(oi);
tlbe = tlb_entry(env, mmu_idx, addr);
/* Check TLB entry and enforce page permissions. */
- if (prot & PAGE_WRITE) {
- tlb_addr = tlb_addr_write(tlbe);
- if (!tlb_hit(tlb_addr, addr)) {
- if (!victim_tlb_hit(env, mmu_idx, index, MMU_DATA_STORE,
- addr & TARGET_PAGE_MASK)) {
- tlb_fill(env_cpu(env), addr, size,
- MMU_DATA_STORE, mmu_idx, retaddr);
- index = tlb_index(env, mmu_idx, addr);
- tlbe = tlb_entry(env, mmu_idx, addr);
- }
- tlb_addr = tlb_addr_write(tlbe) & ~TLB_INVALID_MASK;
+ tlb_addr = tlb_addr_write(tlbe);
+ if (!tlb_hit(tlb_addr, addr)) {
+ if (!victim_tlb_hit(env, mmu_idx, index, MMU_DATA_STORE,
+ addr & TARGET_PAGE_MASK)) {
+ tlb_fill(env_cpu(env), addr, size,
+ MMU_DATA_STORE, mmu_idx, retaddr);
+ index = tlb_index(env, mmu_idx, addr);
+ tlbe = tlb_entry(env, mmu_idx, addr);
}
+ tlb_addr = tlb_addr_write(tlbe) & ~TLB_INVALID_MASK;
+ }
- if (prot & PAGE_READ) {
- /*
- * Let the guest notice RMW on a write-only page.
- * We have just verified that the page is writable.
- * Subpage lookups may have left TLB_INVALID_MASK set,
- * but addr_read will only be -1 if PAGE_READ was unset.
- */
- if (unlikely(tlbe->addr_read == -1)) {
- tlb_fill(env_cpu(env), addr, size,
- MMU_DATA_LOAD, mmu_idx, retaddr);
- /*
- * Since we don't support reads and writes to different
- * addresses, and we do have the proper page loaded for
- * write, this shouldn't ever return. But just in case,
- * handle via stop-the-world.
- */
- goto stop_the_world;
- }
- /* Collect TLB_WATCHPOINT for read. */
- tlb_addr |= tlbe->addr_read;
- }
- } else /* if (prot & PAGE_READ) */ {
- tlb_addr = tlbe->addr_read;
- if (!tlb_hit(tlb_addr, addr)) {
- if (!victim_tlb_hit(env, mmu_idx, index, MMU_DATA_LOAD,
- addr & TARGET_PAGE_MASK)) {
- tlb_fill(env_cpu(env), addr, size,
- MMU_DATA_LOAD, mmu_idx, retaddr);
- index = tlb_index(env, mmu_idx, addr);
- tlbe = tlb_entry(env, mmu_idx, addr);
- }
- tlb_addr = tlbe->addr_read & ~TLB_INVALID_MASK;
- }
+ /*
+ * Let the guest notice RMW on a write-only page.
+ * We have just verified that the page is writable.
+ * Subpage lookups may have left TLB_INVALID_MASK set,
+ * but addr_read will only be -1 if PAGE_READ was unset.
+ */
+ if (unlikely(tlbe->addr_read == -1)) {
+ tlb_fill(env_cpu(env), addr, size, MMU_DATA_LOAD, mmu_idx, retaddr);
+ /*
+ * Since we don't support reads and writes to different
+ * addresses, and we do have the proper page loaded for
+ * write, this shouldn't ever return. But just in case,
+ * handle via stop-the-world.
+ */
+ goto stop_the_world;
}
+ /* Collect TLB_WATCHPOINT for read. */
+ tlb_addr |= tlbe->addr_read;
/* Notice an IO access or a needs-MMU-lookup access */
if (unlikely(tlb_addr & (TLB_MMIO | TLB_DISCARD_WRITE))) {
}
if (unlikely(tlb_addr & TLB_WATCHPOINT)) {
- QEMU_BUILD_BUG_ON(PAGE_READ != BP_MEM_READ);
- QEMU_BUILD_BUG_ON(PAGE_WRITE != BP_MEM_WRITE);
- /* therefore prot == watchpoint bits */
- cpu_check_watchpoint(env_cpu(env), addr, size,
- full->attrs, prot, retaddr);
+ cpu_check_watchpoint(env_cpu(env), addr, size, full->attrs,
+ BP_MEM_READ | BP_MEM_WRITE, retaddr);
}
return hostaddr;
return ret;
}
-uint16_t cpu_ldw_be_mmu(CPUArchState *env, abi_ptr addr,
- MemOpIdx oi, uintptr_t ra)
-{
- uint16_t ret;
-
- tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_BEUW);
- ret = do_ld2_mmu(env, addr, oi, ra, MMU_DATA_LOAD);
- plugin_load_cb(env, addr, oi);
- return ret;
-}
-
-uint32_t cpu_ldl_be_mmu(CPUArchState *env, abi_ptr addr,
- MemOpIdx oi, uintptr_t ra)
-{
- uint32_t ret;
-
- tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_BEUL);
- ret = do_ld4_mmu(env, addr, oi, ra, MMU_DATA_LOAD);
- plugin_load_cb(env, addr, oi);
- return ret;
-}
-
-uint64_t cpu_ldq_be_mmu(CPUArchState *env, abi_ptr addr,
- MemOpIdx oi, uintptr_t ra)
-{
- uint64_t ret;
-
- tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_BEUQ);
- ret = do_ld8_mmu(env, addr, oi, ra, MMU_DATA_LOAD);
- plugin_load_cb(env, addr, oi);
- return ret;
-}
-
-uint16_t cpu_ldw_le_mmu(CPUArchState *env, abi_ptr addr,
- MemOpIdx oi, uintptr_t ra)
+uint16_t cpu_ldw_mmu(CPUArchState *env, abi_ptr addr,
+ MemOpIdx oi, uintptr_t ra)
{
uint16_t ret;
- tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_LEUW);
+ tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_16);
ret = do_ld2_mmu(env, addr, oi, ra, MMU_DATA_LOAD);
plugin_load_cb(env, addr, oi);
return ret;
}
-uint32_t cpu_ldl_le_mmu(CPUArchState *env, abi_ptr addr,
- MemOpIdx oi, uintptr_t ra)
+uint32_t cpu_ldl_mmu(CPUArchState *env, abi_ptr addr,
+ MemOpIdx oi, uintptr_t ra)
{
uint32_t ret;
- tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_LEUL);
+ tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_32);
ret = do_ld4_mmu(env, addr, oi, ra, MMU_DATA_LOAD);
plugin_load_cb(env, addr, oi);
return ret;
}
-uint64_t cpu_ldq_le_mmu(CPUArchState *env, abi_ptr addr,
- MemOpIdx oi, uintptr_t ra)
+uint64_t cpu_ldq_mmu(CPUArchState *env, abi_ptr addr,
+ MemOpIdx oi, uintptr_t ra)
{
uint64_t ret;
- tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_LEUQ);
+ tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_64);
ret = do_ld8_mmu(env, addr, oi, ra, MMU_DATA_LOAD);
plugin_load_cb(env, addr, oi);
return ret;
}
-Int128 cpu_ld16_be_mmu(CPUArchState *env, abi_ptr addr,
- MemOpIdx oi, uintptr_t ra)
-{
- Int128 ret;
-
- tcg_debug_assert((get_memop(oi) & (MO_BSWAP|MO_SIZE)) == (MO_BE|MO_128));
- ret = do_ld16_mmu(env, addr, oi, ra);
- plugin_load_cb(env, addr, oi);
- return ret;
-}
-
-Int128 cpu_ld16_le_mmu(CPUArchState *env, abi_ptr addr,
- MemOpIdx oi, uintptr_t ra)
+Int128 cpu_ld16_mmu(CPUArchState *env, abi_ptr addr,
+ MemOpIdx oi, uintptr_t ra)
{
Int128 ret;
- tcg_debug_assert((get_memop(oi) & (MO_BSWAP|MO_SIZE)) == (MO_LE|MO_128));
+ tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_128);
ret = do_ld16_mmu(env, addr, oi, ra);
plugin_load_cb(env, addr, oi);
return ret;
case MO_ATOM_WITHIN16_PAIR:
/* Since size > 8, this is the half that must be atomic. */
- if (!HAVE_al16) {
+ if (!HAVE_ATOMIC128_RW) {
cpu_loop_exit_atomic(env_cpu(env), ra);
}
return store_whole_le16(p->haddr, p->size, val_le);
plugin_store_cb(env, addr, oi);
}
-void cpu_stw_be_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
- MemOpIdx oi, uintptr_t retaddr)
-{
- tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_BEUW);
- do_st2_mmu(env, addr, val, oi, retaddr);
- plugin_store_cb(env, addr, oi);
-}
-
-void cpu_stl_be_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
- MemOpIdx oi, uintptr_t retaddr)
-{
- tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_BEUL);
- do_st4_mmu(env, addr, val, oi, retaddr);
- plugin_store_cb(env, addr, oi);
-}
-
-void cpu_stq_be_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
- MemOpIdx oi, uintptr_t retaddr)
-{
- tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_BEUQ);
- do_st8_mmu(env, addr, val, oi, retaddr);
- plugin_store_cb(env, addr, oi);
-}
-
-void cpu_stw_le_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
- MemOpIdx oi, uintptr_t retaddr)
+void cpu_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
+ MemOpIdx oi, uintptr_t retaddr)
{
- tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_LEUW);
+ tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_16);
do_st2_mmu(env, addr, val, oi, retaddr);
plugin_store_cb(env, addr, oi);
}
-void cpu_stl_le_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
+void cpu_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
MemOpIdx oi, uintptr_t retaddr)
{
- tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_LEUL);
+ tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_32);
do_st4_mmu(env, addr, val, oi, retaddr);
plugin_store_cb(env, addr, oi);
}
-void cpu_stq_le_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
- MemOpIdx oi, uintptr_t retaddr)
+void cpu_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
+ MemOpIdx oi, uintptr_t retaddr)
{
- tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_LEUQ);
+ tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_64);
do_st8_mmu(env, addr, val, oi, retaddr);
plugin_store_cb(env, addr, oi);
}
-void cpu_st16_be_mmu(CPUArchState *env, target_ulong addr, Int128 val,
- MemOpIdx oi, uintptr_t retaddr)
-{
- tcg_debug_assert((get_memop(oi) & (MO_BSWAP|MO_SIZE)) == (MO_BE|MO_128));
- do_st16_mmu(env, addr, val, oi, retaddr);
- plugin_store_cb(env, addr, oi);
-}
-
-void cpu_st16_le_mmu(CPUArchState *env, target_ulong addr, Int128 val,
- MemOpIdx oi, uintptr_t retaddr)
+void cpu_st16_mmu(CPUArchState *env, target_ulong addr, Int128 val,
+ MemOpIdx oi, uintptr_t retaddr)
{
- tcg_debug_assert((get_memop(oi) & (MO_BSWAP|MO_SIZE)) == (MO_LE|MO_128));
+ tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_128);
do_st16_mmu(env, addr, val, oi, retaddr);
plugin_store_cb(env, addr, oi);
}
#include "atomic_template.h"
#endif
-#if HAVE_CMPXCHG128 || HAVE_ATOMIC128
+#if defined(CONFIG_ATOMIC128) || defined(CONFIG_CMPXCHG128)
#define DATA_SIZE 16
#include "atomic_template.h"
#endif
#endif
#define HAVE_al8_fast (ATOMIC_REG_SIZE >= 8)
-/*
- * If __alignof(unsigned __int128) < 16, GCC may refuse to inline atomics
- * that are supported by the host, e.g. s390x. We can force the pointer to
- * have our known alignment with __builtin_assume_aligned, however prior to
- * GCC 13 that was only reliable with optimization enabled. See
- * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=107389
- */
-#if defined(CONFIG_ATOMIC128_OPT)
-# if !defined(__OPTIMIZE__)
-# define ATTRIBUTE_ATOMIC128_OPT __attribute__((optimize("O1")))
-# endif
-# define CONFIG_ATOMIC128
-#endif
-#ifndef ATTRIBUTE_ATOMIC128_OPT
-# define ATTRIBUTE_ATOMIC128_OPT
-#endif
-
-#if defined(CONFIG_ATOMIC128)
-# define HAVE_al16_fast true
-#else
-# define HAVE_al16_fast false
-#endif
-#if defined(CONFIG_ATOMIC128) || defined(CONFIG_CMPXCHG128)
-# define HAVE_al16 true
-#else
-# define HAVE_al16 false
-#endif
-
-
/**
* required_atomicity:
*
return qatomic_read__nocheck(p);
}
-/**
- * load_atomic16:
- * @pv: host address
- *
- * Atomically load 16 aligned bytes from @pv.
- */
-static inline Int128 ATTRIBUTE_ATOMIC128_OPT
-load_atomic16(void *pv)
-{
-#ifdef CONFIG_ATOMIC128
- __uint128_t *p = __builtin_assume_aligned(pv, 16);
- Int128Alias r;
-
- r.u = qatomic_read__nocheck(p);
- return r.s;
-#else
- qemu_build_not_reached();
-#endif
-}
-
/**
* load_atomic8_or_exit:
* @env: cpu context
{
Int128 *p = __builtin_assume_aligned(pv, 16);
- if (HAVE_al16_fast) {
- return load_atomic16(p);
+ if (HAVE_ATOMIC128_RO) {
+ return atomic16_read_ro(p);
}
#ifdef CONFIG_USER_ONLY
* In system mode all guest pages are writable, and for user-only
* we have just checked writability. Try cmpxchg.
*/
-#if defined(CONFIG_CMPXCHG128)
- /* Swap 0 with 0, with the side-effect of returning the old value. */
- {
- Int128Alias r;
- r.u = __sync_val_compare_and_swap_16((__uint128_t *)p, 0, 0);
- return r.s;
+ if (HAVE_ATOMIC128_RW) {
+ return atomic16_read_rw(p);
}
-#endif
/* Ultimate fallback: re-execute in serial context. */
cpu_loop_exit_atomic(env_cpu(env), ra);
static inline uint64_t ATTRIBUTE_ATOMIC128_OPT
load_atom_extract_al16_or_al8(void *pv, int s)
{
-#if defined(CONFIG_ATOMIC128)
uintptr_t pi = (uintptr_t)pv;
int o = pi & 7;
int shr = (HOST_BIG_ENDIAN ? 16 - s - o : o) * 8;
- __uint128_t r;
+ Int128 r;
pv = (void *)(pi & ~7);
if (pi & 8) {
uint64_t b = qatomic_read__nocheck(p8 + 1);
if (HOST_BIG_ENDIAN) {
- r = ((__uint128_t)a << 64) | b;
+ r = int128_make128(b, a);
} else {
- r = ((__uint128_t)b << 64) | a;
+ r = int128_make128(a, b);
}
} else {
- __uint128_t *p16 = __builtin_assume_aligned(pv, 16, 0);
- r = qatomic_read__nocheck(p16);
+ r = atomic16_read_ro(pv);
}
- return r >> shr;
-#else
- qemu_build_not_reached();
-#endif
+ return int128_getlo(int128_urshift(r, shr));
}
/**
if (likely((pi & 1) == 0)) {
return load_atomic2(pv);
}
- if (HAVE_al16_fast) {
+ if (HAVE_ATOMIC128_RO) {
return load_atom_extract_al16_or_al8(pv, 2);
}
if (likely((pi & 3) == 0)) {
return load_atomic4(pv);
}
- if (HAVE_al16_fast) {
+ if (HAVE_ATOMIC128_RO) {
return load_atom_extract_al16_or_al8(pv, 4);
}
if (HAVE_al8 && likely((pi & 7) == 0)) {
return load_atomic8(pv);
}
- if (HAVE_al16_fast) {
+ if (HAVE_ATOMIC128_RO) {
return load_atom_extract_al16_or_al8(pv, 8);
}
* If the host does not support 16-byte atomics, wait until we have
* examined the atomicity parameters below.
*/
- if (HAVE_al16_fast && likely((pi & 15) == 0)) {
- return load_atomic16(pv);
+ if (HAVE_ATOMIC128_RO && likely((pi & 15) == 0)) {
+ return atomic16_read_ro(pv);
}
atmax = required_atomicity(env, pi, memop);
qatomic_set__nocheck(p, val);
}
-/**
- * store_atomic16:
- * @pv: host address
- * @val: value to store
- *
- * Atomically store 16 aligned bytes to @pv.
- */
-static inline void ATTRIBUTE_ATOMIC128_OPT
-store_atomic16(void *pv, Int128Alias val)
-{
-#if defined(CONFIG_ATOMIC128)
- __uint128_t *pu = __builtin_assume_aligned(pv, 16);
- qatomic_set__nocheck(pu, val.u);
-#elif defined(CONFIG_CMPXCHG128)
- __uint128_t *pu = __builtin_assume_aligned(pv, 16);
- __uint128_t o;
-
- /*
- * Without CONFIG_ATOMIC128, __atomic_compare_exchange_n will always
- * defer to libatomic, so we must use __sync_*_compare_and_swap_16
- * and accept the sequential consistency that comes with it.
- */
- do {
- o = *pu;
- } while (!__sync_bool_compare_and_swap_16(pu, o, val.u));
-#else
- qemu_build_not_reached();
-#endif
-}
-
/**
* store_atom_4x2
*/
int sh = o * 8;
Int128 m, v;
- qemu_build_assert(HAVE_al16);
+ qemu_build_assert(HAVE_ATOMIC128_RW);
/* Like MAKE_64BIT_MASK(0, sz), but larger. */
if (sz <= 64) {
return;
}
} else if ((pi & 15) == 7) {
- if (HAVE_al16) {
+ if (HAVE_ATOMIC128_RW) {
Int128 v = int128_lshift(int128_make64(val), 56);
Int128 m = int128_lshift(int128_make64(0xffff), 56);
store_atom_insert_al16(pv - 7, v, m);
return;
}
} else {
- if (HAVE_al16) {
+ if (HAVE_ATOMIC128_RW) {
store_whole_le16(pv, 4, int128_make64(cpu_to_le32(val)));
return;
}
}
break;
case MO_64:
- if (HAVE_al16) {
+ if (HAVE_ATOMIC128_RW) {
store_whole_le16(pv, 8, int128_make64(cpu_to_le64(val)));
return;
}
uint64_t a, b;
int atmax;
- if (HAVE_al16_fast && likely((pi & 15) == 0)) {
- store_atomic16(pv, val);
+ if (HAVE_ATOMIC128_RW && likely((pi & 15) == 0)) {
+ atomic16_set(pv, val);
return;
}
}
break;
case -MO_64:
- if (HAVE_al16) {
+ if (HAVE_ATOMIC128_RW) {
uint64_t val_le;
int s2 = pi & 15;
int s1 = 16 - s2;
}
break;
case MO_128:
- if (HAVE_al16) {
- store_atomic16(pv, val);
+ if (HAVE_ATOMIC128_RW) {
+ atomic16_set(pv, val);
return;
}
break;
int mmu_idx, uintptr_t ra)
{
MemOpIdx oi = make_memop_idx(MO_BEUW | MO_UNALN, mmu_idx);
- return cpu_ldw_be_mmu(env, addr, oi, ra);
+ return cpu_ldw_mmu(env, addr, oi, ra);
}
int cpu_ldsw_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
int mmu_idx, uintptr_t ra)
{
MemOpIdx oi = make_memop_idx(MO_BEUL | MO_UNALN, mmu_idx);
- return cpu_ldl_be_mmu(env, addr, oi, ra);
+ return cpu_ldl_mmu(env, addr, oi, ra);
}
uint64_t cpu_ldq_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
int mmu_idx, uintptr_t ra)
{
MemOpIdx oi = make_memop_idx(MO_BEUQ | MO_UNALN, mmu_idx);
- return cpu_ldq_be_mmu(env, addr, oi, ra);
+ return cpu_ldq_mmu(env, addr, oi, ra);
}
uint32_t cpu_lduw_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
int mmu_idx, uintptr_t ra)
{
MemOpIdx oi = make_memop_idx(MO_LEUW | MO_UNALN, mmu_idx);
- return cpu_ldw_le_mmu(env, addr, oi, ra);
+ return cpu_ldw_mmu(env, addr, oi, ra);
}
int cpu_ldsw_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
int mmu_idx, uintptr_t ra)
{
MemOpIdx oi = make_memop_idx(MO_LEUL | MO_UNALN, mmu_idx);
- return cpu_ldl_le_mmu(env, addr, oi, ra);
+ return cpu_ldl_mmu(env, addr, oi, ra);
}
uint64_t cpu_ldq_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
int mmu_idx, uintptr_t ra)
{
MemOpIdx oi = make_memop_idx(MO_LEUQ | MO_UNALN, mmu_idx);
- return cpu_ldq_le_mmu(env, addr, oi, ra);
+ return cpu_ldq_mmu(env, addr, oi, ra);
}
void cpu_stb_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val,
int mmu_idx, uintptr_t ra)
{
MemOpIdx oi = make_memop_idx(MO_BEUW | MO_UNALN, mmu_idx);
- cpu_stw_be_mmu(env, addr, val, oi, ra);
+ cpu_stw_mmu(env, addr, val, oi, ra);
}
void cpu_stl_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val,
int mmu_idx, uintptr_t ra)
{
MemOpIdx oi = make_memop_idx(MO_BEUL | MO_UNALN, mmu_idx);
- cpu_stl_be_mmu(env, addr, val, oi, ra);
+ cpu_stl_mmu(env, addr, val, oi, ra);
}
void cpu_stq_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint64_t val,
int mmu_idx, uintptr_t ra)
{
MemOpIdx oi = make_memop_idx(MO_BEUQ | MO_UNALN, mmu_idx);
- cpu_stq_be_mmu(env, addr, val, oi, ra);
+ cpu_stq_mmu(env, addr, val, oi, ra);
}
void cpu_stw_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val,
int mmu_idx, uintptr_t ra)
{
MemOpIdx oi = make_memop_idx(MO_LEUW | MO_UNALN, mmu_idx);
- cpu_stw_le_mmu(env, addr, val, oi, ra);
+ cpu_stw_mmu(env, addr, val, oi, ra);
}
void cpu_stl_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val,
int mmu_idx, uintptr_t ra)
{
MemOpIdx oi = make_memop_idx(MO_LEUL | MO_UNALN, mmu_idx);
- cpu_stl_le_mmu(env, addr, val, oi, ra);
+ cpu_stl_mmu(env, addr, val, oi, ra);
}
void cpu_stq_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint64_t val,
int mmu_idx, uintptr_t ra)
{
MemOpIdx oi = make_memop_idx(MO_LEUQ | MO_UNALN, mmu_idx);
- cpu_stq_le_mmu(env, addr, val, oi, ra);
+ cpu_stq_mmu(env, addr, val, oi, ra);
}
/*--------------------------*/
i128, env, i64, i128, i128, i32)
#endif
-DEF_HELPER_FLAGS_5(nonatomic_cmpxchgo_be, TCG_CALL_NO_WG,
- i128, env, i64, i128, i128, i32)
-DEF_HELPER_FLAGS_5(nonatomic_cmpxchgo_le, TCG_CALL_NO_WG,
+DEF_HELPER_FLAGS_5(nonatomic_cmpxchgo, TCG_CALL_NO_WG,
i128, env, i64, i128, i128, i32)
#ifdef CONFIG_ATOMIC64
qatomic_set(&prof->search_out_len, prof->search_out_len + search_size);
#endif
-#ifdef DEBUG_DISAS
if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) &&
qemu_log_in_addr_range(pc)) {
FILE *logfile = qemu_log_trylock();
qemu_log_unlock(logfile);
}
}
-#endif
qatomic_set(&tcg_ctx->code_gen_ptr, (void *)
ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size,
tb->size = db->pc_next - db->pc_first;
tb->icount = db->num_insns;
-#ifdef DEBUG_DISAS
if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)
&& qemu_log_in_addr_range(db->pc_first)) {
FILE *logfile = qemu_log_trylock();
qemu_log_unlock(logfile);
}
}
-#endif
}
static void *translator_access(CPUArchState *env, DisasContextBase *db,
return ret;
}
-static uint16_t do_ld2_he_mmu(CPUArchState *env, abi_ptr addr,
- MemOp mop, uintptr_t ra)
+static uint16_t do_ld2_mmu(CPUArchState *env, abi_ptr addr,
+ MemOp mop, uintptr_t ra)
{
void *haddr;
uint16_t ret;
haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_LOAD);
ret = load_atom_2(env, ra, haddr, mop);
clear_helper_retaddr();
- return ret;
-}
-
-tcg_target_ulong helper_lduw_mmu(CPUArchState *env, uint64_t addr,
- MemOpIdx oi, uintptr_t ra)
-{
- MemOp mop = get_memop(oi);
- uint16_t ret = do_ld2_he_mmu(env, addr, mop, ra);
if (mop & MO_BSWAP) {
ret = bswap16(ret);
return ret;
}
-tcg_target_ulong helper_ldsw_mmu(CPUArchState *env, uint64_t addr,
+tcg_target_ulong helper_lduw_mmu(CPUArchState *env, uint64_t addr,
MemOpIdx oi, uintptr_t ra)
{
- MemOp mop = get_memop(oi);
- int16_t ret = do_ld2_he_mmu(env, addr, mop, ra);
-
- if (mop & MO_BSWAP) {
- ret = bswap16(ret);
- }
- return ret;
+ return do_ld2_mmu(env, addr, get_memop(oi), ra);
}
-uint16_t cpu_ldw_be_mmu(CPUArchState *env, abi_ptr addr,
- MemOpIdx oi, uintptr_t ra)
+tcg_target_ulong helper_ldsw_mmu(CPUArchState *env, uint64_t addr,
+ MemOpIdx oi, uintptr_t ra)
{
- MemOp mop = get_memop(oi);
- uint16_t ret;
-
- tcg_debug_assert((mop & MO_BSWAP) == MO_BE);
- ret = do_ld2_he_mmu(env, addr, mop, ra);
- qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R);
- return cpu_to_be16(ret);
+ return (int16_t)do_ld2_mmu(env, addr, get_memop(oi), ra);
}
-uint16_t cpu_ldw_le_mmu(CPUArchState *env, abi_ptr addr,
- MemOpIdx oi, uintptr_t ra)
+uint16_t cpu_ldw_mmu(CPUArchState *env, abi_ptr addr,
+ MemOpIdx oi, uintptr_t ra)
{
- MemOp mop = get_memop(oi);
- uint16_t ret;
-
- tcg_debug_assert((mop & MO_BSWAP) == MO_LE);
- ret = do_ld2_he_mmu(env, addr, mop, ra);
+ uint16_t ret = do_ld2_mmu(env, addr, get_memop(oi), ra);
qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R);
- return cpu_to_le16(ret);
+ return ret;
}
-static uint32_t do_ld4_he_mmu(CPUArchState *env, abi_ptr addr,
- MemOp mop, uintptr_t ra)
+static uint32_t do_ld4_mmu(CPUArchState *env, abi_ptr addr,
+ MemOp mop, uintptr_t ra)
{
void *haddr;
uint32_t ret;
haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_LOAD);
ret = load_atom_4(env, ra, haddr, mop);
clear_helper_retaddr();
- return ret;
-}
-
-tcg_target_ulong helper_ldul_mmu(CPUArchState *env, uint64_t addr,
- MemOpIdx oi, uintptr_t ra)
-{
- MemOp mop = get_memop(oi);
- uint32_t ret = do_ld4_he_mmu(env, addr, mop, ra);
if (mop & MO_BSWAP) {
ret = bswap32(ret);
return ret;
}
-tcg_target_ulong helper_ldsl_mmu(CPUArchState *env, uint64_t addr,
+tcg_target_ulong helper_ldul_mmu(CPUArchState *env, uint64_t addr,
MemOpIdx oi, uintptr_t ra)
{
- MemOp mop = get_memop(oi);
- int32_t ret = do_ld4_he_mmu(env, addr, mop, ra);
-
- if (mop & MO_BSWAP) {
- ret = bswap32(ret);
- }
- return ret;
+ return do_ld4_mmu(env, addr, get_memop(oi), ra);
}
-uint32_t cpu_ldl_be_mmu(CPUArchState *env, abi_ptr addr,
- MemOpIdx oi, uintptr_t ra)
+tcg_target_ulong helper_ldsl_mmu(CPUArchState *env, uint64_t addr,
+ MemOpIdx oi, uintptr_t ra)
{
- MemOp mop = get_memop(oi);
- uint32_t ret;
-
- tcg_debug_assert((mop & MO_BSWAP) == MO_BE);
- ret = do_ld4_he_mmu(env, addr, mop, ra);
- qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R);
- return cpu_to_be32(ret);
+ return (int32_t)do_ld4_mmu(env, addr, get_memop(oi), ra);
}
-uint32_t cpu_ldl_le_mmu(CPUArchState *env, abi_ptr addr,
- MemOpIdx oi, uintptr_t ra)
+uint32_t cpu_ldl_mmu(CPUArchState *env, abi_ptr addr,
+ MemOpIdx oi, uintptr_t ra)
{
- MemOp mop = get_memop(oi);
- uint32_t ret;
-
- tcg_debug_assert((mop & MO_BSWAP) == MO_LE);
- ret = do_ld4_he_mmu(env, addr, mop, ra);
+ uint32_t ret = do_ld4_mmu(env, addr, get_memop(oi), ra);
qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R);
- return cpu_to_le32(ret);
+ return ret;
}
-static uint64_t do_ld8_he_mmu(CPUArchState *env, abi_ptr addr,
- MemOp mop, uintptr_t ra)
+static uint64_t do_ld8_mmu(CPUArchState *env, abi_ptr addr,
+ MemOp mop, uintptr_t ra)
{
void *haddr;
uint64_t ret;
haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_LOAD);
ret = load_atom_8(env, ra, haddr, mop);
clear_helper_retaddr();
- return ret;
-}
-
-uint64_t helper_ldq_mmu(CPUArchState *env, uint64_t addr,
- MemOpIdx oi, uintptr_t ra)
-{
- MemOp mop = get_memop(oi);
- uint64_t ret = do_ld8_he_mmu(env, addr, mop, ra);
if (mop & MO_BSWAP) {
ret = bswap64(ret);
return ret;
}
-uint64_t cpu_ldq_be_mmu(CPUArchState *env, abi_ptr addr,
+uint64_t helper_ldq_mmu(CPUArchState *env, uint64_t addr,
MemOpIdx oi, uintptr_t ra)
{
- MemOp mop = get_memop(oi);
- uint64_t ret;
-
- tcg_debug_assert((mop & MO_BSWAP) == MO_BE);
- ret = do_ld8_he_mmu(env, addr, mop, ra);
- qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R);
- return cpu_to_be64(ret);
+ return do_ld8_mmu(env, addr, get_memop(oi), ra);
}
-uint64_t cpu_ldq_le_mmu(CPUArchState *env, abi_ptr addr,
- MemOpIdx oi, uintptr_t ra)
+uint64_t cpu_ldq_mmu(CPUArchState *env, abi_ptr addr,
+ MemOpIdx oi, uintptr_t ra)
{
- MemOp mop = get_memop(oi);
- uint64_t ret;
-
- tcg_debug_assert((mop & MO_BSWAP) == MO_LE);
- ret = do_ld8_he_mmu(env, addr, mop, ra);
+ uint64_t ret = do_ld8_mmu(env, addr, get_memop(oi), ra);
qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R);
- return cpu_to_le64(ret);
+ return ret;
}
-static Int128 do_ld16_he_mmu(CPUArchState *env, abi_ptr addr,
- MemOp mop, uintptr_t ra)
+static Int128 do_ld16_mmu(CPUArchState *env, abi_ptr addr,
+ MemOp mop, uintptr_t ra)
{
void *haddr;
Int128 ret;
haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_LOAD);
ret = load_atom_16(env, ra, haddr, mop);
clear_helper_retaddr();
- return ret;
-}
-
-Int128 helper_ld16_mmu(CPUArchState *env, uint64_t addr,
- MemOpIdx oi, uintptr_t ra)
-{
- MemOp mop = get_memop(oi);
- Int128 ret = do_ld16_he_mmu(env, addr, mop, ra);
if (mop & MO_BSWAP) {
ret = bswap128(ret);
return ret;
}
-Int128 helper_ld_i128(CPUArchState *env, uint64_t addr, MemOpIdx oi)
+Int128 helper_ld16_mmu(CPUArchState *env, uint64_t addr,
+ MemOpIdx oi, uintptr_t ra)
{
- return helper_ld16_mmu(env, addr, oi, GETPC());
+ return do_ld16_mmu(env, addr, get_memop(oi), ra);
}
-Int128 cpu_ld16_be_mmu(CPUArchState *env, abi_ptr addr,
- MemOpIdx oi, uintptr_t ra)
+Int128 helper_ld_i128(CPUArchState *env, uint64_t addr, MemOpIdx oi)
{
- MemOp mop = get_memop(oi);
- Int128 ret;
-
- tcg_debug_assert((mop & MO_BSWAP) == MO_BE);
- ret = do_ld16_he_mmu(env, addr, mop, ra);
- qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R);
- if (!HOST_BIG_ENDIAN) {
- ret = bswap128(ret);
- }
- return ret;
+ return helper_ld16_mmu(env, addr, oi, GETPC());
}
-Int128 cpu_ld16_le_mmu(CPUArchState *env, abi_ptr addr,
- MemOpIdx oi, uintptr_t ra)
+Int128 cpu_ld16_mmu(CPUArchState *env, abi_ptr addr,
+ MemOpIdx oi, uintptr_t ra)
{
- MemOp mop = get_memop(oi);
- Int128 ret;
-
- tcg_debug_assert((mop & MO_BSWAP) == MO_LE);
- ret = do_ld16_he_mmu(env, addr, mop, ra);
+ Int128 ret = do_ld16_mmu(env, addr, get_memop(oi), ra);
qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R);
- if (HOST_BIG_ENDIAN) {
- ret = bswap128(ret);
- }
return ret;
}
qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W);
}
-static void do_st2_he_mmu(CPUArchState *env, abi_ptr addr, uint16_t val,
- MemOp mop, uintptr_t ra)
+static void do_st2_mmu(CPUArchState *env, abi_ptr addr, uint16_t val,
+ MemOp mop, uintptr_t ra)
{
void *haddr;
tcg_debug_assert((mop & MO_SIZE) == MO_16);
haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_STORE);
- store_atom_2(env, ra, haddr, mop, val);
- clear_helper_retaddr();
-}
-
-void helper_stw_mmu(CPUArchState *env, uint64_t addr, uint32_t val,
- MemOpIdx oi, uintptr_t ra)
-{
- MemOp mop = get_memop(oi);
if (mop & MO_BSWAP) {
val = bswap16(val);
}
- do_st2_he_mmu(env, addr, val, mop, ra);
+ store_atom_2(env, ra, haddr, mop, val);
+ clear_helper_retaddr();
}
-void cpu_stw_be_mmu(CPUArchState *env, abi_ptr addr, uint16_t val,
+void helper_stw_mmu(CPUArchState *env, uint64_t addr, uint32_t val,
MemOpIdx oi, uintptr_t ra)
{
- MemOp mop = get_memop(oi);
-
- tcg_debug_assert((mop & MO_BSWAP) == MO_BE);
- do_st2_he_mmu(env, addr, be16_to_cpu(val), mop, ra);
- qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W);
+ do_st2_mmu(env, addr, val, get_memop(oi), ra);
}
-void cpu_stw_le_mmu(CPUArchState *env, abi_ptr addr, uint16_t val,
+void cpu_stw_mmu(CPUArchState *env, abi_ptr addr, uint16_t val,
MemOpIdx oi, uintptr_t ra)
{
- MemOp mop = get_memop(oi);
-
- tcg_debug_assert((mop & MO_BSWAP) == MO_LE);
- do_st2_he_mmu(env, addr, le16_to_cpu(val), mop, ra);
+ do_st2_mmu(env, addr, val, get_memop(oi), ra);
qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W);
}
-static void do_st4_he_mmu(CPUArchState *env, abi_ptr addr, uint32_t val,
- MemOp mop, uintptr_t ra)
+static void do_st4_mmu(CPUArchState *env, abi_ptr addr, uint32_t val,
+ MemOp mop, uintptr_t ra)
{
void *haddr;
tcg_debug_assert((mop & MO_SIZE) == MO_32);
haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_STORE);
- store_atom_4(env, ra, haddr, mop, val);
- clear_helper_retaddr();
-}
-
-void helper_stl_mmu(CPUArchState *env, uint64_t addr, uint32_t val,
- MemOpIdx oi, uintptr_t ra)
-{
- MemOp mop = get_memop(oi);
if (mop & MO_BSWAP) {
val = bswap32(val);
}
- do_st4_he_mmu(env, addr, val, mop, ra);
+ store_atom_4(env, ra, haddr, mop, val);
+ clear_helper_retaddr();
}
-void cpu_stl_be_mmu(CPUArchState *env, abi_ptr addr, uint32_t val,
+void helper_stl_mmu(CPUArchState *env, uint64_t addr, uint32_t val,
MemOpIdx oi, uintptr_t ra)
{
- MemOp mop = get_memop(oi);
-
- tcg_debug_assert((mop & MO_BSWAP) == MO_BE);
- do_st4_he_mmu(env, addr, be32_to_cpu(val), mop, ra);
- qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W);
+ do_st4_mmu(env, addr, val, get_memop(oi), ra);
}
-void cpu_stl_le_mmu(CPUArchState *env, abi_ptr addr, uint32_t val,
- MemOpIdx oi, uintptr_t ra)
+void cpu_stl_mmu(CPUArchState *env, abi_ptr addr, uint32_t val,
+ MemOpIdx oi, uintptr_t ra)
{
- MemOp mop = get_memop(oi);
-
- tcg_debug_assert((mop & MO_BSWAP) == MO_LE);
- do_st4_he_mmu(env, addr, le32_to_cpu(val), mop, ra);
+ do_st4_mmu(env, addr, val, get_memop(oi), ra);
qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W);
}
-static void do_st8_he_mmu(CPUArchState *env, abi_ptr addr, uint64_t val,
- MemOp mop, uintptr_t ra)
+static void do_st8_mmu(CPUArchState *env, abi_ptr addr, uint64_t val,
+ MemOp mop, uintptr_t ra)
{
void *haddr;
tcg_debug_assert((mop & MO_SIZE) == MO_64);
haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_STORE);
- store_atom_8(env, ra, haddr, mop, val);
- clear_helper_retaddr();
-}
-
-void helper_stq_mmu(CPUArchState *env, uint64_t addr, uint64_t val,
- MemOpIdx oi, uintptr_t ra)
-{
- MemOp mop = get_memop(oi);
if (mop & MO_BSWAP) {
val = bswap64(val);
}
- do_st8_he_mmu(env, addr, val, mop, ra);
+ store_atom_8(env, ra, haddr, mop, val);
+ clear_helper_retaddr();
}
-void cpu_stq_be_mmu(CPUArchState *env, abi_ptr addr, uint64_t val,
+void helper_stq_mmu(CPUArchState *env, uint64_t addr, uint64_t val,
MemOpIdx oi, uintptr_t ra)
{
- MemOp mop = get_memop(oi);
-
- tcg_debug_assert((mop & MO_BSWAP) == MO_BE);
- do_st8_he_mmu(env, addr, cpu_to_be64(val), mop, ra);
- qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W);
+ do_st8_mmu(env, addr, val, get_memop(oi), ra);
}
-void cpu_stq_le_mmu(CPUArchState *env, abi_ptr addr, uint64_t val,
+void cpu_stq_mmu(CPUArchState *env, abi_ptr addr, uint64_t val,
MemOpIdx oi, uintptr_t ra)
{
- MemOp mop = get_memop(oi);
-
- tcg_debug_assert((mop & MO_BSWAP) == MO_LE);
- do_st8_he_mmu(env, addr, cpu_to_le64(val), mop, ra);
+ do_st8_mmu(env, addr, val, get_memop(oi), ra);
qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W);
}
-static void do_st16_he_mmu(CPUArchState *env, abi_ptr addr, Int128 val,
- MemOp mop, uintptr_t ra)
+static void do_st16_mmu(CPUArchState *env, abi_ptr addr, Int128 val,
+ MemOp mop, uintptr_t ra)
{
void *haddr;
tcg_debug_assert((mop & MO_SIZE) == MO_128);
haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_STORE);
+
+ if (mop & MO_BSWAP) {
+ val = bswap128(val);
+ }
store_atom_16(env, ra, haddr, mop, val);
clear_helper_retaddr();
}
void helper_st16_mmu(CPUArchState *env, uint64_t addr, Int128 val,
MemOpIdx oi, uintptr_t ra)
{
- MemOp mop = get_memop(oi);
-
- if (mop & MO_BSWAP) {
- val = bswap128(val);
- }
- do_st16_he_mmu(env, addr, val, mop, ra);
+ do_st16_mmu(env, addr, val, get_memop(oi), ra);
}
void helper_st_i128(CPUArchState *env, uint64_t addr, Int128 val, MemOpIdx oi)
helper_st16_mmu(env, addr, val, oi, GETPC());
}
-void cpu_st16_be_mmu(CPUArchState *env, abi_ptr addr,
- Int128 val, MemOpIdx oi, uintptr_t ra)
+void cpu_st16_mmu(CPUArchState *env, abi_ptr addr,
+ Int128 val, MemOpIdx oi, uintptr_t ra)
{
- MemOp mop = get_memop(oi);
-
- tcg_debug_assert((mop & MO_BSWAP) == MO_BE);
- if (!HOST_BIG_ENDIAN) {
- val = bswap128(val);
- }
- do_st16_he_mmu(env, addr, val, mop, ra);
- qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W);
-}
-
-void cpu_st16_le_mmu(CPUArchState *env, abi_ptr addr,
- Int128 val, MemOpIdx oi, uintptr_t ra)
-{
- MemOp mop = get_memop(oi);
-
- tcg_debug_assert((mop & MO_BSWAP) == MO_LE);
- if (HOST_BIG_ENDIAN) {
- val = bswap128(val);
- }
- do_st16_he_mmu(env, addr, val, mop, ra);
+ do_st16_mmu(env, addr, val, get_memop(oi), ra);
qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W);
}
/*
* Do not allow unaligned operations to proceed. Return the host address.
- *
- * @prot may be PAGE_READ, PAGE_WRITE, or PAGE_READ|PAGE_WRITE.
*/
static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, int size, int prot,
- uintptr_t retaddr)
+ MemOpIdx oi, int size, uintptr_t retaddr)
{
MemOp mop = get_memop(oi);
int a_bits = get_alignment_bits(mop);
/* Enforce guest required alignment. */
if (unlikely(addr & ((1 << a_bits) - 1))) {
- MMUAccessType t = prot == PAGE_READ ? MMU_DATA_LOAD : MMU_DATA_STORE;
- cpu_loop_exit_sigbus(env_cpu(env), addr, t, retaddr);
+ cpu_loop_exit_sigbus(env_cpu(env), addr, MMU_DATA_STORE, retaddr);
}
/* Enforce qemu required alignment. */
#include "atomic_template.h"
#endif
-#if HAVE_ATOMIC128 || HAVE_CMPXCHG128
+#if defined(CONFIG_ATOMIC128) || defined(CONFIG_CMPXCHG128)
#define DATA_SIZE 16
#include "atomic_template.h"
#endif
char *mem_path;
uint64_t align;
+ uint64_t offset;
bool discard_data;
bool is_pmem;
bool readonly;
ram_flags |= fb->is_pmem ? RAM_PMEM : 0;
memory_region_init_ram_from_file(&backend->mr, OBJECT(backend), name,
backend->size, fb->align, ram_flags,
- fb->mem_path, fb->readonly, errp);
+ fb->mem_path, fb->offset, fb->readonly,
+ errp);
g_free(name);
#endif
}
fb->align = val;
}
+static void file_memory_backend_get_offset(Object *o, Visitor *v,
+ const char *name, void *opaque,
+ Error **errp)
+{
+ HostMemoryBackendFile *fb = MEMORY_BACKEND_FILE(o);
+ uint64_t val = fb->offset;
+
+ visit_type_size(v, name, &val, errp);
+}
+
+static void file_memory_backend_set_offset(Object *o, Visitor *v,
+ const char *name, void *opaque,
+ Error **errp)
+{
+ HostMemoryBackend *backend = MEMORY_BACKEND(o);
+ HostMemoryBackendFile *fb = MEMORY_BACKEND_FILE(o);
+ uint64_t val;
+
+ if (host_memory_backend_mr_inited(backend)) {
+ error_setg(errp, "cannot change property '%s' of %s", name,
+ object_get_typename(o));
+ return;
+ }
+
+ if (!visit_type_size(v, name, &val, errp)) {
+ return;
+ }
+ fb->offset = val;
+}
+
#ifdef CONFIG_LIBPMEM
static bool file_memory_backend_get_pmem(Object *o, Error **errp)
{
file_memory_backend_get_align,
file_memory_backend_set_align,
NULL, NULL);
+ object_class_property_add(oc, "offset", "int",
+ file_memory_backend_get_offset,
+ file_memory_backend_set_offset,
+ NULL, NULL);
+ object_class_property_set_description(oc, "offset",
+ "Offset into the target file (ex: 1G)");
#ifdef CONFIG_LIBPMEM
object_class_property_add_bool(oc, "pmem",
file_memory_backend_get_pmem, file_memory_backend_set_pmem);
===========
This igb implementation was tested with Linux Test Project [2]_ and Windows HLK
-[3]_ during the initial development. The command used when testing with LTP is:
+[3]_ during the initial development. Later it was also tested with DPDK Test
+Suite [4]_. The command used when testing with LTP is:
.. code-block:: shell
Be aware that this implementation lacks many functionalities available with the
actual hardware, and you may experience various failures if you try to use it
-with a different operating system other than Linux and Windows or if you try
-functionalities not covered by the tests.
+with a different operating system other than DPDK, Linux, and Windows or if you
+try functionalities not covered by the tests.
Using igb
=========
:ref:`Network_emulation` in general.
However, you may also need to perform additional steps to activate SR-IOV
-feature on your guest. For Linux, refer to [4]_.
+feature on your guest. For Linux, refer to [5]_.
Developing igb
==============
.. code:: shell
- make check-avocado AVOCADO_TESTS=tests/avocado/igb.py
+ make check-avocado AVOCADO_TESTS=tests/avocado/netdev-ethtool.py
References
==========
.. [1] https://www.intel.com/content/dam/www/public/us/en/documents/datasheets/82576eb-gigabit-ethernet-controller-datasheet.pdf
.. [2] https://github.com/linux-test-project/ltp
.. [3] https://learn.microsoft.com/en-us/windows-hardware/test/hlk/
-.. [4] https://docs.kernel.org/PCI/pci-iov-howto.html
+.. [4] https://doc.dpdk.org/dts/gsg/
+.. [5] https://docs.kernel.org/PCI/pci-iov-howto.html
--- /dev/null
+/*
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ * Compare-and-swap for 128-bit atomic operations, AArch64 version.
+ *
+ * Copyright (C) 2018, 2023 Linaro, Ltd.
+ *
+ * See docs/devel/atomics.rst for discussion about the guarantees each
+ * atomic primitive is meant to provide.
+ */
+
+#ifndef AARCH64_ATOMIC128_CAS_H
+#define AARCH64_ATOMIC128_CAS_H
+
+/* Through gcc 10, aarch64 has no support for 128-bit atomics. */
+#if defined(CONFIG_ATOMIC128) || defined(CONFIG_CMPXCHG128)
+#include "host/include/generic/host/atomic128-cas.h"
+#else
+static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
+{
+ uint64_t cmpl = int128_getlo(cmp), cmph = int128_gethi(cmp);
+ uint64_t newl = int128_getlo(new), newh = int128_gethi(new);
+ uint64_t oldl, oldh;
+ uint32_t tmp;
+
+ asm("0: ldaxp %[oldl], %[oldh], %[mem]\n\t"
+ "cmp %[oldl], %[cmpl]\n\t"
+ "ccmp %[oldh], %[cmph], #0, eq\n\t"
+ "b.ne 1f\n\t"
+ "stlxp %w[tmp], %[newl], %[newh], %[mem]\n\t"
+ "cbnz %w[tmp], 0b\n"
+ "1:"
+ : [mem] "+m"(*ptr), [tmp] "=&r"(tmp),
+ [oldl] "=&r"(oldl), [oldh] "=&r"(oldh)
+ : [cmpl] "r"(cmpl), [cmph] "r"(cmph),
+ [newl] "r"(newl), [newh] "r"(newh)
+ : "memory", "cc");
+
+ return int128_make128(oldl, oldh);
+}
+
+# define CONFIG_CMPXCHG128 1
+# define HAVE_CMPXCHG128 1
+#endif
+
+#endif /* AARCH64_ATOMIC128_CAS_H */
--- /dev/null
+/*
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ * Load/store for 128-bit atomic operations, AArch64 version.
+ *
+ * Copyright (C) 2018, 2023 Linaro, Ltd.
+ *
+ * See docs/devel/atomics.rst for discussion about the guarantees each
+ * atomic primitive is meant to provide.
+ */
+
+#ifndef AARCH64_ATOMIC128_LDST_H
+#define AARCH64_ATOMIC128_LDST_H
+
+#include "host/cpuinfo.h"
+#include "tcg/debug-assert.h"
+
+/*
+ * Through gcc 10, aarch64 has no support for 128-bit atomics.
+ * Through clang 16, without -march=armv8.4-a, __atomic_load_16
+ * is incorrectly expanded to a read-write operation.
+ *
+ * Anyway, this method allows runtime detection of FEAT_LSE2.
+ */
+
+#define HAVE_ATOMIC128_RO (cpuinfo & CPUINFO_LSE2)
+#define HAVE_ATOMIC128_RW 1
+
+static inline Int128 atomic16_read_ro(const Int128 *ptr)
+{
+ uint64_t l, h;
+
+ tcg_debug_assert(HAVE_ATOMIC128_RO);
+ /* With FEAT_LSE2, 16-byte aligned LDP is atomic. */
+ asm("ldp %[l], %[h], %[mem]"
+ : [l] "=r"(l), [h] "=r"(h) : [mem] "m"(*ptr));
+
+ return int128_make128(l, h);
+}
+
+static inline Int128 atomic16_read_rw(Int128 *ptr)
+{
+ uint64_t l, h;
+ uint32_t tmp;
+
+ if (cpuinfo & CPUINFO_LSE2) {
+ /* With FEAT_LSE2, 16-byte aligned LDP is atomic. */
+ asm("ldp %[l], %[h], %[mem]"
+ : [l] "=r"(l), [h] "=r"(h) : [mem] "m"(*ptr));
+ } else {
+ /* The load must be paired with the store to guarantee not tearing. */
+ asm("0: ldxp %[l], %[h], %[mem]\n\t"
+ "stxp %w[tmp], %[l], %[h], %[mem]\n\t"
+ "cbnz %w[tmp], 0b"
+ : [mem] "+m"(*ptr), [tmp] "=&r"(tmp), [l] "=&r"(l), [h] "=&r"(h));
+ }
+
+ return int128_make128(l, h);
+}
+
+static inline void atomic16_set(Int128 *ptr, Int128 val)
+{
+ uint64_t l = int128_getlo(val), h = int128_gethi(val);
+ uint64_t t1, t2;
+
+ if (cpuinfo & CPUINFO_LSE2) {
+ /* With FEAT_LSE2, 16-byte aligned STP is atomic. */
+ asm("stp %[l], %[h], %[mem]"
+ : [mem] "=m"(*ptr) : [l] "r"(l), [h] "r"(h));
+ } else {
+ /* Load into temporaries to acquire the exclusive access lock. */
+ asm("0: ldxp %[t1], %[t2], %[mem]\n\t"
+ "stxp %w[t1], %[l], %[h], %[mem]\n\t"
+ "cbnz %w[t1], 0b"
+ : [mem] "+m"(*ptr), [t1] "=&r"(t1), [t2] "=&r"(t2)
+ : [l] "r"(l), [h] "r"(h));
+ }
+}
+
+#endif /* AARCH64_ATOMIC128_LDST_H */
--- /dev/null
+/*
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ * Host specific cpu indentification for AArch64.
+ */
+
+#ifndef HOST_CPUINFO_H
+#define HOST_CPUINFO_H
+
+#define CPUINFO_ALWAYS (1u << 0) /* so cpuinfo is nonzero */
+#define CPUINFO_LSE (1u << 1)
+#define CPUINFO_LSE2 (1u << 2)
+
+/* Initialized with a constructor. */
+extern unsigned cpuinfo;
+
+/*
+ * We cannot rely on constructor ordering, so other constructors must
+ * use the function interface rather than the variable above.
+ */
+unsigned cpuinfo_init(void);
+
+#endif /* HOST_CPUINFO_H */
--- /dev/null
+/*
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ * Compare-and-swap for 128-bit atomic operations, generic version.
+ *
+ * Copyright (C) 2018, 2023 Linaro, Ltd.
+ *
+ * See docs/devel/atomics.rst for discussion about the guarantees each
+ * atomic primitive is meant to provide.
+ */
+
+#ifndef HOST_ATOMIC128_CAS_H
+#define HOST_ATOMIC128_CAS_H
+
+#if defined(CONFIG_ATOMIC128)
+static inline Int128 ATTRIBUTE_ATOMIC128_OPT
+atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
+{
+ __int128_t *ptr_align = __builtin_assume_aligned(ptr, 16);
+ Int128Alias r, c, n;
+
+ c.s = cmp;
+ n.s = new;
+ r.i = qatomic_cmpxchg__nocheck(ptr_align, c.i, n.i);
+ return r.s;
+}
+# define HAVE_CMPXCHG128 1
+#elif defined(CONFIG_CMPXCHG128)
+static inline Int128 ATTRIBUTE_ATOMIC128_OPT
+atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
+{
+ __int128_t *ptr_align = __builtin_assume_aligned(ptr, 16);
+ Int128Alias r, c, n;
+
+ c.s = cmp;
+ n.s = new;
+ r.i = __sync_val_compare_and_swap_16(ptr_align, c.i, n.i);
+ return r.s;
+}
+# define HAVE_CMPXCHG128 1
+#else
+/* Fallback definition that must be optimized away, or error. */
+Int128 QEMU_ERROR("unsupported atomic")
+ atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new);
+# define HAVE_CMPXCHG128 0
+#endif
+
+#endif /* HOST_ATOMIC128_CAS_H */
--- /dev/null
+/*
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ * Load/store for 128-bit atomic operations, generic version.
+ *
+ * Copyright (C) 2018, 2023 Linaro, Ltd.
+ *
+ * See docs/devel/atomics.rst for discussion about the guarantees each
+ * atomic primitive is meant to provide.
+ */
+
+#ifndef HOST_ATOMIC128_LDST_H
+#define HOST_ATOMIC128_LDST_H
+
+#if defined(CONFIG_ATOMIC128)
+# define HAVE_ATOMIC128_RO 1
+# define HAVE_ATOMIC128_RW 1
+
+static inline Int128 ATTRIBUTE_ATOMIC128_OPT
+atomic16_read_ro(const Int128 *ptr)
+{
+ const __int128_t *ptr_align = __builtin_assume_aligned(ptr, 16);
+ Int128Alias r;
+
+ r.i = qatomic_read__nocheck(ptr_align);
+ return r.s;
+}
+
+static inline Int128 ATTRIBUTE_ATOMIC128_OPT
+atomic16_read_rw(Int128 *ptr)
+{
+ return atomic16_read_ro(ptr);
+}
+
+static inline void ATTRIBUTE_ATOMIC128_OPT
+atomic16_set(Int128 *ptr, Int128 val)
+{
+ __int128_t *ptr_align = __builtin_assume_aligned(ptr, 16);
+ Int128Alias v;
+
+ v.s = val;
+ qatomic_set__nocheck(ptr_align, v.i);
+}
+
+#elif defined(CONFIG_CMPXCHG128)
+# define HAVE_ATOMIC128_RO 0
+# define HAVE_ATOMIC128_RW 1
+
+Int128 QEMU_ERROR("unsupported atomic") atomic16_read_ro(const Int128 *ptr);
+
+static inline Int128 ATTRIBUTE_ATOMIC128_OPT
+atomic16_read_rw(Int128 *ptr)
+{
+ /* Maybe replace 0 with 0, returning the old value. */
+ Int128 z = int128_make64(0);
+ return atomic16_cmpxchg(ptr, z, z);
+}
+
+static inline void ATTRIBUTE_ATOMIC128_OPT
+atomic16_set(Int128 *ptr, Int128 val)
+{
+ __int128_t *ptr_align = __builtin_assume_aligned(ptr, 16);
+ __int128_t old;
+ Int128Alias new;
+
+ new.s = val;
+ do {
+ old = *ptr_align;
+ } while (!__sync_bool_compare_and_swap_16(ptr_align, old, new.i));
+}
+
+#else
+# define HAVE_ATOMIC128_RO 0
+# define HAVE_ATOMIC128_RW 0
+
+/* Fallback definitions that must be optimized away, or error. */
+Int128 QEMU_ERROR("unsupported atomic") atomic16_read_ro(const Int128 *ptr);
+Int128 QEMU_ERROR("unsupported atomic") atomic16_read_rw(Int128 *ptr);
+void QEMU_ERROR("unsupported atomic") atomic16_set(Int128 *ptr, Int128 val);
+#endif
+
+#endif /* HOST_ATOMIC128_LDST_H */
--- /dev/null
+/*
+ * No host specific cpu indentification.
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
--- /dev/null
+/*
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ * Host specific cpu indentification for x86.
+ */
+
+#ifndef HOST_CPUINFO_H
+#define HOST_CPUINFO_H
+
+/* Digested version of <cpuid.h> */
+
+#define CPUINFO_ALWAYS (1u << 0) /* so cpuinfo is nonzero */
+#define CPUINFO_CMOV (1u << 1)
+#define CPUINFO_MOVBE (1u << 2)
+#define CPUINFO_LZCNT (1u << 3)
+#define CPUINFO_POPCNT (1u << 4)
+#define CPUINFO_BMI1 (1u << 5)
+#define CPUINFO_BMI2 (1u << 6)
+#define CPUINFO_SSE2 (1u << 7)
+#define CPUINFO_SSE4 (1u << 8)
+#define CPUINFO_AVX1 (1u << 9)
+#define CPUINFO_AVX2 (1u << 10)
+#define CPUINFO_AVX512F (1u << 11)
+#define CPUINFO_AVX512VL (1u << 12)
+#define CPUINFO_AVX512BW (1u << 13)
+#define CPUINFO_AVX512DQ (1u << 14)
+#define CPUINFO_AVX512VBMI2 (1u << 15)
+#define CPUINFO_ATOMIC_VMOVDQA (1u << 16)
+#define CPUINFO_ATOMIC_VMOVDQU (1u << 17)
+
+/* Initialized with a constructor. */
+extern unsigned cpuinfo;
+
+/*
+ * We cannot rely on constructor ordering, so other constructors must
+ * use the function interface rather than the variable above.
+ */
+unsigned cpuinfo_init(void);
+
+#endif /* HOST_CPUINFO_H */
--- /dev/null
+#include "host/include/i386/host/cpuinfo.h"
config VMXNET3_PCI
bool
- default y if PCI_DEVICES && PC_PCI
+ default y if PCI_DEVICES
depends on PCI
config SMC91C111
e1000x_inc_reg_if_not_full(s->mac_reg, TPT);
e1000x_grow_8reg_if_not_full(s->mac_reg, TOTL, s->tx.size + 4);
- s->mac_reg[GPTC] = s->mac_reg[TPT];
- s->mac_reg[GOTCL] = s->mac_reg[TOTL];
- s->mac_reg[GOTCH] = s->mac_reg[TOTH];
+ e1000x_inc_reg_if_not_full(s->mac_reg, GPTC);
+ e1000x_grow_8reg_if_not_full(s->mac_reg, GOTCL, s->tx.size + 4);
}
static void
}
static int
-receive_filter(E1000State *s, const uint8_t *buf, int size)
+receive_filter(E1000State *s, const void *buf)
{
- uint32_t rctl = s->mac_reg[RCTL];
- int isbcast = is_broadcast_ether_addr(buf);
- int ismcast = is_multicast_ether_addr(buf);
-
- if (e1000x_is_vlan_packet(buf, le16_to_cpu(s->mac_reg[VET])) &&
- e1000x_vlan_rx_filter_enabled(s->mac_reg)) {
- uint16_t vid = lduw_be_p(&PKT_GET_VLAN_HDR(buf)->h_tci);
- uint32_t vfta =
- ldl_le_p((uint32_t *)(s->mac_reg + VFTA) +
- ((vid >> E1000_VFTA_ENTRY_SHIFT) & E1000_VFTA_ENTRY_MASK));
- if ((vfta & (1 << (vid & E1000_VFTA_ENTRY_BIT_SHIFT_MASK))) == 0) {
- return 0;
- }
- }
-
- if (!isbcast && !ismcast && (rctl & E1000_RCTL_UPE)) { /* promiscuous ucast */
- return 1;
- }
-
- if (ismcast && (rctl & E1000_RCTL_MPE)) { /* promiscuous mcast */
- e1000x_inc_reg_if_not_full(s->mac_reg, MPRC);
- return 1;
- }
-
- if (isbcast && (rctl & E1000_RCTL_BAM)) { /* broadcast enabled */
- e1000x_inc_reg_if_not_full(s->mac_reg, BPRC);
- return 1;
- }
-
- return e1000x_rx_group_filter(s->mac_reg, buf);
+ return (!e1000x_is_vlan_packet(buf, s->mac_reg[VET]) ||
+ e1000x_rx_vlan_filter(s->mac_reg, PKT_GET_VLAN_HDR(buf))) &&
+ e1000x_rx_group_filter(s->mac_reg, buf);
}
static void
size_t desc_offset;
size_t desc_size;
size_t total_size;
+ eth_pkt_types_e pkt_type;
if (!e1000x_hw_rx_enabled(s->mac_reg)) {
return -1;
return size;
}
- if (!receive_filter(s, filter_buf, size)) {
+ if (!receive_filter(s, filter_buf)) {
return size;
}
size -= 4;
}
+ pkt_type = get_eth_packet_type(PKT_GET_ETH_HDR(filter_buf));
rdh_start = s->mac_reg[RDH];
desc_offset = 0;
total_size = size + e1000x_fcs_len(s->mac_reg);
}
} while (desc_offset < total_size);
- e1000x_update_rx_total_stats(s->mac_reg, size, total_size);
+ e1000x_update_rx_total_stats(s->mac_reg, pkt_type, size, total_size);
n = E1000_ICS_RXT0;
if ((rdt = s->mac_reg[RDT]) < s->mac_reg[RDH])
timer->running = false;
- if (msi_enabled(timer->core->owner)) {
- trace_e1000e_irq_msi_notify_postponed();
- /* Clear msi_causes_pending to fire MSI eventually */
- timer->core->msi_causes_pending = 0;
- e1000e_set_interrupt_cause(timer->core, 0);
- } else {
- trace_e1000e_irq_legacy_notify_postponed();
- e1000e_set_interrupt_cause(timer->core, 0);
+ if (timer->core->mac[IMS] & timer->core->mac[ICR]) {
+ if (msi_enabled(timer->core->owner)) {
+ trace_e1000e_irq_msi_notify_postponed();
+ msi_notify(timer->core->owner, 0);
+ } else {
+ trace_e1000e_irq_legacy_notify_postponed();
+ e1000e_raise_legacy_irq(timer->core);
+ }
}
}
e1000e_intrmgr_fire_all_timers(E1000ECore *core)
{
int i;
- uint32_t val = e1000e_intmgr_collect_delayed_causes(core);
-
- trace_e1000e_irq_adding_delayed_causes(val, core->mac[ICR]);
- core->mac[ICR] |= val;
if (core->itr.running) {
timer_del(core->itr.timer);
ip6info->rss_ex_dst_valid,
ip6info->rss_ex_src_valid,
core->mac[MRQC],
- E1000_MRQC_EN_TCPIPV6(core->mac[MRQC]),
+ E1000_MRQC_EN_TCPIPV6EX(core->mac[MRQC]),
E1000_MRQC_EN_IPV6EX(core->mac[MRQC]),
E1000_MRQC_EN_IPV6(core->mac[MRQC]));
ip6info->rss_ex_src_valid))) {
if (l4hdr_proto == ETH_L4_HDR_PROTO_TCP &&
- E1000_MRQC_EN_TCPIPV6(core->mac[MRQC])) {
- return E1000_MRQ_RSS_TYPE_IPV6TCP;
+ E1000_MRQC_EN_TCPIPV6EX(core->mac[MRQC])) {
+ return E1000_MRQ_RSS_TYPE_IPV6TCPEX;
}
if (E1000_MRQC_EN_IPV6EX(core->mac[MRQC])) {
case E1000_MRQ_RSS_TYPE_IPV4TCP:
type = NetPktRssIpV4Tcp;
break;
- case E1000_MRQ_RSS_TYPE_IPV6TCP:
+ case E1000_MRQ_RSS_TYPE_IPV6TCPEX:
type = NetPktRssIpV6TcpEx;
break;
case E1000_MRQ_RSS_TYPE_IPV6:
g_assert_not_reached();
}
- core->mac[GPTC] = core->mac[TPT];
- core->mac[GOTCL] = core->mac[TOTL];
- core->mac[GOTCH] = core->mac[TOTH];
+ e1000x_inc_reg_if_not_full(core->mac, GPTC);
+ e1000x_grow_8reg_if_not_full(core->mac, GOTCL, tot_len);
}
static void
addr = le64_to_cpu(dp->buffer_addr);
if (!tx->skip_cp) {
- if (!net_tx_pkt_add_raw_fragment(tx->tx_pkt, addr, split_size)) {
+ if (!net_tx_pkt_add_raw_fragment_pci(tx->tx_pkt, core->owner,
+ addr, split_size)) {
tx->skip_cp = true;
}
}
}
tx->skip_cp = false;
- net_tx_pkt_reset(tx->tx_pkt, core->owner);
+ net_tx_pkt_reset(tx->tx_pkt, net_tx_pkt_unmap_frag_pci, core->owner);
tx->sum_needed = 0;
tx->cptse = 0;
if (!ide || !e1000e_intrmgr_delay_tx_causes(core, &cause)) {
e1000e_set_interrupt_cause(core, cause);
}
+
+ net_tx_pkt_reset(txr->tx->tx_pkt, net_tx_pkt_unmap_frag_pci, core->owner);
}
static bool
}
static bool
-e1000e_receive_filter(E1000ECore *core, const uint8_t *buf, int size)
+e1000e_receive_filter(E1000ECore *core, const void *buf)
{
- uint32_t rctl = core->mac[RCTL];
-
- if (e1000x_is_vlan_packet(buf, core->mac[VET]) &&
- e1000x_vlan_rx_filter_enabled(core->mac)) {
- uint16_t vid = lduw_be_p(&PKT_GET_VLAN_HDR(buf)->h_tci);
- uint32_t vfta =
- ldl_le_p((uint32_t *)(core->mac + VFTA) +
- ((vid >> E1000_VFTA_ENTRY_SHIFT) & E1000_VFTA_ENTRY_MASK));
- if ((vfta & (1 << (vid & E1000_VFTA_ENTRY_BIT_SHIFT_MASK))) == 0) {
- trace_e1000e_rx_flt_vlan_mismatch(vid);
- return false;
- } else {
- trace_e1000e_rx_flt_vlan_match(vid);
- }
- }
-
- switch (net_rx_pkt_get_packet_type(core->rx_pkt)) {
- case ETH_PKT_UCAST:
- if (rctl & E1000_RCTL_UPE) {
- return true; /* promiscuous ucast */
- }
- break;
-
- case ETH_PKT_BCAST:
- if (rctl & E1000_RCTL_BAM) {
- return true; /* broadcast enabled */
- }
- break;
-
- case ETH_PKT_MCAST:
- if (rctl & E1000_RCTL_MPE) {
- return true; /* promiscuous mcast */
- }
- break;
-
- default:
- g_assert_not_reached();
- }
-
- return e1000x_rx_group_filter(core->mac, buf);
+ return (!e1000x_is_vlan_packet(buf, core->mac[VET]) ||
+ e1000x_rx_vlan_filter(core->mac, PKT_GET_VLAN_HDR(buf))) &&
+ e1000x_rx_group_filter(core->mac, buf);
}
static inline void
return;
}
+ if (l4hdr_proto != ETH_L4_HDR_PROTO_TCP &&
+ l4hdr_proto != ETH_L4_HDR_PROTO_UDP) {
+ return;
+ }
+
if (!net_rx_pkt_validate_l4_csum(pkt, &csum_valid)) {
trace_e1000e_rx_metadata_l4_csum_validation_failed();
return;
trace_e1000e_rx_metadata_l4_cso_disabled();
}
- trace_e1000e_rx_metadata_status_flags(*status_flags);
-
func_exit:
+ trace_e1000e_rx_metadata_status_flags(*status_flags);
*status_flags = cpu_to_le32(*status_flags);
}
}
static void
-e1000e_update_rx_stats(E1000ECore *core,
- size_t data_size,
- size_t data_fcs_size)
+e1000e_update_rx_stats(E1000ECore *core, size_t pkt_size, size_t pkt_fcs_size)
{
- e1000x_update_rx_total_stats(core->mac, data_size, data_fcs_size);
-
- switch (net_rx_pkt_get_packet_type(core->rx_pkt)) {
- case ETH_PKT_BCAST:
- e1000x_inc_reg_if_not_full(core->mac, BPRC);
- break;
-
- case ETH_PKT_MCAST:
- e1000x_inc_reg_if_not_full(core->mac, MPRC);
- break;
-
- default:
- break;
- }
+ eth_pkt_types_e pkt_type = net_rx_pkt_get_packet_type(core->rx_pkt);
+ e1000x_update_rx_total_stats(core->mac, pkt_type, pkt_size, pkt_fcs_size);
}
static inline bool
e1000e_receive_internal(E1000ECore *core, const struct iovec *iov, int iovcnt,
bool has_vnet)
{
- static const int maximum_ethernet_hdr_len = (ETH_HLEN + 4);
-
- uint32_t n = 0;
- uint8_t min_buf[ETH_ZLEN];
+ uint32_t causes = 0;
+ uint8_t buf[ETH_ZLEN];
struct iovec min_iov;
- uint8_t *filter_buf;
size_t size, orig_size;
size_t iov_ofs = 0;
E1000E_RxRing rxr;
net_rx_pkt_unset_vhdr(core->rx_pkt);
}
- filter_buf = iov->iov_base + iov_ofs;
orig_size = iov_size(iov, iovcnt);
size = orig_size - iov_ofs;
/* Pad to minimum Ethernet frame length */
- if (size < sizeof(min_buf)) {
- iov_to_buf(iov, iovcnt, iov_ofs, min_buf, size);
- memset(&min_buf[size], 0, sizeof(min_buf) - size);
+ if (size < sizeof(buf)) {
+ iov_to_buf(iov, iovcnt, iov_ofs, buf, size);
+ memset(&buf[size], 0, sizeof(buf) - size);
e1000x_inc_reg_if_not_full(core->mac, RUC);
- min_iov.iov_base = filter_buf = min_buf;
- min_iov.iov_len = size = sizeof(min_buf);
+ min_iov.iov_base = buf;
+ min_iov.iov_len = size = sizeof(buf);
iovcnt = 1;
iov = &min_iov;
iov_ofs = 0;
- } else if (iov->iov_len < maximum_ethernet_hdr_len) {
- /* This is very unlikely, but may happen. */
- iov_to_buf(iov, iovcnt, iov_ofs, min_buf, maximum_ethernet_hdr_len);
- filter_buf = min_buf;
+ } else {
+ iov_to_buf(iov, iovcnt, iov_ofs, buf, ETH_HLEN + 4);
}
/* Discard oversized packets if !LPE and !SBP. */
}
net_rx_pkt_set_packet_type(core->rx_pkt,
- get_eth_packet_type(PKT_GET_ETH_HDR(filter_buf)));
+ get_eth_packet_type(PKT_GET_ETH_HDR(buf)));
- if (!e1000e_receive_filter(core, filter_buf, size)) {
+ if (!e1000e_receive_filter(core, buf)) {
trace_e1000e_rx_flt_dropped();
return orig_size;
}
net_rx_pkt_attach_iovec_ex(core->rx_pkt, iov, iovcnt, iov_ofs,
- e1000x_vlan_enabled(core->mac), core->mac[VET]);
+ e1000x_vlan_enabled(core->mac) ? 0 : -1,
+ core->mac[VET], 0);
e1000e_rss_parse_packet(core, core->rx_pkt, &rss_info);
e1000e_rx_ring_init(core, &rxr, rss_info.queue);
/* Perform small receive detection (RSRPD) */
if (total_size < core->mac[RSRPD]) {
- n |= E1000_ICS_SRPD;
+ causes |= E1000_ICS_SRPD;
}
/* Perform ACK receive detection */
if (!(core->mac[RFCTL] & E1000_RFCTL_ACK_DIS) &&
(e1000e_is_tcp_ack(core, core->rx_pkt))) {
- n |= E1000_ICS_ACK;
+ causes |= E1000_ICS_ACK;
}
/* Check if receive descriptor minimum threshold hit */
rdmts_hit = e1000e_rx_descr_threshold_hit(core, rxr.i);
- n |= e1000e_rx_wb_interrupt_cause(core, rxr.i->idx, rdmts_hit);
+ causes |= e1000e_rx_wb_interrupt_cause(core, rxr.i->idx, rdmts_hit);
trace_e1000e_rx_written_to_guest(rxr.i->idx);
} else {
- n |= E1000_ICS_RXO;
+ causes |= E1000_ICS_RXO;
retval = 0;
trace_e1000e_rx_not_written_to_guest(rxr.i->idx);
}
- if (!e1000e_intrmgr_delay_rx_causes(core, &n)) {
- trace_e1000e_rx_interrupt_set(n);
- e1000e_set_interrupt_cause(core, n);
+ if (!e1000e_intrmgr_delay_rx_causes(core, &causes)) {
+ trace_e1000e_rx_interrupt_set(causes);
+ e1000e_set_interrupt_cause(core, causes);
} else {
- trace_e1000e_rx_interrupt_delayed(n);
+ trace_e1000e_rx_interrupt_delayed(causes);
}
return retval;
}
};
-static inline void
-e1000e_clear_ims_bits(E1000ECore *core, uint32_t bits)
-{
- trace_e1000e_irq_clear_ims(bits, core->mac[IMS], core->mac[IMS] & ~bits);
- core->mac[IMS] &= ~bits;
-}
-
static inline bool
e1000e_postpone_interrupt(E1000IntrDelayTimer *timer)
{
effective_eiac = core->mac[EIAC] & cause;
core->mac[ICR] &= ~effective_eiac;
- core->msi_causes_pending &= ~effective_eiac;
if (!(core->mac[CTRL_EXT] & E1000_CTRL_EXT_IAME)) {
core->mac[IMS] &= ~effective_eiac;
trace_e1000e_irq_fix_icr_asserted(core->mac[ICR]);
}
-static void
-e1000e_send_msi(E1000ECore *core, bool msix)
+static void e1000e_raise_interrupts(E1000ECore *core,
+ size_t index, uint32_t causes)
{
- uint32_t causes = core->mac[ICR] & core->mac[IMS] & ~E1000_ICR_ASSERTED;
-
- core->msi_causes_pending &= causes;
- causes ^= core->msi_causes_pending;
- if (causes == 0) {
- return;
- }
- core->msi_causes_pending |= causes;
+ bool is_msix = msix_enabled(core->owner);
+ uint32_t old_causes = core->mac[IMS] & core->mac[ICR];
+ uint32_t raised_causes;
- if (msix) {
- e1000e_msix_notify(core, causes);
- } else {
- if (!e1000e_itr_should_postpone(core)) {
- trace_e1000e_irq_msi_notify(causes);
- msi_notify(core->owner, 0);
- }
- }
-}
+ trace_e1000e_irq_set(index << 2,
+ core->mac[index], core->mac[index] | causes);
-static void
-e1000e_update_interrupt_state(E1000ECore *core)
-{
- bool interrupts_pending;
- bool is_msix = msix_enabled(core->owner);
+ core->mac[index] |= causes;
/* Set ICR[OTHER] for MSI-X */
if (is_msix) {
*/
core->mac[ICS] = core->mac[ICR];
- interrupts_pending = (core->mac[IMS] & core->mac[ICR]) ? true : false;
- if (!interrupts_pending) {
- core->msi_causes_pending = 0;
- }
-
trace_e1000e_irq_pending_interrupts(core->mac[ICR] & core->mac[IMS],
core->mac[ICR], core->mac[IMS]);
- if (is_msix || msi_enabled(core->owner)) {
- if (interrupts_pending) {
- e1000e_send_msi(core, is_msix);
- }
- } else {
- if (interrupts_pending) {
- if (!e1000e_itr_should_postpone(core)) {
- e1000e_raise_legacy_irq(core);
- }
+ raised_causes = core->mac[IMS] & core->mac[ICR] & ~old_causes;
+ if (!raised_causes) {
+ return;
+ }
+
+ if (is_msix) {
+ e1000e_msix_notify(core, raised_causes & ~E1000_ICR_ASSERTED);
+ } else if (!e1000e_itr_should_postpone(core)) {
+ if (msi_enabled(core->owner)) {
+ trace_e1000e_irq_msi_notify(raised_causes);
+ msi_notify(core->owner, 0);
} else {
- e1000e_lower_legacy_irq(core);
+ e1000e_raise_legacy_irq(core);
}
}
}
-static void
-e1000e_set_interrupt_cause(E1000ECore *core, uint32_t val)
+static void e1000e_lower_interrupts(E1000ECore *core,
+ size_t index, uint32_t causes)
{
- trace_e1000e_irq_set_cause_entry(val, core->mac[ICR]);
+ trace_e1000e_irq_clear(index << 2,
+ core->mac[index], core->mac[index] & ~causes);
- val |= e1000e_intmgr_collect_delayed_causes(core);
- core->mac[ICR] |= val;
+ core->mac[index] &= ~causes;
+
+ /*
+ * Make sure ICR and ICS registers have the same value.
+ * The spec says that the ICS register is write-only. However in practice,
+ * on real hardware ICS is readable, and for reads it has the same value as
+ * ICR (except that ICS does not have the clear on read behaviour of ICR).
+ *
+ * The VxWorks PRO/1000 driver uses this behaviour.
+ */
+ core->mac[ICS] = core->mac[ICR];
+
+ trace_e1000e_irq_pending_interrupts(core->mac[ICR] & core->mac[IMS],
+ core->mac[ICR], core->mac[IMS]);
- trace_e1000e_irq_set_cause_exit(val, core->mac[ICR]);
+ if (!(core->mac[IMS] & core->mac[ICR]) &&
+ !msix_enabled(core->owner) && !msi_enabled(core->owner)) {
+ e1000e_lower_legacy_irq(core);
+ }
+}
- e1000e_update_interrupt_state(core);
+static void
+e1000e_set_interrupt_cause(E1000ECore *core, uint32_t val)
+{
+ val |= e1000e_intmgr_collect_delayed_causes(core);
+ e1000e_raise_interrupts(core, ICR, val);
}
static inline void
static void
e1000e_set_icr(E1000ECore *core, int index, uint32_t val)
{
- uint32_t icr = 0;
if ((core->mac[ICR] & E1000_ICR_ASSERTED) &&
(core->mac[CTRL_EXT] & E1000_CTRL_EXT_IAME)) {
trace_e1000e_irq_icr_process_iame();
- e1000e_clear_ims_bits(core, core->mac[IAM]);
+ e1000e_lower_interrupts(core, IMS, core->mac[IAM]);
}
- icr = core->mac[ICR] & ~val;
/*
* Windows driver expects that the "receive overrun" bit and other
* ones to be cleared when the "Other" bit (#24) is cleared.
*/
- icr = (val & E1000_ICR_OTHER) ? (icr & ~E1000_ICR_OTHER_CAUSES) : icr;
- trace_e1000e_irq_icr_write(val, core->mac[ICR], icr);
- core->mac[ICR] = icr;
- e1000e_update_interrupt_state(core);
+ if (val & E1000_ICR_OTHER) {
+ val |= E1000_ICR_OTHER_CAUSES;
+ }
+ e1000e_lower_interrupts(core, ICR, val);
}
static void
e1000e_set_imc(E1000ECore *core, int index, uint32_t val)
{
trace_e1000e_irq_ims_clear_set_imc(val);
- e1000e_clear_ims_bits(core, val);
- e1000e_update_interrupt_state(core);
+ e1000e_lower_interrupts(core, IMS, val);
}
static void
uint32_t valid_val = val & ims_valid_mask;
- trace_e1000e_irq_set_ims(val, core->mac[IMS], core->mac[IMS] | valid_val);
- core->mac[IMS] |= valid_val;
-
if ((valid_val & ims_ext_mask) &&
(core->mac[CTRL_EXT] & E1000_CTRL_EXT_PBA_CLR) &&
msix_enabled(core->owner)) {
e1000e_intrmgr_fire_all_timers(core);
}
- e1000e_update_interrupt_state(core);
+ e1000e_raise_interrupts(core, IMS, valid_val);
}
static void
e1000e_mac_icr_read(E1000ECore *core, int index)
{
uint32_t ret = core->mac[ICR];
- trace_e1000e_irq_icr_read_entry(ret);
if (core->mac[IMS] == 0) {
trace_e1000e_irq_icr_clear_zero_ims();
- core->mac[ICR] = 0;
+ e1000e_lower_interrupts(core, ICR, 0xffffffff);
}
if (!msix_enabled(core->owner)) {
trace_e1000e_irq_icr_clear_nonmsix_icr_read();
- core->mac[ICR] = 0;
+ e1000e_lower_interrupts(core, ICR, 0xffffffff);
}
if ((core->mac[ICR] & E1000_ICR_ASSERTED) &&
(core->mac[CTRL_EXT] & E1000_CTRL_EXT_IAME)) {
trace_e1000e_irq_icr_clear_iame();
- core->mac[ICR] = 0;
+ e1000e_lower_interrupts(core, ICR, 0xffffffff);
trace_e1000e_irq_icr_process_iame();
- e1000e_clear_ims_bits(core, core->mac[IAM]);
+ e1000e_lower_interrupts(core, IMS, core->mac[IAM]);
}
- trace_e1000e_irq_icr_read_exit(core->mac[ICR]);
- e1000e_update_interrupt_state(core);
return ret;
}
qemu_add_vm_change_state_handler(e1000e_vm_state_change, core);
for (i = 0; i < E1000E_NUM_QUEUES; i++) {
- net_tx_pkt_init(&core->tx[i].tx_pkt, core->owner, E1000E_MAX_TX_FRAGS);
+ net_tx_pkt_init(&core->tx[i].tx_pkt, E1000E_MAX_TX_FRAGS);
}
net_rx_pkt_init(&core->rx_pkt);
qemu_del_vm_change_state_handler(core->vmstate);
for (i = 0; i < E1000E_NUM_QUEUES; i++) {
- net_tx_pkt_reset(core->tx[i].tx_pkt, core->owner);
net_tx_pkt_uninit(core->tx[i].tx_pkt);
}
e1000x_reset_mac_addr(core->owner_nic, core->mac, core->permanent_mac);
for (i = 0; i < ARRAY_SIZE(core->tx); i++) {
- net_tx_pkt_reset(core->tx[i].tx_pkt, core->owner);
memset(&core->tx[i].props, 0, sizeof(core->tx[i].props));
core->tx[i].skip_cp = false;
}
PCIDevice *owner;
void (*owner_start_recv)(PCIDevice *d);
- uint32_t msi_causes_pending;
-
int64_t timadj;
};
return res;
}
-bool e1000x_rx_group_filter(uint32_t *mac, const uint8_t *buf)
+bool e1000x_rx_vlan_filter(uint32_t *mac, const struct vlan_header *vhdr)
+{
+ if (e1000x_vlan_rx_filter_enabled(mac)) {
+ uint16_t vid = lduw_be_p(&vhdr->h_tci);
+ uint32_t vfta =
+ ldl_le_p((uint32_t *)(mac + VFTA) +
+ ((vid >> E1000_VFTA_ENTRY_SHIFT) & E1000_VFTA_ENTRY_MASK));
+ if ((vfta & (1 << (vid & E1000_VFTA_ENTRY_BIT_SHIFT_MASK))) == 0) {
+ trace_e1000x_rx_flt_vlan_mismatch(vid);
+ return false;
+ }
+
+ trace_e1000x_rx_flt_vlan_match(vid);
+ }
+
+ return true;
+}
+
+bool e1000x_rx_group_filter(uint32_t *mac, const struct eth_header *ehdr)
{
static const int mta_shift[] = { 4, 3, 2, 0 };
uint32_t f, ra[2], *rp, rctl = mac[RCTL];
+ if (is_broadcast_ether_addr(ehdr->h_dest)) {
+ if (rctl & E1000_RCTL_BAM) {
+ return true;
+ }
+ } else if (is_multicast_ether_addr(ehdr->h_dest)) {
+ if (rctl & E1000_RCTL_MPE) {
+ return true;
+ }
+ } else {
+ if (rctl & E1000_RCTL_UPE) {
+ return true;
+ }
+ }
+
for (rp = mac + RA; rp < mac + RA + 32; rp += 2) {
if (!(rp[1] & E1000_RAH_AV)) {
continue;
}
ra[0] = cpu_to_le32(rp[0]);
ra[1] = cpu_to_le32(rp[1]);
- if (!memcmp(buf, (uint8_t *)ra, ETH_ALEN)) {
+ if (!memcmp(ehdr->h_dest, (uint8_t *)ra, ETH_ALEN)) {
trace_e1000x_rx_flt_ucast_match((int)(rp - mac - RA) / 2,
- MAC_ARG(buf));
+ MAC_ARG(ehdr->h_dest));
return true;
}
}
- trace_e1000x_rx_flt_ucast_mismatch(MAC_ARG(buf));
+ trace_e1000x_rx_flt_ucast_mismatch(MAC_ARG(ehdr->h_dest));
f = mta_shift[(rctl >> E1000_RCTL_MO_SHIFT) & 3];
- f = (((buf[5] << 8) | buf[4]) >> f) & 0xfff;
+ f = (((ehdr->h_dest[5] << 8) | ehdr->h_dest[4]) >> f) & 0xfff;
if (mac[MTA + (f >> 5)] & (1 << (f & 0x1f))) {
- e1000x_inc_reg_if_not_full(mac, MPRC);
return true;
}
- trace_e1000x_rx_flt_inexact_mismatch(MAC_ARG(buf),
+ trace_e1000x_rx_flt_inexact_mismatch(MAC_ARG(ehdr->h_dest),
(rctl >> E1000_RCTL_MO_SHIFT) & 3,
f >> 5,
mac[MTA + (f >> 5)]);
bool e1000x_is_oversized(uint32_t *mac, size_t size)
{
+ size_t header_size = sizeof(struct eth_header) + sizeof(struct vlan_header);
/* this is the size past which hardware will
drop packets when setting LPE=0 */
- static const int maximum_ethernet_vlan_size = 1522;
+ size_t maximum_short_size = header_size + ETH_MTU;
/* this is the size past which hardware will
drop packets when setting LPE=1 */
- static const int maximum_ethernet_lpe_size = 16 * KiB;
+ size_t maximum_large_size = 16 * KiB - ETH_FCS_LEN;
- if ((size > maximum_ethernet_lpe_size ||
- (size > maximum_ethernet_vlan_size
- && !(mac[RCTL] & E1000_RCTL_LPE)))
+ if ((size > maximum_large_size ||
+ (size > maximum_short_size && !(mac[RCTL] & E1000_RCTL_LPE)))
&& !(mac[RCTL] & E1000_RCTL_SBP)) {
e1000x_inc_reg_if_not_full(mac, ROC);
trace_e1000x_rx_oversized(size);
void
e1000x_update_rx_total_stats(uint32_t *mac,
- size_t data_size,
- size_t data_fcs_size)
+ eth_pkt_types_e pkt_type,
+ size_t pkt_size,
+ size_t pkt_fcs_size)
{
static const int PRCregs[6] = { PRC64, PRC127, PRC255, PRC511,
PRC1023, PRC1522 };
- e1000x_increase_size_stats(mac, PRCregs, data_fcs_size);
+ e1000x_increase_size_stats(mac, PRCregs, pkt_fcs_size);
e1000x_inc_reg_if_not_full(mac, TPR);
- mac[GPRC] = mac[TPR];
+ e1000x_inc_reg_if_not_full(mac, GPRC);
/* TOR - Total Octets Received:
* This register includes bytes received in a packet from the <Destination
* Address> field through the <CRC> field, inclusively.
* Always include FCS length (4) in size.
*/
- e1000x_grow_8reg_if_not_full(mac, TORL, data_size + 4);
- mac[GORCL] = mac[TORL];
- mac[GORCH] = mac[TORH];
+ e1000x_grow_8reg_if_not_full(mac, TORL, pkt_size + 4);
+ e1000x_grow_8reg_if_not_full(mac, GORCL, pkt_size + 4);
+
+ switch (pkt_type) {
+ case ETH_PKT_BCAST:
+ e1000x_inc_reg_if_not_full(mac, BPRC);
+ break;
+
+ case ETH_PKT_MCAST:
+ e1000x_inc_reg_if_not_full(mac, MPRC);
+ break;
+
+ default:
+ break;
+ }
}
void
}
void e1000x_update_rx_total_stats(uint32_t *mac,
- size_t data_size,
- size_t data_fcs_size);
+ eth_pkt_types_e pkt_type,
+ size_t pkt_size,
+ size_t pkt_fcs_size);
void e1000x_core_prepare_eeprom(uint16_t *eeprom,
const uint16_t *templ,
bool e1000x_is_vlan_packet(const void *buf, uint16_t vet);
-bool e1000x_rx_group_filter(uint32_t *mac, const uint8_t *buf);
+bool e1000x_rx_vlan_filter(uint32_t *mac, const struct vlan_header *vhdr);
+
+bool e1000x_rx_group_filter(uint32_t *mac, const struct eth_header *ehdr);
bool e1000x_hw_rx_enabled(uint32_t *mac);
#define E1000_RETA_IDX(hash) ((hash) & (BIT(7) - 1))
#define E1000_RETA_VAL(reta, hash) (((uint8_t *)(reta))[E1000_RETA_IDX(hash)])
-#define E1000_MRQC_EN_TCPIPV4(mrqc) ((mrqc) & BIT(16))
-#define E1000_MRQC_EN_IPV4(mrqc) ((mrqc) & BIT(17))
-#define E1000_MRQC_EN_TCPIPV6(mrqc) ((mrqc) & BIT(18))
-#define E1000_MRQC_EN_IPV6EX(mrqc) ((mrqc) & BIT(19))
-#define E1000_MRQC_EN_IPV6(mrqc) ((mrqc) & BIT(20))
-
-#define E1000_MRQ_RSS_TYPE_NONE (0)
-#define E1000_MRQ_RSS_TYPE_IPV4TCP (1)
-#define E1000_MRQ_RSS_TYPE_IPV4 (2)
-#define E1000_MRQ_RSS_TYPE_IPV6TCP (3)
-#define E1000_MRQ_RSS_TYPE_IPV6EX (4)
-#define E1000_MRQ_RSS_TYPE_IPV6 (5)
+#define E1000_MRQC_EN_TCPIPV4(mrqc) ((mrqc) & BIT(16))
+#define E1000_MRQC_EN_IPV4(mrqc) ((mrqc) & BIT(17))
+#define E1000_MRQC_EN_TCPIPV6EX(mrqc) ((mrqc) & BIT(18))
+#define E1000_MRQC_EN_IPV6EX(mrqc) ((mrqc) & BIT(19))
+#define E1000_MRQC_EN_IPV6(mrqc) ((mrqc) & BIT(20))
+
+#define E1000_MRQ_RSS_TYPE_NONE (0)
+#define E1000_MRQ_RSS_TYPE_IPV4TCP (1)
+#define E1000_MRQ_RSS_TYPE_IPV4 (2)
+#define E1000_MRQ_RSS_TYPE_IPV6TCPEX (3)
+#define E1000_MRQ_RSS_TYPE_IPV6EX (4)
+#define E1000_MRQ_RSS_TYPE_IPV6 (5)
#define E1000_ICR_ASSERTED BIT(31)
#define E1000_EIAC_MASK 0x01F00000
pcie_ari_init(pci_dev, 0x150, 1);
- pcie_sriov_pf_init(pci_dev, IGB_CAP_SRIOV_OFFSET, "igbvf",
+ pcie_sriov_pf_init(pci_dev, IGB_CAP_SRIOV_OFFSET, TYPE_IGBVF,
IGB_82576_VF_DEV_ID, IGB_MAX_VF_FUNCTIONS, IGB_MAX_VF_FUNCTIONS,
IGB_VF_OFFSET, IGB_VF_STRIDE);
- pcie_sriov_pf_init_vf_bar(pci_dev, 0,
+ pcie_sriov_pf_init_vf_bar(pci_dev, IGBVF_MMIO_BAR_IDX,
PCI_BASE_ADDRESS_MEM_TYPE_64 | PCI_BASE_ADDRESS_MEM_PREFETCH,
- 16 * KiB);
- pcie_sriov_pf_init_vf_bar(pci_dev, 3,
+ IGBVF_MMIO_SIZE);
+ pcie_sriov_pf_init_vf_bar(pci_dev, IGBVF_MSIX_BAR_IDX,
PCI_BASE_ADDRESS_MEM_TYPE_64 | PCI_BASE_ADDRESS_MEM_PREFETCH,
- 16 * KiB);
+ IGBVF_MSIX_SIZE);
igb_init_net_peer(s, pci_dev, macaddr);
#include "igb_regs.h"
+#define TYPE_IGBVF "igbvf"
+
+#define IGBVF_MMIO_BAR_IDX (0)
+#define IGBVF_MSIX_BAR_IDX (3)
+
+#define IGBVF_MMIO_SIZE (16 * 1024)
+#define IGBVF_MSIX_SIZE (16 * 1024)
+
#define defreg(x) x = (E1000_##x >> 2)
#define defreg_indexed(x, i) x##i = (E1000_##x(i) >> 2)
#define defreg_indexeda(x, i) x##i##_A = (E1000_##x##_A(i) >> 2)
defreg_indexeda(x, 0), defreg_indexeda(x, 1), \
defreg_indexeda(x, 2), defreg_indexeda(x, 3)
-#define defregv(x) defreg_indexed(x, 0), defreg_indexed(x, 1), \
+#define defreg8(x) defreg_indexed(x, 0), defreg_indexed(x, 1), \
defreg_indexed(x, 2), defreg_indexed(x, 3), \
defreg_indexed(x, 4), defreg_indexed(x, 5), \
defreg_indexed(x, 6), defreg_indexed(x, 7)
defreg(EICS), defreg(EIMS), defreg(EIMC), defreg(EIAM),
defreg(EICR), defreg(IVAR_MISC), defreg(GPIE),
+ defreg(TSYNCRXCFG), defreg8(ETQF),
+
defreg(RXPBS), defregd(RDBAL), defregd(RDBAH), defregd(RDLEN),
defregd(SRRCTL), defregd(RDH), defregd(RDT),
defregd(RXDCTL), defregd(RXCTL), defregd(RQDPC), defreg(RA2),
defreg(VT_CTL),
- defregv(P2VMAILBOX), defregv(V2PMAILBOX), defreg(MBVFICR), defreg(MBVFIMR),
+ defreg8(P2VMAILBOX), defreg8(V2PMAILBOX), defreg(MBVFICR), defreg(MBVFIMR),
defreg(VFLRE), defreg(VFRE), defreg(VFTE), defreg(WVBR),
defreg(QDE), defreg(DTXSWC), defreg_indexed(VLVF, 0),
- defregv(VMOLR), defreg(RPLOLR), defregv(VMBMEM), defregv(VMVIR),
+ defreg8(VMOLR), defreg(RPLOLR), defreg8(VMBMEM), defreg8(VMVIR),
- defregv(PVTCTRL), defregv(PVTEICS), defregv(PVTEIMS), defregv(PVTEIMC),
- defregv(PVTEIAC), defregv(PVTEIAM), defregv(PVTEICR), defregv(PVFGPRC),
- defregv(PVFGPTC), defregv(PVFGORC), defregv(PVFGOTC), defregv(PVFMPRC),
- defregv(PVFGPRLBC), defregv(PVFGPTLBC), defregv(PVFGORLBC), defregv(PVFGOTLBC),
+ defreg8(PVTCTRL), defreg8(PVTEICS), defreg8(PVTEIMS), defreg8(PVTEIMC),
+ defreg8(PVTEIAC), defreg8(PVTEIAM), defreg8(PVTEICR), defreg8(PVFGPRC),
+ defreg8(PVFGPTC), defreg8(PVFGORC), defreg8(PVFGOTC), defreg8(PVFMPRC),
+ defreg8(PVFGPRLBC), defreg8(PVFGPTLBC), defreg8(PVFGORLBC), defreg8(PVFGOTLBC),
defreg(MTA_A),
NetClientState *nc;
} IGBTxPktVmdqCallbackContext;
+typedef struct L2Header {
+ struct eth_header eth;
+ struct vlan_header vlan[2];
+} L2Header;
+
+typedef struct PTP2 {
+ uint8_t message_id_transport_specific;
+ uint8_t version_ptp;
+ uint16_t message_length;
+ uint8_t subdomain_number;
+ uint8_t reserved0;
+ uint16_t flags;
+ uint64_t correction;
+ uint8_t reserved1[5];
+ uint8_t source_communication_technology;
+ uint32_t source_uuid_lo;
+ uint16_t source_uuid_hi;
+ uint16_t source_port_id;
+ uint16_t sequence_id;
+ uint8_t control;
+ uint8_t log_message_period;
+} PTP2;
+
static ssize_t
igb_receive_internal(IGBCore *core, const struct iovec *iov, int iovcnt,
bool has_vnet, bool *external_tx);
-static inline void
-igb_set_interrupt_cause(IGBCore *core, uint32_t val);
-
-static void igb_update_interrupt_state(IGBCore *core);
+static void igb_raise_interrupts(IGBCore *core, size_t index, uint32_t causes);
static void igb_reset(IGBCore *core, bool sw);
static inline void
pci_set_irq(core->owner, 0);
}
-static void igb_msix_notify(IGBCore *core, unsigned int vector)
+static void igb_msix_notify(IGBCore *core, unsigned int cause)
{
PCIDevice *dev = core->owner;
uint16_t vfn;
+ uint32_t effective_eiac;
+ unsigned int vector;
- vfn = 8 - (vector + 2) / IGBVF_MSIX_VEC_NUM;
+ vfn = 8 - (cause + 2) / IGBVF_MSIX_VEC_NUM;
if (vfn < pcie_sriov_num_vfs(core->owner)) {
dev = pcie_sriov_get_vf_at_index(core->owner, vfn);
assert(dev);
- vector = (vector + 2) % IGBVF_MSIX_VEC_NUM;
- } else if (vector >= IGB_MSIX_VEC_NUM) {
+ vector = (cause + 2) % IGBVF_MSIX_VEC_NUM;
+ } else if (cause >= IGB_MSIX_VEC_NUM) {
qemu_log_mask(LOG_GUEST_ERROR,
"igb: Tried to use vector unavailable for PF");
return;
+ } else {
+ vector = cause;
}
msix_notify(dev, vector);
+
+ trace_e1000e_irq_icr_clear_eiac(core->mac[EICR], core->mac[EIAC]);
+ effective_eiac = core->mac[EIAC] & BIT(cause);
+ core->mac[EICR] &= ~effective_eiac;
}
static inline void
return E1000_MRQ_RSS_TYPE_IPV4TCP;
}
+ if (l4hdr_proto == ETH_L4_HDR_PROTO_UDP &&
+ (core->mac[MRQC] & E1000_MRQC_RSS_FIELD_IPV4_UDP)) {
+ return E1000_MRQ_RSS_TYPE_IPV4UDP;
+ }
+
if (E1000_MRQC_EN_IPV4(core->mac[MRQC])) {
return E1000_MRQ_RSS_TYPE_IPV4;
}
ip6info->rss_ex_dst_valid,
ip6info->rss_ex_src_valid,
core->mac[MRQC],
- E1000_MRQC_EN_TCPIPV6(core->mac[MRQC]),
+ E1000_MRQC_EN_TCPIPV6EX(core->mac[MRQC]),
E1000_MRQC_EN_IPV6EX(core->mac[MRQC]),
E1000_MRQC_EN_IPV6(core->mac[MRQC]));
ip6info->rss_ex_src_valid))) {
if (l4hdr_proto == ETH_L4_HDR_PROTO_TCP &&
- E1000_MRQC_EN_TCPIPV6(core->mac[MRQC])) {
- return E1000_MRQ_RSS_TYPE_IPV6TCP;
+ E1000_MRQC_EN_TCPIPV6EX(core->mac[MRQC])) {
+ return E1000_MRQ_RSS_TYPE_IPV6TCPEX;
+ }
+
+ if (l4hdr_proto == ETH_L4_HDR_PROTO_UDP &&
+ (core->mac[MRQC] & E1000_MRQC_RSS_FIELD_IPV6_UDP)) {
+ return E1000_MRQ_RSS_TYPE_IPV6UDP;
}
if (E1000_MRQC_EN_IPV6EX(core->mac[MRQC])) {
case E1000_MRQ_RSS_TYPE_IPV4TCP:
type = NetPktRssIpV4Tcp;
break;
- case E1000_MRQ_RSS_TYPE_IPV6TCP:
+ case E1000_MRQ_RSS_TYPE_IPV6TCPEX:
type = NetPktRssIpV6TcpEx;
break;
case E1000_MRQ_RSS_TYPE_IPV6:
case E1000_MRQ_RSS_TYPE_IPV6EX:
type = NetPktRssIpV6Ex;
break;
+ case E1000_MRQ_RSS_TYPE_IPV4UDP:
+ type = NetPktRssIpV4Udp;
+ break;
+ case E1000_MRQ_RSS_TYPE_IPV6UDP:
+ type = NetPktRssIpV6Udp;
+ break;
default:
assert(false);
return 0;
}
}
- if (insert_vlan && e1000x_vlan_enabled(core->mac)) {
+ if (insert_vlan) {
net_tx_pkt_setup_vlan_header_ex(tx->tx_pkt, vlan,
core->mac[VET] & 0xffff);
}
static bool
igb_setup_tx_offloads(IGBCore *core, struct igb_tx *tx)
{
+ uint32_t idx = (tx->first_olinfo_status >> 4) & 1;
+
if (tx->first_cmd_type_len & E1000_ADVTXD_DCMD_TSE) {
- uint32_t idx = (tx->first_olinfo_status >> 4) & 1;
- uint32_t mss = tx->ctx[idx].mss_l4len_idx >> 16;
+ uint32_t mss = tx->ctx[idx].mss_l4len_idx >> E1000_ADVTXD_MSS_SHIFT;
if (!net_tx_pkt_build_vheader(tx->tx_pkt, true, true, mss)) {
return false;
}
return true;
}
- if (tx->first_olinfo_status & E1000_ADVTXD_POTS_TXSM) {
- if (!net_tx_pkt_build_vheader(tx->tx_pkt, false, true, 0)) {
- return false;
- }
+ if ((tx->first_olinfo_status & E1000_ADVTXD_POTS_TXSM) &&
+ !((tx->ctx[idx].type_tucmd_mlhl & E1000_ADVTXD_TUCMD_L4T_SCTP) ?
+ net_tx_pkt_update_sctp_checksum(tx->tx_pkt) :
+ net_tx_pkt_build_vheader(tx->tx_pkt, false, true, 0))) {
+ return false;
}
if (tx->first_olinfo_status & E1000_ADVTXD_POTS_IXSM) {
g_assert_not_reached();
}
- core->mac[GPTC] = core->mac[TPT];
- core->mac[GOTCL] = core->mac[TOTL];
- core->mac[GOTCH] = core->mac[TOTH];
+ e1000x_inc_reg_if_not_full(core->mac, GPTC);
+ e1000x_grow_8reg_if_not_full(core->mac, GOTCL, tot_len);
if (core->mac[MRQC] & 1) {
uint16_t pool = qn % IGB_NUM_VM_POOLS;
length = cmd_type_len & 0xFFFF;
if (!tx->skip_cp) {
- if (!net_tx_pkt_add_raw_fragment(tx->tx_pkt, buffer_addr, length)) {
+ if (!net_tx_pkt_add_raw_fragment_pci(tx->tx_pkt, dev,
+ buffer_addr, length)) {
tx->skip_cp = true;
}
}
if (!tx->skip_cp && net_tx_pkt_parse(tx->tx_pkt)) {
idx = (tx->first_olinfo_status >> 4) & 1;
igb_tx_insert_vlan(core, queue_index, tx,
- tx->ctx[idx].vlan_macip_lens >> 16,
- !!(cmd_type_len & E1000_TXD_CMD_VLE));
+ tx->ctx[idx].vlan_macip_lens >> IGB_TX_FLAGS_VLAN_SHIFT,
+ !!(tx->first_cmd_type_len & E1000_TXD_CMD_VLE));
+
+ if ((tx->first_cmd_type_len & E1000_ADVTXD_MAC_TSTAMP) &&
+ (core->mac[TSYNCTXCTL] & E1000_TSYNCTXCTL_ENABLED) &&
+ !(core->mac[TSYNCTXCTL] & E1000_TSYNCTXCTL_VALID)) {
+ core->mac[TSYNCTXCTL] |= E1000_TSYNCTXCTL_VALID;
+ e1000x_timestamp(core->mac, core->timadj, TXSTMPL, TXSTMPH);
+ }
if (igb_tx_pkt_send(core, tx, queue_index)) {
igb_on_tx_done_update_stats(core, tx->tx_pkt, queue_index);
tx->first = true;
tx->skip_cp = false;
- net_tx_pkt_reset(tx->tx_pkt, dev);
+ net_tx_pkt_reset(tx->tx_pkt, net_tx_pkt_unmap_frag_pci, dev);
}
}
d = core->owner;
}
- net_tx_pkt_reset(txr->tx->tx_pkt, d);
-
while (!igb_ring_empty(core, txi)) {
base = igb_ring_head_descr(core, txi);
}
if (eic) {
- core->mac[EICR] |= eic;
- igb_set_interrupt_cause(core, E1000_ICR_TXDW);
+ igb_raise_interrupts(core, EICR, eic);
+ igb_raise_interrupts(core, ICR, E1000_ICR_TXDW);
}
+
+ net_tx_pkt_reset(txr->tx->tx_pkt, net_tx_pkt_unmap_frag_pci, d);
}
static uint32_t
return !!(core->mac[RXCSUM] & E1000_RXCSUM_TUOFLD);
}
-static bool
-igb_rx_is_oversized(IGBCore *core, uint16_t qn, size_t size)
+static bool igb_rx_is_oversized(IGBCore *core, const struct eth_header *ehdr,
+ size_t size, size_t vlan_num,
+ bool lpe, uint16_t rlpml)
{
- uint16_t pool = qn % IGB_NUM_VM_POOLS;
- bool lpe = !!(core->mac[VMOLR0 + pool] & E1000_VMOLR_LPE);
- int max_ethernet_lpe_size =
- core->mac[VMOLR0 + pool] & E1000_VMOLR_RLPML_MASK;
- int max_ethernet_vlan_size = 1522;
-
- return size > (lpe ? max_ethernet_lpe_size : max_ethernet_vlan_size);
+ size_t vlan_header_size = sizeof(struct vlan_header) * vlan_num;
+ size_t header_size = sizeof(struct eth_header) + vlan_header_size;
+ return lpe ? size + ETH_FCS_LEN > rlpml : size > header_size + ETH_MTU;
}
-static uint16_t igb_receive_assign(IGBCore *core, const struct eth_header *ehdr,
- size_t size, E1000E_RSSInfo *rss_info,
- bool *external_tx)
+static uint16_t igb_receive_assign(IGBCore *core, const struct iovec *iov,
+ size_t iovcnt, size_t iov_ofs,
+ const L2Header *l2_header, size_t size,
+ E1000E_RSSInfo *rss_info,
+ uint16_t *etqf, bool *ts, bool *external_tx)
{
static const int ta_shift[] = { 4, 3, 2, 0 };
+ const struct eth_header *ehdr = &l2_header->eth;
uint32_t f, ra[2], *macp, rctl = core->mac[RCTL];
uint16_t queues = 0;
uint16_t oversized = 0;
- uint16_t vid = lduw_be_p(&PKT_GET_VLAN_HDR(ehdr)->h_tci) & VLAN_VID_MASK;
- bool accepted = false;
+ size_t vlan_num = 0;
+ PTP2 ptp2;
+ bool lpe;
+ uint16_t rlpml;
int i;
memset(rss_info, 0, sizeof(E1000E_RSSInfo));
+ *ts = false;
if (external_tx) {
*external_tx = true;
}
- if (e1000x_is_vlan_packet(ehdr, core->mac[VET] & 0xffff) &&
- e1000x_vlan_rx_filter_enabled(core->mac)) {
- uint32_t vfta =
- ldl_le_p((uint32_t *)(core->mac + VFTA) +
- ((vid >> E1000_VFTA_ENTRY_SHIFT) & E1000_VFTA_ENTRY_MASK));
- if ((vfta & (1 << (vid & E1000_VFTA_ENTRY_BIT_SHIFT_MASK))) == 0) {
- trace_e1000e_rx_flt_vlan_mismatch(vid);
- return queues;
- } else {
- trace_e1000e_rx_flt_vlan_match(vid);
+ if (core->mac[CTRL_EXT] & BIT(26)) {
+ if (be16_to_cpu(ehdr->h_proto) == core->mac[VET] >> 16 &&
+ be16_to_cpu(l2_header->vlan[0].h_proto) == (core->mac[VET] & 0xffff)) {
+ vlan_num = 2;
+ }
+ } else {
+ if (be16_to_cpu(ehdr->h_proto) == (core->mac[VET] & 0xffff)) {
+ vlan_num = 1;
}
}
+ lpe = !!(core->mac[RCTL] & E1000_RCTL_LPE);
+ rlpml = core->mac[RLPML];
+ if (!(core->mac[RCTL] & E1000_RCTL_SBP) &&
+ igb_rx_is_oversized(core, ehdr, size, vlan_num, lpe, rlpml)) {
+ trace_e1000x_rx_oversized(size);
+ return queues;
+ }
+
+ for (*etqf = 0; *etqf < 8; (*etqf)++) {
+ if ((core->mac[ETQF0 + *etqf] & E1000_ETQF_FILTER_ENABLE) &&
+ be16_to_cpu(ehdr->h_proto) == (core->mac[ETQF0 + *etqf] & E1000_ETQF_ETYPE_MASK)) {
+ if ((core->mac[ETQF0 + *etqf] & E1000_ETQF_1588) &&
+ (core->mac[TSYNCRXCTL] & E1000_TSYNCRXCTL_ENABLED) &&
+ !(core->mac[TSYNCRXCTL] & E1000_TSYNCRXCTL_VALID) &&
+ iov_to_buf(iov, iovcnt, iov_ofs + ETH_HLEN, &ptp2, sizeof(ptp2)) >= sizeof(ptp2) &&
+ (ptp2.version_ptp & 15) == 2 &&
+ ptp2.message_id_transport_specific == ((core->mac[TSYNCRXCFG] >> 8) & 255)) {
+ e1000x_timestamp(core->mac, core->timadj, RXSTMPL, RXSTMPH);
+ *ts = true;
+ core->mac[TSYNCRXCTL] |= E1000_TSYNCRXCTL_VALID;
+ core->mac[RXSATRL] = le32_to_cpu(ptp2.source_uuid_lo);
+ core->mac[RXSATRH] = le16_to_cpu(ptp2.source_uuid_hi) |
+ (le16_to_cpu(ptp2.sequence_id) << 16);
+ }
+ break;
+ }
+ }
+
+ if (vlan_num &&
+ !e1000x_rx_vlan_filter(core->mac, l2_header->vlan + vlan_num - 1)) {
+ return queues;
+ }
+
if (core->mac[MRQC] & 1) {
if (is_broadcast_ether_addr(ehdr->h_dest)) {
for (i = 0; i < IGB_NUM_VM_POOLS; i++) {
if (e1000x_vlan_rx_filter_enabled(core->mac)) {
uint16_t mask = 0;
- if (e1000x_is_vlan_packet(ehdr, core->mac[VET] & 0xffff)) {
+ if (vlan_num) {
+ uint16_t vid = be16_to_cpu(l2_header->vlan[vlan_num - 1].h_tci) & VLAN_VID_MASK;
+
for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
if ((core->mac[VLVF0 + i] & E1000_VLVF_VLANID_MASK) == vid &&
(core->mac[VLVF0 + i] & E1000_VLVF_VLANID_ENABLE)) {
queues &= core->mac[VFRE];
if (queues) {
for (i = 0; i < IGB_NUM_VM_POOLS; i++) {
- if ((queues & BIT(i)) && igb_rx_is_oversized(core, i, size)) {
+ lpe = !!(core->mac[VMOLR0 + i] & E1000_VMOLR_LPE);
+ rlpml = core->mac[VMOLR0 + i] & E1000_VMOLR_RLPML_MASK;
+ if ((queues & BIT(i)) &&
+ igb_rx_is_oversized(core, ehdr, size, vlan_num,
+ lpe, rlpml)) {
oversized |= BIT(i);
}
}
}
}
} else {
- switch (net_rx_pkt_get_packet_type(core->rx_pkt)) {
- case ETH_PKT_UCAST:
- if (rctl & E1000_RCTL_UPE) {
- accepted = true; /* promiscuous ucast */
- }
- break;
-
- case ETH_PKT_BCAST:
- if (rctl & E1000_RCTL_BAM) {
- accepted = true; /* broadcast enabled */
- }
- break;
-
- case ETH_PKT_MCAST:
- if (rctl & E1000_RCTL_MPE) {
- accepted = true; /* promiscuous mcast */
- }
- break;
-
- default:
- g_assert_not_reached();
- }
-
- if (!accepted) {
- accepted = e1000x_rx_group_filter(core->mac, ehdr->h_dest);
- }
-
+ bool accepted = e1000x_rx_group_filter(core->mac, ehdr);
if (!accepted) {
for (macp = core->mac + RA2; macp < core->mac + RA2 + 16; macp += 2) {
if (!(macp[1] & E1000_RAH_AV)) {
igb_build_rx_metadata(IGBCore *core,
struct NetRxPkt *pkt,
bool is_eop,
- const E1000E_RSSInfo *rss_info,
+ const E1000E_RSSInfo *rss_info, uint16_t etqf, bool ts,
uint16_t *pkt_info, uint16_t *hdr_info,
uint32_t *rss,
uint32_t *status_flags,
uint16_t *vlan_tag)
{
struct virtio_net_hdr *vhdr;
- bool hasip4, hasip6;
+ bool hasip4, hasip6, csum_valid;
EthL4HdrProto l4hdr_proto;
- uint32_t pkt_type;
*status_flags = E1000_RXD_STAT_DD;
trace_e1000e_rx_metadata_ack();
}
- if (hasip6 && (core->mac[RFCTL] & E1000_RFCTL_IPV6_DIS)) {
- trace_e1000e_rx_metadata_ipv6_filtering_disabled();
- pkt_type = E1000_RXD_PKT_MAC;
- } else if (l4hdr_proto == ETH_L4_HDR_PROTO_TCP ||
- l4hdr_proto == ETH_L4_HDR_PROTO_UDP) {
- pkt_type = hasip4 ? E1000_RXD_PKT_IP4_XDP : E1000_RXD_PKT_IP6_XDP;
- } else if (hasip4 || hasip6) {
- pkt_type = hasip4 ? E1000_RXD_PKT_IP4 : E1000_RXD_PKT_IP6;
- } else {
- pkt_type = E1000_RXD_PKT_MAC;
- }
+ if (pkt_info) {
+ *pkt_info = rss_info->enabled ? rss_info->type : 0;
- trace_e1000e_rx_metadata_pkt_type(pkt_type);
+ if (etqf < 8) {
+ *pkt_info |= (BIT(11) | etqf) << 4;
+ } else {
+ if (hasip4) {
+ *pkt_info |= E1000_ADVRXD_PKT_IP4;
+ }
- if (pkt_info) {
- if (rss_info->enabled) {
- *pkt_info = rss_info->type;
- }
+ if (hasip6) {
+ *pkt_info |= E1000_ADVRXD_PKT_IP6;
+ }
- *pkt_info |= (pkt_type << 4);
- } else {
- *status_flags |= E1000_RXD_PKT_TYPE(pkt_type);
+ switch (l4hdr_proto) {
+ case ETH_L4_HDR_PROTO_TCP:
+ *pkt_info |= E1000_ADVRXD_PKT_TCP;
+ break;
+
+ case ETH_L4_HDR_PROTO_UDP:
+ *pkt_info |= E1000_ADVRXD_PKT_UDP;
+ break;
+
+ case ETH_L4_HDR_PROTO_SCTP:
+ *pkt_info |= E1000_ADVRXD_PKT_SCTP;
+ break;
+
+ default:
+ break;
+ }
+ }
}
if (hdr_info) {
*hdr_info = 0;
}
+ if (ts) {
+ *status_flags |= BIT(16);
+ }
+
/* RX CSO information */
if (hasip6 && (core->mac[RFCTL] & E1000_RFCTL_IPV6_XSUM_DIS)) {
trace_e1000e_rx_metadata_ipv6_sum_disabled();
if (igb_rx_l4_cso_enabled(core)) {
switch (l4hdr_proto) {
+ case ETH_L4_HDR_PROTO_SCTP:
+ if (!net_rx_pkt_validate_l4_csum(pkt, &csum_valid)) {
+ trace_e1000e_rx_metadata_l4_csum_validation_failed();
+ goto func_exit;
+ }
+ if (!csum_valid) {
+ *status_flags |= E1000_RXDEXT_STATERR_TCPE;
+ }
+ /* fall through */
case ETH_L4_HDR_PROTO_TCP:
*status_flags |= E1000_RXD_STAT_TCPCS;
break;
break;
default:
- goto func_exit;
+ break;
}
} else {
trace_e1000e_rx_metadata_l4_cso_disabled();
}
- trace_e1000e_rx_metadata_status_flags(*status_flags);
-
func_exit:
+ trace_e1000e_rx_metadata_status_flags(*status_flags);
*status_flags = cpu_to_le32(*status_flags);
}
static inline void
igb_write_lgcy_rx_descr(IGBCore *core, struct e1000_rx_desc *desc,
struct NetRxPkt *pkt,
- const E1000E_RSSInfo *rss_info,
+ const E1000E_RSSInfo *rss_info, uint16_t etqf, bool ts,
uint16_t length)
{
uint32_t status_flags, rss;
desc->csum = 0;
igb_build_rx_metadata(core, pkt, pkt != NULL,
- rss_info,
+ rss_info, etqf, ts,
NULL, NULL, &rss,
&status_flags, &ip_id,
&desc->special);
static inline void
igb_write_adv_rx_descr(IGBCore *core, union e1000_adv_rx_desc *desc,
struct NetRxPkt *pkt,
- const E1000E_RSSInfo *rss_info,
+ const E1000E_RSSInfo *rss_info, uint16_t etqf, bool ts,
uint16_t length)
{
memset(&desc->wb, 0, sizeof(desc->wb));
desc->wb.upper.length = cpu_to_le16(length);
igb_build_rx_metadata(core, pkt, pkt != NULL,
- rss_info,
+ rss_info, etqf, ts,
&desc->wb.lower.lo_dword.pkt_info,
&desc->wb.lower.lo_dword.hdr_info,
&desc->wb.lower.hi_dword.rss,
static inline void
igb_write_rx_descr(IGBCore *core, union e1000_rx_desc_union *desc,
-struct NetRxPkt *pkt, const E1000E_RSSInfo *rss_info, uint16_t length)
+ struct NetRxPkt *pkt, const E1000E_RSSInfo *rss_info,
+ uint16_t etqf, bool ts, uint16_t length)
{
if (igb_rx_use_legacy_descriptor(core)) {
- igb_write_lgcy_rx_descr(core, &desc->legacy, pkt, rss_info, length);
+ igb_write_lgcy_rx_descr(core, &desc->legacy, pkt, rss_info,
+ etqf, ts, length);
} else {
- igb_write_adv_rx_descr(core, &desc->adv, pkt, rss_info, length);
+ igb_write_adv_rx_descr(core, &desc->adv, pkt, rss_info,
+ etqf, ts, length);
}
}
static void
igb_update_rx_stats(IGBCore *core, const E1000E_RingInfo *rxi,
- size_t data_size, size_t data_fcs_size)
+ size_t pkt_size, size_t pkt_fcs_size)
{
- e1000x_update_rx_total_stats(core->mac, data_size, data_fcs_size);
-
- switch (net_rx_pkt_get_packet_type(core->rx_pkt)) {
- case ETH_PKT_BCAST:
- e1000x_inc_reg_if_not_full(core->mac, BPRC);
- break;
-
- case ETH_PKT_MCAST:
- e1000x_inc_reg_if_not_full(core->mac, MPRC);
- break;
-
- default:
- break;
- }
+ eth_pkt_types_e pkt_type = net_rx_pkt_get_packet_type(core->rx_pkt);
+ e1000x_update_rx_total_stats(core->mac, pkt_type, pkt_size, pkt_fcs_size);
if (core->mac[MRQC] & 1) {
uint16_t pool = rxi->idx % IGB_NUM_VM_POOLS;
- core->mac[PVFGORC0 + (pool * 64)] += data_size + 4;
+ core->mac[PVFGORC0 + (pool * 64)] += pkt_size + 4;
core->mac[PVFGPRC0 + (pool * 64)]++;
- if (net_rx_pkt_get_packet_type(core->rx_pkt) == ETH_PKT_MCAST) {
+ if (pkt_type == ETH_PKT_MCAST) {
core->mac[PVFMPRC0 + (pool * 64)]++;
}
}
static void
igb_write_packet_to_guest(IGBCore *core, struct NetRxPkt *pkt,
const E1000E_RxRing *rxr,
- const E1000E_RSSInfo *rss_info)
+ const E1000E_RSSInfo *rss_info,
+ uint16_t etqf, bool ts)
{
PCIDevice *d;
dma_addr_t base;
}
igb_write_rx_descr(core, &desc, is_last ? core->rx_pkt : NULL,
- rss_info, written);
+ rss_info, etqf, ts, written);
igb_pci_dma_write_rx_desc(core, d, base, &desc, core->rx_desc_len);
igb_ring_advance(core, rxi, core->rx_desc_len / E1000_MIN_RX_DESC_LEN);
igb_receive_internal(IGBCore *core, const struct iovec *iov, int iovcnt,
bool has_vnet, bool *external_tx)
{
- static const int maximum_ethernet_hdr_len = (ETH_HLEN + 4);
-
uint16_t queues = 0;
- uint32_t n = 0;
- uint8_t min_buf[ETH_ZLEN];
+ uint32_t causes = 0;
+ uint32_t ecauses = 0;
+ union {
+ L2Header l2_header;
+ uint8_t octets[ETH_ZLEN];
+ } buf;
struct iovec min_iov;
- struct eth_header *ehdr;
- uint8_t *filter_buf;
size_t size, orig_size;
size_t iov_ofs = 0;
E1000E_RxRing rxr;
E1000E_RSSInfo rss_info;
+ uint16_t etqf;
+ bool ts;
size_t total_size;
+ int strip_vlan_index;
int i;
trace_e1000e_rx_receive_iov(iovcnt);
net_rx_pkt_unset_vhdr(core->rx_pkt);
}
- filter_buf = iov->iov_base + iov_ofs;
orig_size = iov_size(iov, iovcnt);
size = orig_size - iov_ofs;
/* Pad to minimum Ethernet frame length */
- if (size < sizeof(min_buf)) {
- iov_to_buf(iov, iovcnt, iov_ofs, min_buf, size);
- memset(&min_buf[size], 0, sizeof(min_buf) - size);
+ if (size < sizeof(buf)) {
+ iov_to_buf(iov, iovcnt, iov_ofs, &buf, size);
+ memset(&buf.octets[size], 0, sizeof(buf) - size);
e1000x_inc_reg_if_not_full(core->mac, RUC);
- min_iov.iov_base = filter_buf = min_buf;
- min_iov.iov_len = size = sizeof(min_buf);
+ min_iov.iov_base = &buf;
+ min_iov.iov_len = size = sizeof(buf);
iovcnt = 1;
iov = &min_iov;
iov_ofs = 0;
- } else if (iov->iov_len < maximum_ethernet_hdr_len) {
- /* This is very unlikely, but may happen. */
- iov_to_buf(iov, iovcnt, iov_ofs, min_buf, maximum_ethernet_hdr_len);
- filter_buf = min_buf;
- }
-
- /* Discard oversized packets if !LPE and !SBP. */
- if (e1000x_is_oversized(core->mac, size)) {
- return orig_size;
+ } else {
+ iov_to_buf(iov, iovcnt, iov_ofs, &buf, sizeof(buf.l2_header));
}
- ehdr = PKT_GET_ETH_HDR(filter_buf);
- net_rx_pkt_set_packet_type(core->rx_pkt, get_eth_packet_type(ehdr));
- net_rx_pkt_set_protocols(core->rx_pkt, filter_buf, size);
+ net_rx_pkt_set_packet_type(core->rx_pkt,
+ get_eth_packet_type(&buf.l2_header.eth));
+ net_rx_pkt_set_protocols(core->rx_pkt, iov, iovcnt, iov_ofs);
- queues = igb_receive_assign(core, ehdr, size, &rss_info, external_tx);
+ queues = igb_receive_assign(core, iov, iovcnt, iov_ofs,
+ &buf.l2_header, size,
+ &rss_info, &etqf, &ts, external_tx);
if (!queues) {
trace_e1000e_rx_flt_dropped();
return orig_size;
igb_rx_ring_init(core, &rxr, i);
+ if (!igb_rx_strip_vlan(core, rxr.i)) {
+ strip_vlan_index = -1;
+ } else if (core->mac[CTRL_EXT] & BIT(26)) {
+ strip_vlan_index = 1;
+ } else {
+ strip_vlan_index = 0;
+ }
+
net_rx_pkt_attach_iovec_ex(core->rx_pkt, iov, iovcnt, iov_ofs,
- igb_rx_strip_vlan(core, rxr.i),
- core->mac[VET] & 0xffff);
+ strip_vlan_index,
+ core->mac[VET] & 0xffff,
+ core->mac[VET] >> 16);
total_size = net_rx_pkt_get_total_len(core->rx_pkt) +
e1000x_fcs_len(core->mac);
if (!igb_has_rxbufs(core, rxr.i, total_size)) {
- n |= E1000_ICS_RXO;
+ causes |= E1000_ICS_RXO;
trace_e1000e_rx_not_written_to_guest(rxr.i->idx);
continue;
}
- n |= E1000_ICR_RXDW;
+ causes |= E1000_ICR_RXDW;
igb_rx_fix_l4_csum(core, core->rx_pkt);
- igb_write_packet_to_guest(core, core->rx_pkt, &rxr, &rss_info);
+ igb_write_packet_to_guest(core, core->rx_pkt, &rxr, &rss_info, etqf, ts);
/* Check if receive descriptor minimum threshold hit */
if (igb_rx_descr_threshold_hit(core, rxr.i)) {
- n |= E1000_ICS_RXDMT0;
+ causes |= E1000_ICS_RXDMT0;
}
- core->mac[EICR] |= igb_rx_wb_eic(core, rxr.i->idx);
+ ecauses |= igb_rx_wb_eic(core, rxr.i->idx);
trace_e1000e_rx_written_to_guest(rxr.i->idx);
}
- trace_e1000e_rx_interrupt_set(n);
- igb_set_interrupt_cause(core, n);
+ trace_e1000e_rx_interrupt_set(causes);
+ igb_raise_interrupts(core, EICR, ecauses);
+ igb_raise_interrupts(core, ICR, causes);
return orig_size;
}
}
if (core->mac[STATUS] != old_status) {
- igb_set_interrupt_cause(core, E1000_ICR_LSC);
+ igb_raise_interrupts(core, ICR, E1000_ICR_LSC);
}
}
}
}
-static inline void
-igb_clear_ims_bits(IGBCore *core, uint32_t bits)
-{
- trace_e1000e_irq_clear_ims(bits, core->mac[IMS], core->mac[IMS] & ~bits);
- core->mac[IMS] &= ~bits;
-}
-
static inline bool
igb_postpone_interrupt(IGBIntrDelayTimer *timer)
{
return igb_postpone_interrupt(&core->eitr[idx]);
}
-static void igb_send_msix(IGBCore *core)
+static void igb_send_msix(IGBCore *core, uint32_t causes)
{
- uint32_t causes = core->mac[EICR] & core->mac[EIMS];
- uint32_t effective_eiac;
int vector;
for (vector = 0; vector < IGB_INTR_NUM; ++vector) {
trace_e1000e_irq_msix_notify_vec(vector);
igb_msix_notify(core, vector);
-
- trace_e1000e_irq_icr_clear_eiac(core->mac[EICR], core->mac[EIAC]);
- effective_eiac = core->mac[EIAC] & BIT(vector);
- core->mac[EICR] &= ~effective_eiac;
}
}
}
trace_e1000e_irq_fix_icr_asserted(core->mac[ICR]);
}
-static void
-igb_update_interrupt_state(IGBCore *core)
+static void igb_raise_interrupts(IGBCore *core, size_t index, uint32_t causes)
{
- uint32_t icr;
- uint32_t causes;
+ uint32_t old_causes = core->mac[ICR] & core->mac[IMS];
+ uint32_t old_ecauses = core->mac[EICR] & core->mac[EIMS];
+ uint32_t raised_causes;
+ uint32_t raised_ecauses;
uint32_t int_alloc;
- icr = core->mac[ICR] & core->mac[IMS];
+ trace_e1000e_irq_set(index << 2,
+ core->mac[index], core->mac[index] | causes);
- if (msix_enabled(core->owner)) {
- if (icr) {
- causes = 0;
- if (icr & E1000_ICR_DRSTA) {
- int_alloc = core->mac[IVAR_MISC] & 0xff;
- if (int_alloc & E1000_IVAR_VALID) {
- causes |= BIT(int_alloc & 0x1f);
- }
+ core->mac[index] |= causes;
+
+ if (core->mac[GPIE] & E1000_GPIE_MSIX_MODE) {
+ raised_causes = core->mac[ICR] & core->mac[IMS] & ~old_causes;
+
+ if (raised_causes & E1000_ICR_DRSTA) {
+ int_alloc = core->mac[IVAR_MISC] & 0xff;
+ if (int_alloc & E1000_IVAR_VALID) {
+ core->mac[EICR] |= BIT(int_alloc & 0x1f);
}
- /* Check if other bits (excluding the TCP Timer) are enabled. */
- if (icr & ~E1000_ICR_DRSTA) {
- int_alloc = (core->mac[IVAR_MISC] >> 8) & 0xff;
- if (int_alloc & E1000_IVAR_VALID) {
- causes |= BIT(int_alloc & 0x1f);
- }
- trace_e1000e_irq_add_msi_other(core->mac[EICR]);
+ }
+ /* Check if other bits (excluding the TCP Timer) are enabled. */
+ if (raised_causes & ~E1000_ICR_DRSTA) {
+ int_alloc = (core->mac[IVAR_MISC] >> 8) & 0xff;
+ if (int_alloc & E1000_IVAR_VALID) {
+ core->mac[EICR] |= BIT(int_alloc & 0x1f);
}
- core->mac[EICR] |= causes;
}
- if ((core->mac[EICR] & core->mac[EIMS])) {
- igb_send_msix(core);
+ raised_ecauses = core->mac[EICR] & core->mac[EIMS] & ~old_ecauses;
+ if (!raised_ecauses) {
+ return;
}
+
+ igb_send_msix(core, raised_ecauses);
} else {
igb_fix_icr_asserted(core);
- if (icr) {
- core->mac[EICR] |= (icr & E1000_ICR_DRSTA) | E1000_EICR_OTHER;
- } else {
- core->mac[EICR] &= ~E1000_EICR_OTHER;
+ raised_causes = core->mac[ICR] & core->mac[IMS] & ~old_causes;
+ if (!raised_causes) {
+ return;
}
- trace_e1000e_irq_pending_interrupts(core->mac[ICR] & core->mac[IMS],
- core->mac[ICR], core->mac[IMS]);
+ core->mac[EICR] |= (raised_causes & E1000_ICR_DRSTA) | E1000_EICR_OTHER;
- if (msi_enabled(core->owner)) {
- if (icr) {
- msi_notify(core->owner, 0);
- }
+ if (msix_enabled(core->owner)) {
+ trace_e1000e_irq_msix_notify_vec(0);
+ msix_notify(core->owner, 0);
+ } else if (msi_enabled(core->owner)) {
+ trace_e1000e_irq_msi_notify(raised_causes);
+ msi_notify(core->owner, 0);
} else {
- if (icr) {
- igb_raise_legacy_irq(core);
- } else {
- igb_lower_legacy_irq(core);
- }
+ igb_raise_legacy_irq(core);
}
}
}
-static void
-igb_set_interrupt_cause(IGBCore *core, uint32_t val)
+static void igb_lower_interrupts(IGBCore *core, size_t index, uint32_t causes)
{
- trace_e1000e_irq_set_cause_entry(val, core->mac[ICR]);
+ trace_e1000e_irq_clear(index << 2,
+ core->mac[index], core->mac[index] & ~causes);
- core->mac[ICR] |= val;
+ core->mac[index] &= ~causes;
- trace_e1000e_irq_set_cause_exit(val, core->mac[ICR]);
+ trace_e1000e_irq_pending_interrupts(core->mac[ICR] & core->mac[IMS],
+ core->mac[ICR], core->mac[IMS]);
- igb_update_interrupt_state(core);
+ if (!(core->mac[ICR] & core->mac[IMS]) &&
+ !(core->mac[GPIE] & E1000_GPIE_MSIX_MODE)) {
+ core->mac[EICR] &= ~E1000_EICR_OTHER;
+
+ if (!msix_enabled(core->owner) && !msi_enabled(core->owner)) {
+ igb_lower_legacy_irq(core);
+ }
+ }
}
static void igb_set_eics(IGBCore *core, int index, uint32_t val)
{
bool msix = !!(core->mac[GPIE] & E1000_GPIE_MSIX_MODE);
+ uint32_t mask = msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK;
trace_igb_irq_write_eics(val, msix);
-
- core->mac[EICS] |=
- val & (msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK);
-
- /*
- * TODO: Move to igb_update_interrupt_state if EICS is modified in other
- * places.
- */
- core->mac[EICR] = core->mac[EICS];
-
- igb_update_interrupt_state(core);
+ igb_raise_interrupts(core, EICR, val & mask);
}
static void igb_set_eims(IGBCore *core, int index, uint32_t val)
{
bool msix = !!(core->mac[GPIE] & E1000_GPIE_MSIX_MODE);
+ uint32_t mask = msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK;
trace_igb_irq_write_eims(val, msix);
-
- core->mac[EIMS] |=
- val & (msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK);
-
- igb_update_interrupt_state(core);
+ igb_raise_interrupts(core, EIMS, val & mask);
}
static void mailbox_interrupt_to_vf(IGBCore *core, uint16_t vfn)
{
uint32_t ent = core->mac[VTIVAR_MISC + vfn];
+ uint32_t causes;
if ((ent & E1000_IVAR_VALID)) {
- core->mac[EICR] |= (ent & 0x3) << (22 - vfn * IGBVF_MSIX_VEC_NUM);
- igb_update_interrupt_state(core);
+ causes = (ent & 0x3) << (22 - vfn * IGBVF_MSIX_VEC_NUM);
+ igb_raise_interrupts(core, EICR, causes);
}
}
static void mailbox_interrupt_to_pf(IGBCore *core)
{
- igb_set_interrupt_cause(core, E1000_ICR_VMMB);
+ igb_raise_interrupts(core, ICR, E1000_ICR_VMMB);
}
static void igb_set_pfmailbox(IGBCore *core, int index, uint32_t val)
static void igb_set_eimc(IGBCore *core, int index, uint32_t val)
{
bool msix = !!(core->mac[GPIE] & E1000_GPIE_MSIX_MODE);
+ uint32_t mask = msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK;
- /* Interrupts are disabled via a write to EIMC and reflected in EIMS. */
- core->mac[EIMS] &=
- ~(val & (msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK));
+ trace_igb_irq_write_eimc(val, msix);
- trace_igb_irq_write_eimc(val, core->mac[EIMS], msix);
- igb_update_interrupt_state(core);
+ /* Interrupts are disabled via a write to EIMC and reflected in EIMS. */
+ igb_lower_interrupts(core, EIMS, val & mask);
}
static void igb_set_eiac(IGBCore *core, int index, uint32_t val)
* TODO: In IOV mode, only bit zero of this vector is available for the PF
* function.
*/
- core->mac[EICR] &=
- ~(val & (msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK));
+ uint32_t mask = msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK;
trace_igb_irq_write_eicr(val, msix);
- igb_update_interrupt_state(core);
+ igb_lower_interrupts(core, EICR, val & mask);
}
static void igb_set_vtctrl(IGBCore *core, int index, uint32_t val)
igb_update_flowctl_status(core);
/* signal link status change to the guest */
- igb_set_interrupt_cause(core, E1000_ICR_LSC);
+ igb_raise_interrupts(core, ICR, E1000_ICR_LSC);
}
}
core->mac[MDIC] = val | E1000_MDIC_READY;
if (val & E1000_MDIC_INT_EN) {
- igb_set_interrupt_cause(core, E1000_ICR_MDAC);
+ igb_raise_interrupts(core, ICR, E1000_ICR_MDAC);
}
}
igb_set_ics(IGBCore *core, int index, uint32_t val)
{
trace_e1000e_irq_write_ics(val);
- igb_set_interrupt_cause(core, val);
+ igb_raise_interrupts(core, ICR, val);
}
static void
igb_set_imc(IGBCore *core, int index, uint32_t val)
{
trace_e1000e_irq_ims_clear_set_imc(val);
- igb_clear_ims_bits(core, val);
- igb_update_interrupt_state(core);
+ igb_lower_interrupts(core, IMS, val);
}
static void
igb_set_ims(IGBCore *core, int index, uint32_t val)
{
- uint32_t valid_val = val & 0x77D4FBFD;
-
- trace_e1000e_irq_set_ims(val, core->mac[IMS], core->mac[IMS] | valid_val);
- core->mac[IMS] |= valid_val;
- igb_update_interrupt_state(core);
+ igb_raise_interrupts(core, IMS, val & 0x77D4FBFD);
}
-static void igb_commit_icr(IGBCore *core)
+static void igb_nsicr(IGBCore *core)
{
/*
- * If GPIE.NSICR = 0, then the copy of IAM to IMS will occur only if at
+ * If GPIE.NSICR = 0, then the clear of IMS will occur only if at
* least one bit is set in the IMS and there is a true interrupt as
* reflected in ICR.INTA.
*/
if ((core->mac[GPIE] & E1000_GPIE_NSICR) ||
(core->mac[IMS] && (core->mac[ICR] & E1000_ICR_INT_ASSERTED))) {
- igb_set_ims(core, IMS, core->mac[IAM]);
- } else {
- igb_update_interrupt_state(core);
+ igb_lower_interrupts(core, IMS, core->mac[IAM]);
}
}
static void igb_set_icr(IGBCore *core, int index, uint32_t val)
{
- uint32_t icr = core->mac[ICR] & ~val;
-
- trace_igb_irq_icr_write(val, core->mac[ICR], icr);
- core->mac[ICR] = icr;
- igb_commit_icr(core);
+ igb_nsicr(core);
+ igb_lower_interrupts(core, ICR, val);
}
static uint32_t
igb_mac_icr_read(IGBCore *core, int index)
{
uint32_t ret = core->mac[ICR];
- trace_e1000e_irq_icr_read_entry(ret);
if (core->mac[GPIE] & E1000_GPIE_NSICR) {
trace_igb_irq_icr_clear_gpie_nsicr();
- core->mac[ICR] = 0;
+ igb_lower_interrupts(core, ICR, 0xffffffff);
} else if (core->mac[IMS] == 0) {
trace_e1000e_irq_icr_clear_zero_ims();
- core->mac[ICR] = 0;
+ igb_lower_interrupts(core, ICR, 0xffffffff);
+ } else if (core->mac[ICR] & E1000_ICR_INT_ASSERTED) {
+ igb_lower_interrupts(core, ICR, 0xffffffff);
} else if (!msix_enabled(core->owner)) {
trace_e1000e_irq_icr_clear_nonmsix_icr_read();
- core->mac[ICR] = 0;
+ igb_lower_interrupts(core, ICR, 0xffffffff);
}
- trace_e1000e_irq_icr_read_exit(core->mac[ICR]);
- igb_commit_icr(core);
+ igb_nsicr(core);
return ret;
}
[EIAM] = igb_mac_readreg,
[IVAR0 ... IVAR0 + 7] = igb_mac_readreg,
igb_getreg(IVAR_MISC),
+ igb_getreg(TSYNCRXCFG),
+ [ETQF0 ... ETQF0 + 7] = igb_mac_readreg,
igb_getreg(VT_CTL),
[P2VMAILBOX0 ... P2VMAILBOX7] = igb_mac_readreg,
[V2PMAILBOX0 ... V2PMAILBOX7] = igb_mac_vfmailbox_read,
[EIMS] = igb_set_eims,
[IVAR0 ... IVAR0 + 7] = igb_mac_writereg,
igb_putreg(IVAR_MISC),
+ igb_putreg(TSYNCRXCFG),
+ [ETQF0 ... ETQF0 + 7] = igb_mac_writereg,
igb_putreg(VT_CTL),
[P2VMAILBOX0 ... P2VMAILBOX7] = igb_set_pfmailbox,
[V2PMAILBOX0 ... V2PMAILBOX7] = igb_set_vfmailbox,
core->vmstate = qemu_add_vm_change_state_handler(igb_vm_state_change, core);
for (i = 0; i < IGB_NUM_QUEUES; i++) {
- net_tx_pkt_init(&core->tx[i].tx_pkt, NULL, E1000E_MAX_TX_FRAGS);
+ net_tx_pkt_init(&core->tx[i].tx_pkt, E1000E_MAX_TX_FRAGS);
}
net_rx_pkt_init(&core->rx_pkt);
qemu_del_vm_change_state_handler(core->vmstate);
for (i = 0; i < IGB_NUM_QUEUES; i++) {
- net_tx_pkt_reset(core->tx[i].tx_pkt, NULL);
net_tx_pkt_uninit(core->tx[i].tx_pkt);
}
[VMOLR0 ... VMOLR0 + 7] = 0x2600 | E1000_VMOLR_STRCRC,
[RPLOLR] = E1000_RPLOLR_STRCRC,
[RLPML] = 0x2600,
- [TXCTL0] = E1000_DCA_TXCTRL_DATA_RRO_EN |
- E1000_DCA_TXCTRL_TX_WB_RO_EN |
- E1000_DCA_TXCTRL_DESC_RRO_EN,
- [TXCTL1] = E1000_DCA_TXCTRL_DATA_RRO_EN |
- E1000_DCA_TXCTRL_TX_WB_RO_EN |
- E1000_DCA_TXCTRL_DESC_RRO_EN,
- [TXCTL2] = E1000_DCA_TXCTRL_DATA_RRO_EN |
- E1000_DCA_TXCTRL_TX_WB_RO_EN |
- E1000_DCA_TXCTRL_DESC_RRO_EN,
- [TXCTL3] = E1000_DCA_TXCTRL_DATA_RRO_EN |
- E1000_DCA_TXCTRL_TX_WB_RO_EN |
- E1000_DCA_TXCTRL_DESC_RRO_EN,
- [TXCTL4] = E1000_DCA_TXCTRL_DATA_RRO_EN |
- E1000_DCA_TXCTRL_TX_WB_RO_EN |
- E1000_DCA_TXCTRL_DESC_RRO_EN,
- [TXCTL5] = E1000_DCA_TXCTRL_DATA_RRO_EN |
- E1000_DCA_TXCTRL_TX_WB_RO_EN |
- E1000_DCA_TXCTRL_DESC_RRO_EN,
- [TXCTL6] = E1000_DCA_TXCTRL_DATA_RRO_EN |
- E1000_DCA_TXCTRL_TX_WB_RO_EN |
- E1000_DCA_TXCTRL_DESC_RRO_EN,
- [TXCTL7] = E1000_DCA_TXCTRL_DATA_RRO_EN |
- E1000_DCA_TXCTRL_TX_WB_RO_EN |
- E1000_DCA_TXCTRL_DESC_RRO_EN,
- [TXCTL8] = E1000_DCA_TXCTRL_DATA_RRO_EN |
- E1000_DCA_TXCTRL_TX_WB_RO_EN |
- E1000_DCA_TXCTRL_DESC_RRO_EN,
- [TXCTL9] = E1000_DCA_TXCTRL_DATA_RRO_EN |
- E1000_DCA_TXCTRL_TX_WB_RO_EN |
- E1000_DCA_TXCTRL_DESC_RRO_EN,
- [TXCTL10] = E1000_DCA_TXCTRL_DATA_RRO_EN |
- E1000_DCA_TXCTRL_TX_WB_RO_EN |
- E1000_DCA_TXCTRL_DESC_RRO_EN,
- [TXCTL11] = E1000_DCA_TXCTRL_DATA_RRO_EN |
- E1000_DCA_TXCTRL_TX_WB_RO_EN |
- E1000_DCA_TXCTRL_DESC_RRO_EN,
- [TXCTL12] = E1000_DCA_TXCTRL_DATA_RRO_EN |
- E1000_DCA_TXCTRL_TX_WB_RO_EN |
- E1000_DCA_TXCTRL_DESC_RRO_EN,
- [TXCTL13] = E1000_DCA_TXCTRL_DATA_RRO_EN |
- E1000_DCA_TXCTRL_TX_WB_RO_EN |
- E1000_DCA_TXCTRL_DESC_RRO_EN,
- [TXCTL14] = E1000_DCA_TXCTRL_DATA_RRO_EN |
- E1000_DCA_TXCTRL_TX_WB_RO_EN |
- E1000_DCA_TXCTRL_DESC_RRO_EN,
- [TXCTL15] = E1000_DCA_TXCTRL_DATA_RRO_EN |
- E1000_DCA_TXCTRL_TX_WB_RO_EN |
- E1000_DCA_TXCTRL_DESC_RRO_EN,
+ [TXCTL0] = E1000_DCA_TXCTRL_DATA_RRO_EN |
+ E1000_DCA_TXCTRL_TX_WB_RO_EN |
+ E1000_DCA_TXCTRL_DESC_RRO_EN,
+ [TXCTL1] = E1000_DCA_TXCTRL_DATA_RRO_EN |
+ E1000_DCA_TXCTRL_TX_WB_RO_EN |
+ E1000_DCA_TXCTRL_DESC_RRO_EN,
+ [TXCTL2] = E1000_DCA_TXCTRL_DATA_RRO_EN |
+ E1000_DCA_TXCTRL_TX_WB_RO_EN |
+ E1000_DCA_TXCTRL_DESC_RRO_EN,
+ [TXCTL3] = E1000_DCA_TXCTRL_DATA_RRO_EN |
+ E1000_DCA_TXCTRL_TX_WB_RO_EN |
+ E1000_DCA_TXCTRL_DESC_RRO_EN,
+ [TXCTL4] = E1000_DCA_TXCTRL_DATA_RRO_EN |
+ E1000_DCA_TXCTRL_TX_WB_RO_EN |
+ E1000_DCA_TXCTRL_DESC_RRO_EN,
+ [TXCTL5] = E1000_DCA_TXCTRL_DATA_RRO_EN |
+ E1000_DCA_TXCTRL_TX_WB_RO_EN |
+ E1000_DCA_TXCTRL_DESC_RRO_EN,
+ [TXCTL6] = E1000_DCA_TXCTRL_DATA_RRO_EN |
+ E1000_DCA_TXCTRL_TX_WB_RO_EN |
+ E1000_DCA_TXCTRL_DESC_RRO_EN,
+ [TXCTL7] = E1000_DCA_TXCTRL_DATA_RRO_EN |
+ E1000_DCA_TXCTRL_TX_WB_RO_EN |
+ E1000_DCA_TXCTRL_DESC_RRO_EN,
+ [TXCTL8] = E1000_DCA_TXCTRL_DATA_RRO_EN |
+ E1000_DCA_TXCTRL_TX_WB_RO_EN |
+ E1000_DCA_TXCTRL_DESC_RRO_EN,
+ [TXCTL9] = E1000_DCA_TXCTRL_DATA_RRO_EN |
+ E1000_DCA_TXCTRL_TX_WB_RO_EN |
+ E1000_DCA_TXCTRL_DESC_RRO_EN,
+ [TXCTL10] = E1000_DCA_TXCTRL_DATA_RRO_EN |
+ E1000_DCA_TXCTRL_TX_WB_RO_EN |
+ E1000_DCA_TXCTRL_DESC_RRO_EN,
+ [TXCTL11] = E1000_DCA_TXCTRL_DATA_RRO_EN |
+ E1000_DCA_TXCTRL_TX_WB_RO_EN |
+ E1000_DCA_TXCTRL_DESC_RRO_EN,
+ [TXCTL12] = E1000_DCA_TXCTRL_DATA_RRO_EN |
+ E1000_DCA_TXCTRL_TX_WB_RO_EN |
+ E1000_DCA_TXCTRL_DESC_RRO_EN,
+ [TXCTL13] = E1000_DCA_TXCTRL_DATA_RRO_EN |
+ E1000_DCA_TXCTRL_TX_WB_RO_EN |
+ E1000_DCA_TXCTRL_DESC_RRO_EN,
+ [TXCTL14] = E1000_DCA_TXCTRL_DATA_RRO_EN |
+ E1000_DCA_TXCTRL_TX_WB_RO_EN |
+ E1000_DCA_TXCTRL_DESC_RRO_EN,
+ [TXCTL15] = E1000_DCA_TXCTRL_DATA_RRO_EN |
+ E1000_DCA_TXCTRL_TX_WB_RO_EN |
+ E1000_DCA_TXCTRL_DESC_RRO_EN,
};
static void igb_reset(IGBCore *core, bool sw)
for (i = 0; i < ARRAY_SIZE(core->tx); i++) {
tx = &core->tx[i];
- net_tx_pkt_reset(tx->tx_pkt, NULL);
memset(tx->ctx, 0, sizeof(tx->ctx));
tx->first = true;
tx->skip_cp = false;
} wb;
};
-#define E1000_ADVTXD_DTYP_CTXT 0x00200000 /* Advanced Context Descriptor */
-#define E1000_ADVTXD_DTYP_DATA 0x00300000 /* Advanced Data Descriptor */
-#define E1000_ADVTXD_DCMD_DEXT 0x20000000 /* Descriptor Extension (1=Adv) */
-#define E1000_ADVTXD_DCMD_TSE 0x80000000 /* TCP/UDP Segmentation Enable */
-
#define E1000_ADVTXD_POTS_IXSM 0x00000100 /* Insert TCP/UDP Checksum */
#define E1000_ADVTXD_POTS_TXSM 0x00000200 /* Insert TCP/UDP Checksum */
#define IGB_82576_VF_DEV_ID 0x10CA
#define IGB_I350_VF_DEV_ID 0x1520
+/* VLAN info */
+#define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
+#define IGB_TX_FLAGS_VLAN_SHIFT 16
+
/* from igb/e1000_82575.h */
#define E1000_MRQC_ENABLE_RSS_MQ 0x00000002
#define E1000_MRQC_RSS_FIELD_IPV6_UDP 0x00800000
#define E1000_MRQC_RSS_FIELD_IPV6_UDP_EX 0x01000000
+/* Adv Transmit Descriptor Config Masks */
+#define E1000_ADVTXD_MAC_TSTAMP 0x00080000 /* IEEE1588 Timestamp packet */
+#define E1000_ADVTXD_DTYP_CTXT 0x00200000 /* Advanced Context Descriptor */
+#define E1000_ADVTXD_DTYP_DATA 0x00300000 /* Advanced Data Descriptor */
+#define E1000_ADVTXD_DCMD_EOP 0x01000000 /* End of Packet */
+#define E1000_ADVTXD_DCMD_IFCS 0x02000000 /* Insert FCS (Ethernet CRC) */
+#define E1000_ADVTXD_DCMD_RS 0x08000000 /* Report Status */
+#define E1000_ADVTXD_DCMD_DEXT 0x20000000 /* Descriptor extension (1=Adv) */
+#define E1000_ADVTXD_DCMD_VLE 0x40000000 /* VLAN pkt enable */
+#define E1000_ADVTXD_DCMD_TSE 0x80000000 /* TCP Seg enable */
+#define E1000_ADVTXD_PAYLEN_SHIFT 14 /* Adv desc PAYLEN shift */
+
+#define E1000_ADVTXD_MACLEN_SHIFT 9 /* Adv ctxt desc mac len shift */
+#define E1000_ADVTXD_TUCMD_L4T_UDP 0x00000000 /* L4 Packet TYPE of UDP */
+#define E1000_ADVTXD_TUCMD_IPV4 0x00000400 /* IP Packet Type: 1=IPv4 */
+#define E1000_ADVTXD_TUCMD_L4T_TCP 0x00000800 /* L4 Packet TYPE of TCP */
+#define E1000_ADVTXD_TUCMD_L4T_SCTP 0x00001000 /* L4 packet TYPE of SCTP */
+/* IPSec Encrypt Enable for ESP */
+#define E1000_ADVTXD_L4LEN_SHIFT 8 /* Adv ctxt L4LEN shift */
+#define E1000_ADVTXD_MSS_SHIFT 16 /* Adv ctxt MSS shift */
+/* Adv ctxt IPSec SA IDX mask */
+/* Adv ctxt IPSec ESP len mask */
+
/* Additional Transmit Descriptor Control definitions */
#define E1000_TXDCTL_QUEUE_ENABLE 0x02000000 /* Enable specific Tx Queue */
#define E1000_DCA_TXCTRL_CPUID_SHIFT 24 /* Tx CPUID now in the last byte */
#define E1000_DCA_RXCTRL_CPUID_SHIFT 24 /* Rx CPUID now in the last byte */
+/* ETQF register bit definitions */
+#define E1000_ETQF_FILTER_ENABLE BIT(26)
+#define E1000_ETQF_1588 BIT(30)
+#define E1000_ETQF_IMM_INT BIT(29)
+#define E1000_ETQF_QUEUE_ENABLE BIT(31)
+#define E1000_ETQF_QUEUE_SHIFT 16
+#define E1000_ETQF_QUEUE_MASK 0x00070000
+#define E1000_ETQF_ETYPE_MASK 0x0000FFFF
+
#define E1000_DTXSWC_MAC_SPOOF_MASK 0x000000FF /* Per VF MAC spoof control */
#define E1000_DTXSWC_VLAN_SPOOF_MASK 0x0000FF00 /* Per VF VLAN spoof control */
#define E1000_DTXSWC_LLE_MASK 0x00FF0000 /* Per VF Local LB enables */
/* E1000_EITR_CNT_IGNR is only for 82576 and newer */
#define E1000_EITR_CNT_IGNR 0x80000000 /* Don't reset counters on write */
+#define E1000_TSYNCTXCTL_VALID 0x00000001 /* tx timestamp valid */
+#define E1000_TSYNCTXCTL_ENABLED 0x00000010 /* enable tx timestampping */
+
/* PCI Express Control */
#define E1000_GCR_CMPL_TMOUT_MASK 0x0000F000
#define E1000_GCR_CMPL_TMOUT_10ms 0x00001000
#define E1000_FRTIMER 0x01048 /* Free Running Timer - RW */
#define E1000_FCRTV 0x02460 /* Flow Control Refresh Timer Value - RW */
+#define E1000_TSYNCRXCFG 0x05F50 /* Time Sync Rx Configuration - RW */
+
+/* Filtering Registers */
+#define E1000_SAQF(_n) (0x5980 + 4 * (_n))
+#define E1000_DAQF(_n) (0x59A0 + 4 * (_n))
+#define E1000_SPQF(_n) (0x59C0 + 4 * (_n))
+#define E1000_FTQF(_n) (0x59E0 + 4 * (_n))
+#define E1000_SAQF0 E1000_SAQF(0)
+#define E1000_DAQF0 E1000_DAQF(0)
+#define E1000_SPQF0 E1000_SPQF(0)
+#define E1000_FTQF0 E1000_FTQF(0)
+#define E1000_SYNQF(_n) (0x055FC + (4 * (_n))) /* SYN Packet Queue Fltr */
+#define E1000_ETQF(_n) (0x05CB0 + (4 * (_n))) /* EType Queue Fltr */
+
#define E1000_RQDPC(_n) (0x0C030 + ((_n) * 0x40))
#define E1000_RXPBS 0x02404 /* Rx Packet Buffer Size - RW */
#define E1000_RSS_QUEUE(reta, hash) (E1000_RETA_VAL(reta, hash) & 0x0F)
+#define E1000_MRQ_RSS_TYPE_IPV4UDP 7
+#define E1000_MRQ_RSS_TYPE_IPV6UDP 8
+
#define E1000_STATUS_IOV_MODE 0x00040000
#define E1000_STATUS_NUM_VFS_SHIFT 14
+#define E1000_ADVRXD_PKT_IP4 BIT(4)
+#define E1000_ADVRXD_PKT_IP6 BIT(6)
+#define E1000_ADVRXD_PKT_TCP BIT(8)
+#define E1000_ADVRXD_PKT_UDP BIT(9)
+#define E1000_ADVRXD_PKT_SCTP BIT(10)
+
static inline uint8_t igb_ivar_entry_rx(uint8_t i)
{
return i < 8 ? i * 4 : (i - 8) * 4 + 2;
#include "trace.h"
#include "qapi/error.h"
-#define TYPE_IGBVF "igbvf"
OBJECT_DECLARE_SIMPLE_TYPE(IgbVfState, IGBVF)
-#define IGBVF_MMIO_BAR_IDX (0)
-#define IGBVF_MSIX_BAR_IDX (3)
-
-#define IGBVF_MMIO_SIZE (16 * 1024)
-#define IGBVF_MSIX_SIZE (16 * 1024)
-
struct IgbVfState {
PCIDevice parent_obj;
*/
#include "qemu/osdep.h"
+#include "qemu/crc32c.h"
#include "trace.h"
#include "net_rx_pkt.h"
#include "net/checksum.h"
struct NetRxPkt {
struct virtio_net_hdr virt_hdr;
- uint8_t ehdr_buf[sizeof(struct eth_header) + sizeof(struct vlan_header)];
+ struct {
+ struct eth_header eth;
+ struct vlan_header vlan;
+ } ehdr_buf;
struct iovec *vec;
uint16_t vec_len_total;
uint16_t vec_len;
if (pkt->ehdr_buf_len) {
net_rx_pkt_iovec_realloc(pkt, iovcnt + 1);
- pkt->vec[0].iov_base = pkt->ehdr_buf;
+ pkt->vec[0].iov_base = &pkt->ehdr_buf;
pkt->vec[0].iov_len = pkt->ehdr_buf_len;
pkt->tot_len = pllen + pkt->ehdr_buf_len;
iov, iovcnt, ploff, pkt->tot_len);
}
- eth_get_protocols(pkt->vec, pkt->vec_len, &pkt->hasip4, &pkt->hasip6,
+ eth_get_protocols(pkt->vec, pkt->vec_len, 0, &pkt->hasip4, &pkt->hasip6,
&pkt->l3hdr_off, &pkt->l4hdr_off, &pkt->l5hdr_off,
&pkt->ip6hdr_info, &pkt->ip4hdr_info, &pkt->l4hdr_info);
assert(pkt);
if (strip_vlan) {
- pkt->ehdr_buf_len = eth_strip_vlan(iov, iovcnt, iovoff, pkt->ehdr_buf,
+ pkt->ehdr_buf_len = eth_strip_vlan(iov, iovcnt, iovoff, &pkt->ehdr_buf,
&ploff, &tci);
} else {
pkt->ehdr_buf_len = 0;
void net_rx_pkt_attach_iovec_ex(struct NetRxPkt *pkt,
const struct iovec *iov, int iovcnt,
- size_t iovoff, bool strip_vlan,
- uint16_t vet)
+ size_t iovoff, int strip_vlan_index,
+ uint16_t vet, uint16_t vet_ext)
{
uint16_t tci = 0;
uint16_t ploff = iovoff;
assert(pkt);
- if (strip_vlan) {
- pkt->ehdr_buf_len = eth_strip_vlan_ex(iov, iovcnt, iovoff, vet,
- pkt->ehdr_buf,
- &ploff, &tci);
- } else {
- pkt->ehdr_buf_len = 0;
- }
+ pkt->ehdr_buf_len = eth_strip_vlan_ex(iov, iovcnt, iovoff,
+ strip_vlan_index, vet, vet_ext,
+ &pkt->ehdr_buf,
+ &ploff, &tci);
pkt->tci = tci;
return pkt->tot_len;
}
-void net_rx_pkt_set_protocols(struct NetRxPkt *pkt, const void *data,
- size_t len)
+void net_rx_pkt_set_protocols(struct NetRxPkt *pkt,
+ const struct iovec *iov, size_t iovcnt,
+ size_t iovoff)
{
- const struct iovec iov = {
- .iov_base = (void *)data,
- .iov_len = len
- };
-
assert(pkt);
- eth_get_protocols(&iov, 1, &pkt->hasip4, &pkt->hasip6,
+ eth_get_protocols(iov, iovcnt, iovoff, &pkt->hasip4, &pkt->hasip6,
&pkt->l3hdr_off, &pkt->l4hdr_off, &pkt->l5hdr_off,
&pkt->ip6hdr_info, &pkt->ip4hdr_info, &pkt->l4hdr_info);
}
return &pkt->ip4hdr_info;
}
-eth_l4_hdr_info *net_rx_pkt_get_l4_info(struct NetRxPkt *pkt)
-{
- return &pkt->l4hdr_info;
-}
-
static inline void
_net_rx_rss_add_chunk(uint8_t *rss_input, size_t *bytes_written,
void *ptr, size_t size)
return csum;
}
-bool net_rx_pkt_validate_l4_csum(struct NetRxPkt *pkt, bool *csum_valid)
+static bool
+_net_rx_pkt_validate_sctp_sum(struct NetRxPkt *pkt)
{
- uint16_t csum;
+ size_t csum_off;
+ size_t off = pkt->l4hdr_off;
+ size_t vec_len = pkt->vec_len;
+ struct iovec *vec;
+ uint32_t calculated = 0;
+ uint32_t original;
+ bool valid;
- trace_net_rx_pkt_l4_csum_validate_entry();
+ for (vec = pkt->vec; vec->iov_len < off; vec++) {
+ off -= vec->iov_len;
+ vec_len--;
+ }
- if (pkt->l4hdr_info.proto != ETH_L4_HDR_PROTO_TCP &&
- pkt->l4hdr_info.proto != ETH_L4_HDR_PROTO_UDP) {
- trace_net_rx_pkt_l4_csum_validate_not_xxp();
+ csum_off = off + 8;
+
+ if (!iov_to_buf(vec, vec_len, csum_off, &original, sizeof(original))) {
return false;
}
- if (pkt->l4hdr_info.proto == ETH_L4_HDR_PROTO_UDP &&
- pkt->l4hdr_info.hdr.udp.uh_sum == 0) {
- trace_net_rx_pkt_l4_csum_validate_udp_with_no_checksum();
+ if (!iov_from_buf(vec, vec_len, csum_off,
+ &calculated, sizeof(calculated))) {
return false;
}
+ calculated = crc32c(0xffffffff,
+ (uint8_t *)vec->iov_base + off, vec->iov_len - off);
+ calculated = iov_crc32c(calculated ^ 0xffffffff, vec + 1, vec_len - 1);
+ valid = calculated == le32_to_cpu(original);
+ iov_from_buf(vec, vec_len, csum_off, &original, sizeof(original));
+
+ return valid;
+}
+
+bool net_rx_pkt_validate_l4_csum(struct NetRxPkt *pkt, bool *csum_valid)
+{
+ uint32_t csum;
+
+ trace_net_rx_pkt_l4_csum_validate_entry();
+
if (pkt->hasip4 && pkt->ip4hdr_info.fragment) {
trace_net_rx_pkt_l4_csum_validate_ip4_fragment();
return false;
}
- csum = _net_rx_pkt_calc_l4_csum(pkt);
+ switch (pkt->l4hdr_info.proto) {
+ case ETH_L4_HDR_PROTO_UDP:
+ if (pkt->l4hdr_info.hdr.udp.uh_sum == 0) {
+ trace_net_rx_pkt_l4_csum_validate_udp_with_no_checksum();
+ return false;
+ }
+ /* fall through */
+ case ETH_L4_HDR_PROTO_TCP:
+ csum = _net_rx_pkt_calc_l4_csum(pkt);
+ *csum_valid = ((csum == 0) || (csum == 0xFFFF));
+ break;
+
+ case ETH_L4_HDR_PROTO_SCTP:
+ *csum_valid = _net_rx_pkt_validate_sctp_sum(pkt);
+ break;
- *csum_valid = ((csum == 0) || (csum == 0xFFFF));
+ default:
+ trace_net_rx_pkt_l4_csum_validate_not_xxp();
+ return false;
+ }
trace_net_rx_pkt_l4_csum_validate_csum(*csum_valid);
* parse and set packet analysis results
*
* @pkt: packet
- * @data: pointer to the data buffer to be parsed
- * @len: data length
+ * @iov: received data scatter-gather list
+ * @iovcnt: number of elements in iov
+ * @iovoff: data start offset in the iov
*
*/
-void net_rx_pkt_set_protocols(struct NetRxPkt *pkt, const void *data,
- size_t len);
+void net_rx_pkt_set_protocols(struct NetRxPkt *pkt,
+ const struct iovec *iov, size_t iovcnt,
+ size_t iovoff);
/**
* fetches packet analysis results
*/
eth_ip4_hdr_info *net_rx_pkt_get_ip4_info(struct NetRxPkt *pkt);
-/**
- * fetches L4 header analysis results
- *
- * Return: pointer to analysis results structure which is stored in internal
- * packet area.
- *
- */
-eth_l4_hdr_info *net_rx_pkt_get_l4_info(struct NetRxPkt *pkt);
-
typedef enum {
NetPktRssIpV4,
NetPktRssIpV4Tcp,
/**
* attach scatter-gather data to rx packet
*
-* @pkt: packet
-* @iov: received data scatter-gather list
-* @iovcnt number of elements in iov
-* @iovoff data start offset in the iov
-* @strip_vlan: should the module strip vlan from data
-* @vet: VLAN tag Ethernet type
+* @pkt: packet
+* @iov: received data scatter-gather list
+* @iovcnt: number of elements in iov
+* @iovoff: data start offset in the iov
+* @strip_vlan_index: index of Q tag if it is to be stripped. negative otherwise.
+* @vet: VLAN tag Ethernet type
+* @vet_ext: outer VLAN tag Ethernet type
*
*/
void net_rx_pkt_attach_iovec_ex(struct NetRxPkt *pkt,
- const struct iovec *iov, int iovcnt,
- size_t iovoff, bool strip_vlan,
- uint16_t vet);
+ const struct iovec *iov, int iovcnt,
+ size_t iovoff, int strip_vlan_index,
+ uint16_t vet, uint16_t vet_ext);
/**
* attach data to rx packet
*/
#include "qemu/osdep.h"
-#include "net_tx_pkt.h"
+#include "qemu/crc32c.h"
#include "net/eth.h"
#include "net/checksum.h"
#include "net/tap.h"
#include "net/net.h"
#include "hw/pci/pci_device.h"
+#include "net_tx_pkt.h"
enum {
NET_TX_PKT_VHDR_FRAG = 0,
/* TX packet private context */
struct NetTxPkt {
- PCIDevice *pci_dev;
-
struct virtio_net_hdr virt_hdr;
struct iovec *raw;
struct iovec *vec;
- uint8_t l2_hdr[ETH_MAX_L2_HDR_LEN];
+ struct {
+ struct eth_header eth;
+ struct vlan_header vlan[3];
+ } l2_hdr;
union {
struct ip_header ip;
struct ip6_header ip6;
uint8_t l4proto;
};
-void net_tx_pkt_init(struct NetTxPkt **pkt, PCIDevice *pci_dev,
- uint32_t max_frags)
+void net_tx_pkt_init(struct NetTxPkt **pkt, uint32_t max_frags)
{
struct NetTxPkt *p = g_malloc0(sizeof *p);
- p->pci_dev = pci_dev;
-
p->vec = g_new(struct iovec, max_frags + NET_TX_PKT_PL_START_FRAG);
p->raw = g_new(struct iovec, max_frags);
pkt->virt_hdr.csum_offset, &csum, sizeof(csum));
}
+bool net_tx_pkt_update_sctp_checksum(struct NetTxPkt *pkt)
+{
+ uint32_t csum = 0;
+ struct iovec *pl_start_frag = pkt->vec + NET_TX_PKT_PL_START_FRAG;
+
+ if (iov_from_buf(pl_start_frag, pkt->payload_frags, 8, &csum, sizeof(csum)) < sizeof(csum)) {
+ return false;
+ }
+
+ csum = cpu_to_le32(iov_crc32c(0xffffffff, pl_start_frag, pkt->payload_frags));
+ if (iov_from_buf(pl_start_frag, pkt->payload_frags, 8, &csum, sizeof(csum)) < sizeof(csum)) {
+ return false;
+ }
+
+ return true;
+}
+
static void net_tx_pkt_calculate_hdr_len(struct NetTxPkt *pkt)
{
pkt->hdr_len = pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len +
void net_tx_pkt_setup_vlan_header_ex(struct NetTxPkt *pkt,
uint16_t vlan, uint16_t vlan_ethtype)
{
- bool is_new;
assert(pkt);
- eth_setup_vlan_headers_ex(pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_base,
- vlan, vlan_ethtype, &is_new);
+ eth_setup_vlan_headers(pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_base,
+ &pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len,
+ vlan, vlan_ethtype);
- /* update l2hdrlen */
- if (is_new) {
- pkt->hdr_len += sizeof(struct vlan_header);
- pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len +=
- sizeof(struct vlan_header);
- }
+ pkt->hdr_len += sizeof(struct vlan_header);
}
-bool net_tx_pkt_add_raw_fragment(struct NetTxPkt *pkt, hwaddr pa,
- size_t len)
+bool net_tx_pkt_add_raw_fragment(struct NetTxPkt *pkt, void *base, size_t len)
{
- hwaddr mapped_len = 0;
struct iovec *ventry;
assert(pkt);
return false;
}
- if (!len) {
- return true;
- }
-
ventry = &pkt->raw[pkt->raw_frags];
- mapped_len = len;
-
- ventry->iov_base = pci_dma_map(pkt->pci_dev, pa,
- &mapped_len, DMA_DIRECTION_TO_DEVICE);
+ ventry->iov_base = base;
+ ventry->iov_len = len;
+ pkt->raw_frags++;
- if ((ventry->iov_base != NULL) && (len == mapped_len)) {
- ventry->iov_len = mapped_len;
- pkt->raw_frags++;
- return true;
- } else {
- return false;
- }
+ return true;
}
bool net_tx_pkt_has_fragments(struct NetTxPkt *pkt)
#endif
}
-void net_tx_pkt_reset(struct NetTxPkt *pkt, PCIDevice *pci_dev)
+void net_tx_pkt_reset(struct NetTxPkt *pkt,
+ NetTxPktFreeFrag callback, void *context)
{
int i;
assert(pkt->raw);
for (i = 0; i < pkt->raw_frags; i++) {
assert(pkt->raw[i].iov_base);
- pci_dma_unmap(pkt->pci_dev, pkt->raw[i].iov_base,
- pkt->raw[i].iov_len, DMA_DIRECTION_TO_DEVICE, 0);
+ callback(context, pkt->raw[i].iov_base, pkt->raw[i].iov_len);
}
}
- pkt->pci_dev = pci_dev;
pkt->raw_frags = 0;
pkt->hdr_len = 0;
pkt->l4proto = 0;
}
+void net_tx_pkt_unmap_frag_pci(void *context, void *base, size_t len)
+{
+ pci_dma_unmap(context, base, len, DMA_DIRECTION_TO_DEVICE, 0);
+}
+
+bool net_tx_pkt_add_raw_fragment_pci(struct NetTxPkt *pkt, PCIDevice *pci_dev,
+ dma_addr_t pa, size_t len)
+{
+ dma_addr_t mapped_len = len;
+ void *base = pci_dma_map(pci_dev, pa, &mapped_len, DMA_DIRECTION_TO_DEVICE);
+ if (!base) {
+ return false;
+ }
+
+ if (mapped_len != len || !net_tx_pkt_add_raw_fragment(pkt, base, len)) {
+ net_tx_pkt_unmap_frag_pci(pci_dev, base, mapped_len);
+ return false;
+ }
+
+ return true;
+}
+
static void net_tx_pkt_do_sw_csum(struct NetTxPkt *pkt,
struct iovec *iov, uint32_t iov_len,
uint16_t csl)
}
static bool net_tx_pkt_do_sw_fragmentation(struct NetTxPkt *pkt,
- NetTxPktCallback callback,
+ NetTxPktSend callback,
void *context)
{
uint8_t gso_type = pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN;
}
bool net_tx_pkt_send_custom(struct NetTxPkt *pkt, bool offload,
- NetTxPktCallback callback, void *context)
+ NetTxPktSend callback, void *context)
{
assert(pkt);
struct NetTxPkt;
-typedef void (* NetTxPktCallback)(void *, const struct iovec *, int, const struct iovec *, int);
+typedef void (*NetTxPktFreeFrag)(void *, void *, size_t);
+typedef void (*NetTxPktSend)(void *, const struct iovec *, int, const struct iovec *, int);
/**
* Init function for tx packet functionality
*
* @pkt: packet pointer
- * @pci_dev: PCI device processing this packet
* @max_frags: max tx ip fragments
*/
-void net_tx_pkt_init(struct NetTxPkt **pkt, PCIDevice *pci_dev,
- uint32_t max_frags);
+void net_tx_pkt_init(struct NetTxPkt **pkt, uint32_t max_frags);
/**
* Clean all tx packet resources.
* populate data fragment into pkt context.
*
* @pkt: packet
- * @pa: physical address of fragment
+ * @base: pointer to fragment
* @len: length of fragment
*
*/
-bool net_tx_pkt_add_raw_fragment(struct NetTxPkt *pkt, hwaddr pa,
- size_t len);
+bool net_tx_pkt_add_raw_fragment(struct NetTxPkt *pkt, void *base, size_t len);
/**
* Fix ip header fields and calculate IP header and pseudo header checksums.
*/
void net_tx_pkt_update_ip_hdr_checksum(struct NetTxPkt *pkt);
+/**
+ * Calculate the SCTP checksum.
+ *
+ * @pkt: packet
+ *
+ */
+bool net_tx_pkt_update_sctp_checksum(struct NetTxPkt *pkt);
+
/**
* get length of all populated data.
*
* reset tx packet private context (needed to be called between packets)
*
* @pkt: packet
- * @dev: PCI device processing the next packet
+ * @callback: function to free the fragments
+ * @context: pointer to be passed to the callback
+ */
+void net_tx_pkt_reset(struct NetTxPkt *pkt,
+ NetTxPktFreeFrag callback, void *context);
+
+/**
+ * Unmap a fragment mapped from a PCI device.
+ *
+ * @context: PCI device owning fragment
+ * @base: pointer to fragment
+ * @len: length of fragment
+ */
+void net_tx_pkt_unmap_frag_pci(void *context, void *base, size_t len);
+
+/**
+ * map data fragment from PCI device and populate it into pkt context.
*
+ * @pci_dev: PCI device owning fragment
+ * @pa: physical address of fragment
+ * @len: length of fragment
*/
-void net_tx_pkt_reset(struct NetTxPkt *pkt, PCIDevice *dev);
+bool net_tx_pkt_add_raw_fragment_pci(struct NetTxPkt *pkt, PCIDevice *pci_dev,
+ dma_addr_t pa, size_t len);
/**
* Send packet to qemu. handles sw offloads if vhdr is not supported.
* @ret: operation result
*/
bool net_tx_pkt_send_custom(struct NetTxPkt *pkt, bool offload,
- NetTxPktCallback callback, void *context);
+ NetTxPktSend callback, void *context);
/**
* parse raw packet data and analyze offload requirements.
int large_send_mss = (txdw0 >> CP_TC_LGSEN_MSS_SHIFT) &
CP_TC_LGSEN_MSS_MASK;
+ if (large_send_mss == 0) {
+ goto skip_offload;
+ }
DPRINTF("+++ C+ mode offloaded task TSO IP data %d "
"frame data %d specified MSS=%d\n",
# e1000x_common.c
e1000x_rx_can_recv_disabled(bool link_up, bool rx_enabled, bool pci_master) "link_up: %d, rx_enabled %d, pci_master %d"
e1000x_vlan_is_vlan_pkt(bool is_vlan_pkt, uint16_t eth_proto, uint16_t vet) "Is VLAN packet: %d, ETH proto: 0x%X, VET: 0x%X"
+e1000x_rx_flt_vlan_mismatch(uint16_t vid) "VID mismatch: 0x%X"
+e1000x_rx_flt_vlan_match(uint16_t vid) "VID match: 0x%X"
e1000x_rx_flt_ucast_match(uint32_t idx, uint8_t b0, uint8_t b1, uint8_t b2, uint8_t b3, uint8_t b4, uint8_t b5) "unicast match[%d]: %02x:%02x:%02x:%02x:%02x:%02x"
e1000x_rx_flt_ucast_mismatch(uint8_t b0, uint8_t b1, uint8_t b2, uint8_t b3, uint8_t b4, uint8_t b5) "unicast mismatch: %02x:%02x:%02x:%02x:%02x:%02x"
e1000x_rx_flt_inexact_mismatch(uint8_t b0, uint8_t b1, uint8_t b2, uint8_t b3, uint8_t b4, uint8_t b5, uint32_t mo, uint32_t mta, uint32_t mta_val) "inexact mismatch: %02x:%02x:%02x:%02x:%02x:%02x MO %d MTA[%d] 0x%x"
e1000e_rx_can_recv(void) "Can receive"
e1000e_rx_has_buffers(int ridx, uint32_t free_desc, size_t total_size, uint32_t desc_buf_size) "ring #%d: free descr: %u, packet size %zu, descr buffer size %u"
e1000e_rx_null_descriptor(void) "Null RX descriptor!!"
-e1000e_rx_flt_vlan_mismatch(uint16_t vid) "VID mismatch: 0x%X"
-e1000e_rx_flt_vlan_match(uint16_t vid) "VID match: 0x%X"
e1000e_rx_desc_ps_read(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3) "buffers: [0x%"PRIx64", 0x%"PRIx64", 0x%"PRIx64", 0x%"PRIx64"]"
e1000e_rx_desc_ps_write(uint16_t a0, uint16_t a1, uint16_t a2, uint16_t a3) "bytes written: [%u, %u, %u, %u]"
e1000e_rx_desc_buff_sizes(uint32_t b0, uint32_t b1, uint32_t b2, uint32_t b3) "buffer sizes: [%u, %u, %u, %u]"
e1000e_rx_rss_type(uint32_t type) "RSS type is %u"
e1000e_rx_rss_ip4(int l4hdr_proto, uint32_t mrqc, bool tcpipv4_enabled, bool ipv4_enabled) "RSS IPv4: L4 header protocol %d, mrqc 0x%X, tcpipv4 enabled %d, ipv4 enabled %d"
e1000e_rx_rss_ip6_rfctl(uint32_t rfctl) "RSS IPv6: rfctl 0x%X"
-e1000e_rx_rss_ip6(bool ex_dis, bool new_ex_dis, int l4hdr_proto, bool has_ext_headers, bool ex_dst_valid, bool ex_src_valid, uint32_t mrqc, bool tcpipv6_enabled, bool ipv6ex_enabled, bool ipv6_enabled) "RSS IPv6: ex_dis: %d, new_ex_dis: %d, L4 header protocol %d, has_ext_headers %d, ex_dst_valid %d, ex_src_valid %d, mrqc 0x%X, tcpipv6 enabled %d, ipv6ex enabled %d, ipv6 enabled %d"
+e1000e_rx_rss_ip6(bool ex_dis, bool new_ex_dis, int l4hdr_proto, bool has_ext_headers, bool ex_dst_valid, bool ex_src_valid, uint32_t mrqc, bool tcpipv6ex_enabled, bool ipv6ex_enabled, bool ipv6_enabled) "RSS IPv6: ex_dis: %d, new_ex_dis: %d, L4 header protocol %d, has_ext_headers %d, ex_dst_valid %d, ex_src_valid %d, mrqc 0x%X, tcpipv6ex enabled %d, ipv6ex enabled %d, ipv6 enabled %d"
e1000e_rx_metadata_protocols(bool hasip4, bool hasip6, int l4hdr_protocol) "protocols: ip4: %d, ip6: %d, l4hdr: %d"
e1000e_rx_metadata_vlan(uint16_t vlan_tag) "VLAN tag is 0x%X"
e1000e_irq_legacy_notify(bool level) "IRQ line state: %d"
e1000e_irq_msix_notify_vec(uint32_t vector) "MSI-X notify vector 0x%x"
e1000e_irq_postponed_by_xitr(uint32_t reg) "Interrupt postponed by [E]ITR register 0x%x"
-e1000e_irq_clear_ims(uint32_t bits, uint32_t old_ims, uint32_t new_ims) "Clearing IMS bits 0x%x: 0x%x --> 0x%x"
-e1000e_irq_set_ims(uint32_t bits, uint32_t old_ims, uint32_t new_ims) "Setting IMS bits 0x%x: 0x%x --> 0x%x"
+e1000e_irq_clear(uint32_t offset, uint32_t old, uint32_t new) "Clearing interrupt register 0x%x: 0x%x --> 0x%x"
+e1000e_irq_set(uint32_t offset, uint32_t old, uint32_t new) "Setting interrupt register 0x%x: 0x%x --> 0x%x"
e1000e_irq_fix_icr_asserted(uint32_t new_val) "ICR_ASSERTED bit fixed: 0x%x"
e1000e_irq_add_msi_other(uint32_t new_val) "ICR_OTHER bit added: 0x%x"
e1000e_irq_pending_interrupts(uint32_t pending, uint32_t icr, uint32_t ims) "ICR PENDING: 0x%x (ICR: 0x%x, IMS: 0x%x)"
-e1000e_irq_set_cause_entry(uint32_t val, uint32_t icr) "Going to set IRQ cause 0x%x, ICR: 0x%x"
-e1000e_irq_set_cause_exit(uint32_t val, uint32_t icr) "Set IRQ cause 0x%x, ICR: 0x%x"
-e1000e_irq_icr_write(uint32_t bits, uint32_t old_icr, uint32_t new_icr) "Clearing ICR bits 0x%x: 0x%x --> 0x%x"
e1000e_irq_write_ics(uint32_t val) "Adding ICR bits 0x%x"
e1000e_irq_icr_process_iame(void) "Clearing IMS bits due to IAME"
e1000e_irq_read_ics(uint32_t ics) "Current ICS: 0x%x"
e1000e_irq_read_ims(uint32_t ims) "Current IMS: 0x%x"
e1000e_irq_icr_clear_nonmsix_icr_read(void) "Clearing ICR on read due to non MSI-X int"
-e1000e_irq_icr_read_entry(uint32_t icr) "Starting ICR read. Current ICR: 0x%x"
-e1000e_irq_icr_read_exit(uint32_t icr) "Ending ICR read. Current ICR: 0x%x"
e1000e_irq_icr_clear_zero_ims(void) "Clearing ICR on read due to zero IMS"
e1000e_irq_icr_clear_iame(void) "Clearing ICR on read due to IAME"
e1000e_irq_iam_clear_eiame(uint32_t iam, uint32_t cause) "Clearing IMS due to EIAME, IAM: 0x%X, cause: 0x%X"
e1000e_irq_eitr_set(uint32_t eitr_num, uint32_t val) "EITR[%u] = %u"
e1000e_irq_itr_set(uint32_t val) "ITR = %u"
e1000e_irq_fire_all_timers(uint32_t val) "Firing all delay/throttling timers on all interrupts enable (0x%X written to IMS)"
-e1000e_irq_adding_delayed_causes(uint32_t val, uint32_t icr) "Merging delayed causes 0x%X to ICR 0x%X"
e1000e_irq_msix_pending_clearing(uint32_t cause, uint32_t int_cfg, uint32_t vec) "Clearing MSI-X pending bit for cause 0x%x, IVAR config 0x%x, vector %u"
e1000e_wrn_msix_vec_wrong(uint32_t cause, uint32_t cfg) "Invalid configuration for cause 0x%x: 0x%x"
igb_rx_metadata_rss(uint32_t rss) "RSS data: 0x%X"
igb_irq_icr_clear_gpie_nsicr(void) "Clearing ICR on read due to GPIE.NSICR enabled"
-igb_irq_icr_write(uint32_t bits, uint32_t old_icr, uint32_t new_icr) "Clearing ICR bits 0x%x: 0x%x --> 0x%x"
igb_irq_set_iam(uint32_t icr) "Update IAM: 0x%x"
igb_irq_read_iam(uint32_t icr) "Current IAM: 0x%x"
igb_irq_write_eics(uint32_t val, bool msix) "Update EICS: 0x%x MSI-X: %d"
igb_irq_write_eims(uint32_t val, bool msix) "Update EIMS: 0x%x MSI-X: %d"
-igb_irq_write_eimc(uint32_t val, uint32_t eims, bool msix) "Update EIMC: 0x%x EIMS: 0x%x MSI-X: %d"
+igb_irq_write_eimc(uint32_t val, bool msix) "Update EIMC: 0x%x MSI-X: %d"
igb_irq_write_eiac(uint32_t val) "Update EIAC: 0x%x"
igb_irq_write_eiam(uint32_t val, bool msix) "Update EIAM: 0x%x MSI-X: %d"
igb_irq_write_eicr(uint32_t val, bool msix) "Update EICR: 0x%x MSI-X: %d"
VIRTIO_NET_HASH_REPORT_UDPv6,
VIRTIO_NET_HASH_REPORT_UDPv6_EX
};
+ struct iovec iov = {
+ .iov_base = (void *)buf,
+ .iov_len = size
+ };
- net_rx_pkt_set_protocols(pkt, buf + n->host_hdr_len,
- size - n->host_hdr_len);
+ net_rx_pkt_set_protocols(pkt, &iov, 1, n->host_hdr_len);
net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto);
net_hash_type = virtio_net_get_hash_type(hasip4, hasip6, l4hdr_proto,
n->rss_data.hash_types);
data_len = (txd.len > 0) ? txd.len : VMXNET3_MAX_TX_BUF_SIZE;
data_pa = txd.addr;
- if (!net_tx_pkt_add_raw_fragment(s->tx_pkt,
- data_pa,
- data_len)) {
+ if (!net_tx_pkt_add_raw_fragment_pci(s->tx_pkt, PCI_DEVICE(s),
+ data_pa, data_len)) {
s->skip_current_tx_pkt = true;
}
}
vmxnet3_complete_packet(s, qidx, txd_idx);
s->tx_sop = true;
s->skip_current_tx_pkt = false;
- net_tx_pkt_reset(s->tx_pkt, PCI_DEVICE(s));
+ net_tx_pkt_reset(s->tx_pkt,
+ net_tx_pkt_unmap_frag_pci, PCI_DEVICE(s));
}
}
+
+ net_tx_pkt_reset(s->tx_pkt, net_tx_pkt_unmap_frag_pci, PCI_DEVICE(s));
}
static inline void
{
if (s->device_active) {
VMW_CBPRN("Deactivating vmxnet3...");
- net_tx_pkt_reset(s->tx_pkt, PCI_DEVICE(s));
net_tx_pkt_uninit(s->tx_pkt);
net_rx_pkt_uninit(s->rx_pkt);
s->device_active = false;
/* Preallocate TX packet wrapper */
VMW_CFPRN("Max TX fragments is %u", s->max_tx_frags);
- net_tx_pkt_init(&s->tx_pkt, PCI_DEVICE(s), s->max_tx_frags);
+ net_tx_pkt_init(&s->tx_pkt, s->max_tx_frags);
net_rx_pkt_init(&s->rx_pkt);
/* Read rings memory locations for RX queues */
get_eth_packet_type(PKT_GET_ETH_HDR(buf)));
if (vmxnet3_rx_filter_may_indicate(s, buf, size)) {
- net_rx_pkt_set_protocols(s->rx_pkt, buf, size);
+ struct iovec iov = {
+ .iov_base = (void *)buf,
+ .iov_len = size
+ };
+
+ net_rx_pkt_set_protocols(s->rx_pkt, &iov, 1, 0);
vmxnet3_rx_need_csum_calculate(s->rx_pkt, buf, size);
net_rx_pkt_attach_data(s->rx_pkt, buf, size, s->rx_vlan_stripping);
bytes_indicated = vmxnet3_indicate_packet(s) ? size : -1;
{
VMXNET3State *s = opaque;
- net_tx_pkt_init(&s->tx_pkt, PCI_DEVICE(s), s->max_tx_frags);
+ net_tx_pkt_init(&s->tx_pkt, s->max_tx_frags);
net_rx_pkt_init(&s->rx_pkt);
if (s->msix_used) {
qemu_uuid_unparse(&vdev->vf_token, uuid);
name = g_strdup_printf("%s vf_token=%s", vbasedev->name, uuid);
} else {
- name = vbasedev->name;
+ name = g_strdup(vbasedev->name);
}
ret = vfio_get_device(group, name, vbasedev, errp);
assert((uintptr_t)addr == addr);
mr = memory_region_from_host((void *)(uintptr_t)addr, offset);
*fd = memory_region_get_fd(mr);
+ *offset += mr->ram_block->fd_offset;
return mr;
}
int mmu_idx, uintptr_t ra);
uint8_t cpu_ldb_mmu(CPUArchState *env, abi_ptr ptr, MemOpIdx oi, uintptr_t ra);
-uint16_t cpu_ldw_be_mmu(CPUArchState *env, abi_ptr ptr,
- MemOpIdx oi, uintptr_t ra);
-uint32_t cpu_ldl_be_mmu(CPUArchState *env, abi_ptr ptr,
- MemOpIdx oi, uintptr_t ra);
-uint64_t cpu_ldq_be_mmu(CPUArchState *env, abi_ptr ptr,
- MemOpIdx oi, uintptr_t ra);
-uint16_t cpu_ldw_le_mmu(CPUArchState *env, abi_ptr ptr,
- MemOpIdx oi, uintptr_t ra);
-uint32_t cpu_ldl_le_mmu(CPUArchState *env, abi_ptr ptr,
- MemOpIdx oi, uintptr_t ra);
-uint64_t cpu_ldq_le_mmu(CPUArchState *env, abi_ptr ptr,
- MemOpIdx oi, uintptr_t ra);
-
-Int128 cpu_ld16_be_mmu(CPUArchState *env, abi_ptr addr,
- MemOpIdx oi, uintptr_t ra);
-Int128 cpu_ld16_le_mmu(CPUArchState *env, abi_ptr addr,
- MemOpIdx oi, uintptr_t ra);
+uint16_t cpu_ldw_mmu(CPUArchState *env, abi_ptr ptr, MemOpIdx oi, uintptr_t ra);
+uint32_t cpu_ldl_mmu(CPUArchState *env, abi_ptr ptr, MemOpIdx oi, uintptr_t ra);
+uint64_t cpu_ldq_mmu(CPUArchState *env, abi_ptr ptr, MemOpIdx oi, uintptr_t ra);
+Int128 cpu_ld16_mmu(CPUArchState *env, abi_ptr addr, MemOpIdx oi, uintptr_t ra);
void cpu_stb_mmu(CPUArchState *env, abi_ptr ptr, uint8_t val,
MemOpIdx oi, uintptr_t ra);
-void cpu_stw_be_mmu(CPUArchState *env, abi_ptr ptr, uint16_t val,
- MemOpIdx oi, uintptr_t ra);
-void cpu_stl_be_mmu(CPUArchState *env, abi_ptr ptr, uint32_t val,
- MemOpIdx oi, uintptr_t ra);
-void cpu_stq_be_mmu(CPUArchState *env, abi_ptr ptr, uint64_t val,
- MemOpIdx oi, uintptr_t ra);
-void cpu_stw_le_mmu(CPUArchState *env, abi_ptr ptr, uint16_t val,
- MemOpIdx oi, uintptr_t ra);
-void cpu_stl_le_mmu(CPUArchState *env, abi_ptr ptr, uint32_t val,
- MemOpIdx oi, uintptr_t ra);
-void cpu_stq_le_mmu(CPUArchState *env, abi_ptr ptr, uint64_t val,
- MemOpIdx oi, uintptr_t ra);
-
-void cpu_st16_be_mmu(CPUArchState *env, abi_ptr addr, Int128 val,
- MemOpIdx oi, uintptr_t ra);
-void cpu_st16_le_mmu(CPUArchState *env, abi_ptr addr, Int128 val,
- MemOpIdx oi, uintptr_t ra);
+void cpu_stw_mmu(CPUArchState *env, abi_ptr ptr, uint16_t val,
+ MemOpIdx oi, uintptr_t ra);
+void cpu_stl_mmu(CPUArchState *env, abi_ptr ptr, uint32_t val,
+ MemOpIdx oi, uintptr_t ra);
+void cpu_stq_mmu(CPUArchState *env, abi_ptr ptr, uint64_t val,
+ MemOpIdx oi, uintptr_t ra);
+void cpu_st16_mmu(CPUArchState *env, abi_ptr addr, Int128 val,
+ MemOpIdx oi, uintptr_t ra);
uint32_t cpu_atomic_cmpxchgb_mmu(CPUArchState *env, target_ulong addr,
uint32_t cmpv, uint32_t newv,
Int128 cmpv, Int128 newv,
MemOpIdx oi, uintptr_t retaddr);
-Int128 cpu_atomic_ldo_le_mmu(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, uintptr_t retaddr);
-Int128 cpu_atomic_ldo_be_mmu(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, uintptr_t retaddr);
-void cpu_atomic_sto_le_mmu(CPUArchState *env, target_ulong addr, Int128 val,
- MemOpIdx oi, uintptr_t retaddr);
-void cpu_atomic_sto_be_mmu(CPUArchState *env, target_ulong addr, Int128 val,
- MemOpIdx oi, uintptr_t retaddr);
-
#if defined(CONFIG_USER_ONLY)
extern __thread uintptr_t helper_retaddr;
# define cpu_ldsw_mmuidx_ra cpu_ldsw_be_mmuidx_ra
# define cpu_ldl_mmuidx_ra cpu_ldl_be_mmuidx_ra
# define cpu_ldq_mmuidx_ra cpu_ldq_be_mmuidx_ra
-# define cpu_ldw_mmu cpu_ldw_be_mmu
-# define cpu_ldl_mmu cpu_ldl_be_mmu
-# define cpu_ldq_mmu cpu_ldq_be_mmu
# define cpu_stw_data cpu_stw_be_data
# define cpu_stl_data cpu_stl_be_data
# define cpu_stq_data cpu_stq_be_data
# define cpu_stw_mmuidx_ra cpu_stw_be_mmuidx_ra
# define cpu_stl_mmuidx_ra cpu_stl_be_mmuidx_ra
# define cpu_stq_mmuidx_ra cpu_stq_be_mmuidx_ra
-# define cpu_stw_mmu cpu_stw_be_mmu
-# define cpu_stl_mmu cpu_stl_be_mmu
-# define cpu_stq_mmu cpu_stq_be_mmu
#else
# define cpu_lduw_data cpu_lduw_le_data
# define cpu_ldsw_data cpu_ldsw_le_data
# define cpu_ldsw_mmuidx_ra cpu_ldsw_le_mmuidx_ra
# define cpu_ldl_mmuidx_ra cpu_ldl_le_mmuidx_ra
# define cpu_ldq_mmuidx_ra cpu_ldq_le_mmuidx_ra
-# define cpu_ldw_mmu cpu_ldw_le_mmu
-# define cpu_ldl_mmu cpu_ldl_le_mmu
-# define cpu_ldq_mmu cpu_ldq_le_mmu
# define cpu_stw_data cpu_stw_le_data
# define cpu_stl_data cpu_stl_le_data
# define cpu_stq_data cpu_stq_le_data
# define cpu_stw_mmuidx_ra cpu_stw_le_mmuidx_ra
# define cpu_stl_mmuidx_ra cpu_stl_le_mmuidx_ra
# define cpu_stq_mmuidx_ra cpu_stq_le_mmuidx_ra
-# define cpu_stw_mmu cpu_stw_le_mmu
-# define cpu_stl_mmu cpu_stl_le_mmu
-# define cpu_stq_mmu cpu_stq_le_mmu
#endif
uint8_t cpu_ldb_code_mmu(CPUArchState *env, abi_ptr addr,
#include "qemu/interval-tree.h"
#include "qemu/clang-tsa.h"
-/* allow to see translation results - the slowdown should be negligible, so we leave it */
-#define DEBUG_DISAS
-
/* Page tracking code uses ram addresses in system mode, and virtual
addresses in userspace mode. Define tb_page_addr_t to be an appropriate
type. */
* @ram_flags: RamBlock flags. Supported flags: RAM_SHARED, RAM_PMEM,
* RAM_NORESERVE,
* @path: the path in which to allocate the RAM.
+ * @offset: offset within the file referenced by path
* @readonly: true to open @path for reading, false for read/write.
* @errp: pointer to Error*, to store an error if it happens.
*
uint64_t align,
uint32_t ram_flags,
const char *path,
+ ram_addr_t offset,
bool readonly,
Error **errp);
* @ram_flags: RamBlock flags. Supported flags: RAM_SHARED, RAM_PMEM,
* RAM_NORESERVE.
* @mem_path or @fd: specify the backing file or device
+ * @offset: Offset into target file
* @readonly: true to open @path for reading, false for read/write.
* @errp: pointer to Error*, to store an error if it happens
*
*/
RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
uint32_t ram_flags, const char *mem_path,
- bool readonly, Error **errp);
+ off_t offset, bool readonly, Error **errp);
RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr,
uint32_t ram_flags, int fd, off_t offset,
bool readonly, Error **errp);
QLIST_ENTRY(RAMBlock) next;
QLIST_HEAD(, RAMBlockNotifier) ramblock_notifiers;
int fd;
+ uint64_t fd_offset;
size_t page_size;
/* dirty bitmap used during migration */
unsigned long *bmap;
#define ETH_ALEN 6
#define ETH_HLEN 14
#define ETH_ZLEN 60 /* Min. octets in frame without FCS */
+#define ETH_FCS_LEN 4
+#define ETH_MTU 1500
struct eth_header {
uint8_t h_dest[ETH_ALEN]; /* destination eth addr */
#define IP_HEADER_VERSION_6 (6)
#define IP_PROTO_TCP (6)
#define IP_PROTO_UDP (17)
+#define IP_PROTO_SCTP (132)
#define IPTOS_ECN_MASK 0x03
#define IPTOS_ECN(x) ((x) & IPTOS_ECN_MASK)
#define IPTOS_ECN_CE 0x03
}
static inline uint32_t
-eth_get_l2_hdr_length_iov(const struct iovec *iov, int iovcnt)
+eth_get_l2_hdr_length_iov(const struct iovec *iov, size_t iovcnt, size_t iovoff)
{
uint8_t p[sizeof(struct eth_header) + sizeof(struct vlan_header)];
- size_t copied = iov_to_buf(iov, iovcnt, 0, p, ARRAY_SIZE(p));
+ size_t copied = iov_to_buf(iov, iovcnt, iovoff, p, ARRAY_SIZE(p));
if (copied < ARRAY_SIZE(p)) {
return copied;
size_t
eth_strip_vlan(const struct iovec *iov, int iovcnt, size_t iovoff,
- uint8_t *new_ehdr_buf,
+ void *new_ehdr_buf,
uint16_t *payload_offset, uint16_t *tci);
size_t
-eth_strip_vlan_ex(const struct iovec *iov, int iovcnt, size_t iovoff,
- uint16_t vet, uint8_t *new_ehdr_buf,
+eth_strip_vlan_ex(const struct iovec *iov, int iovcnt, size_t iovoff, int index,
+ uint16_t vet, uint16_t vet_ext, void *new_ehdr_buf,
uint16_t *payload_offset, uint16_t *tci);
uint16_t
eth_get_l3_proto(const struct iovec *l2hdr_iov, int iovcnt, size_t l2hdr_len);
-void eth_setup_vlan_headers_ex(struct eth_header *ehdr, uint16_t vlan_tag,
- uint16_t vlan_ethtype, bool *is_new);
-
-static inline void
-eth_setup_vlan_headers(struct eth_header *ehdr, uint16_t vlan_tag,
- bool *is_new)
-{
- eth_setup_vlan_headers_ex(ehdr, vlan_tag, ETH_P_VLAN, is_new);
-}
+void eth_setup_vlan_headers(struct eth_header *ehdr, size_t *ehdr_size,
+ uint16_t vlan_tag, uint16_t vlan_ethtype);
uint8_t eth_get_gso_type(uint16_t l3_proto, uint8_t *l3_hdr, uint8_t l4proto);
typedef enum EthL4HdrProto {
ETH_L4_HDR_PROTO_INVALID,
ETH_L4_HDR_PROTO_TCP,
- ETH_L4_HDR_PROTO_UDP
+ ETH_L4_HDR_PROTO_UDP,
+ ETH_L4_HDR_PROTO_SCTP
} EthL4HdrProto;
typedef struct eth_l4_hdr_info_st {
bool has_tcp_data;
} eth_l4_hdr_info;
-void eth_get_protocols(const struct iovec *iov, int iovcnt,
+void eth_get_protocols(const struct iovec *iov, size_t iovcnt, size_t iovoff,
bool *hasip4, bool *hasip6,
size_t *l3hdr_off,
size_t *l4hdr_off,
#include "qemu/int128.h"
+/*
+ * If __alignof(unsigned __int128) < 16, GCC may refuse to inline atomics
+ * that are supported by the host, e.g. s390x. We can force the pointer to
+ * have our known alignment with __builtin_assume_aligned, however prior to
+ * GCC 13 that was only reliable with optimization enabled. See
+ * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=107389
+ */
+#if defined(CONFIG_ATOMIC128_OPT)
+# if !defined(__OPTIMIZE__)
+# define ATTRIBUTE_ATOMIC128_OPT __attribute__((optimize("O1")))
+# endif
+# define CONFIG_ATOMIC128
+#endif
+#ifndef ATTRIBUTE_ATOMIC128_OPT
+# define ATTRIBUTE_ATOMIC128_OPT
+#endif
+
/*
* GCC is a house divided about supporting large atomic operations.
*
* Therefore, special case each platform.
*/
-#if defined(CONFIG_ATOMIC128)
-static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
-{
- Int128Alias r, c, n;
-
- c.s = cmp;
- n.s = new;
- r.i = qatomic_cmpxchg__nocheck((__int128_t *)ptr, c.i, n.i);
- return r.s;
-}
-# define HAVE_CMPXCHG128 1
-#elif defined(CONFIG_CMPXCHG128)
-static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
-{
- Int128Alias r, c, n;
-
- c.s = cmp;
- n.s = new;
- r.i = __sync_val_compare_and_swap_16((__int128_t *)ptr, c.i, n.i);
- return r.s;
-}
-# define HAVE_CMPXCHG128 1
-#elif defined(__aarch64__)
-/* Through gcc 8, aarch64 has no support for 128-bit at all. */
-static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
-{
- uint64_t cmpl = int128_getlo(cmp), cmph = int128_gethi(cmp);
- uint64_t newl = int128_getlo(new), newh = int128_gethi(new);
- uint64_t oldl, oldh;
- uint32_t tmp;
-
- asm("0: ldaxp %[oldl], %[oldh], %[mem]\n\t"
- "cmp %[oldl], %[cmpl]\n\t"
- "ccmp %[oldh], %[cmph], #0, eq\n\t"
- "b.ne 1f\n\t"
- "stlxp %w[tmp], %[newl], %[newh], %[mem]\n\t"
- "cbnz %w[tmp], 0b\n"
- "1:"
- : [mem] "+m"(*ptr), [tmp] "=&r"(tmp),
- [oldl] "=&r"(oldl), [oldh] "=&r"(oldh)
- : [cmpl] "r"(cmpl), [cmph] "r"(cmph),
- [newl] "r"(newl), [newh] "r"(newh)
- : "memory", "cc");
-
- return int128_make128(oldl, oldh);
-}
-# define HAVE_CMPXCHG128 1
-#else
-/* Fallback definition that must be optimized away, or error. */
-Int128 QEMU_ERROR("unsupported atomic")
- atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new);
-# define HAVE_CMPXCHG128 0
-#endif /* Some definition for HAVE_CMPXCHG128 */
-
-
-#if defined(CONFIG_ATOMIC128)
-static inline Int128 atomic16_read(Int128 *ptr)
-{
- Int128Alias r;
-
- r.i = qatomic_read__nocheck((__int128_t *)ptr);
- return r.s;
-}
-
-static inline void atomic16_set(Int128 *ptr, Int128 val)
-{
- Int128Alias v;
-
- v.s = val;
- qatomic_set__nocheck((__int128_t *)ptr, v.i);
-}
-
-# define HAVE_ATOMIC128 1
-#elif !defined(CONFIG_USER_ONLY) && defined(__aarch64__)
-/* We can do better than cmpxchg for AArch64. */
-static inline Int128 atomic16_read(Int128 *ptr)
-{
- uint64_t l, h;
- uint32_t tmp;
-
- /* The load must be paired with the store to guarantee not tearing. */
- asm("0: ldxp %[l], %[h], %[mem]\n\t"
- "stxp %w[tmp], %[l], %[h], %[mem]\n\t"
- "cbnz %w[tmp], 0b"
- : [mem] "+m"(*ptr), [tmp] "=r"(tmp), [l] "=r"(l), [h] "=r"(h));
-
- return int128_make128(l, h);
-}
-
-static inline void atomic16_set(Int128 *ptr, Int128 val)
-{
- uint64_t l = int128_getlo(val), h = int128_gethi(val);
- uint64_t t1, t2;
-
- /* Load into temporaries to acquire the exclusive access lock. */
- asm("0: ldxp %[t1], %[t2], %[mem]\n\t"
- "stxp %w[t1], %[l], %[h], %[mem]\n\t"
- "cbnz %w[t1], 0b"
- : [mem] "+m"(*ptr), [t1] "=&r"(t1), [t2] "=&r"(t2)
- : [l] "r"(l), [h] "r"(h));
-}
-
-# define HAVE_ATOMIC128 1
-#elif !defined(CONFIG_USER_ONLY) && HAVE_CMPXCHG128
-static inline Int128 atomic16_read(Int128 *ptr)
-{
- /* Maybe replace 0 with 0, returning the old value. */
- Int128 z = int128_make64(0);
- return atomic16_cmpxchg(ptr, z, z);
-}
-
-static inline void atomic16_set(Int128 *ptr, Int128 val)
-{
- Int128 old = *ptr, cmp;
- do {
- cmp = old;
- old = atomic16_cmpxchg(ptr, cmp, val);
- } while (int128_ne(old, cmp));
-}
-
-# define HAVE_ATOMIC128 1
-#else
-/* Fallback definitions that must be optimized away, or error. */
-Int128 QEMU_ERROR("unsupported atomic") atomic16_read(Int128 *ptr);
-void QEMU_ERROR("unsupported atomic") atomic16_set(Int128 *ptr, Int128 val);
-# define HAVE_ATOMIC128 0
-#endif /* Some definition for HAVE_ATOMIC128 */
+#include "host/atomic128-cas.h"
+#include "host/atomic128-ldst.h"
#endif /* QEMU_ATOMIC128_H */
uint32_t crc32c(uint32_t crc, const uint8_t *data, unsigned int length);
+uint32_t iov_crc32c(uint32_t crc, const struct iovec *iov, size_t iov_cnt);
#endif
--- /dev/null
+/* SPDX-License-Identifier: MIT */
+/*
+ * Define tcg_debug_assert
+ * Copyright (c) 2008 Fabrice Bellard
+ */
+
+#ifndef TCG_DEBUG_ASSERT_H
+#define TCG_DEBUG_ASSERT_H
+
+#if defined CONFIG_DEBUG_TCG || defined QEMU_STATIC_ANALYSIS
+# define tcg_debug_assert(X) do { assert(X); } while (0)
+#else
+# define tcg_debug_assert(X) \
+ do { if (!(X)) { __builtin_unreachable(); } } while (0)
+#endif
+
+#endif
#include "tcg/tcg-mo.h"
#include "tcg-target.h"
#include "tcg/tcg-cond.h"
+#include "tcg/debug-assert.h"
/* XXX: make safe guess about sizes */
#define MAX_OP_PER_INSTR 266
/* The port better have done this. */
#endif
-
-#if defined CONFIG_DEBUG_TCG || defined QEMU_STATIC_ANALYSIS
-# define tcg_debug_assert(X) do { assert(X); } while (0)
-#else
-# define tcg_debug_assert(X) \
- do { if (!(X)) { __builtin_unreachable(); } } while (0)
-#endif
-
typedef struct TCGRelocation TCGRelocation;
struct TCGRelocation {
QSIMPLEQ_ENTRY(TCGRelocation) next;
'-iquote', meson.current_source_dir() / 'include',
language: all_languages)
+# If a host-specific include directory exists, list that first...
+host_include = meson.current_source_dir() / 'host/include/'
+if fs.is_dir(host_include / host_arch)
+ add_project_arguments('-iquote', host_include / host_arch,
+ language: all_languages)
+endif
+# ... followed by the generic fallback.
+add_project_arguments('-iquote', host_include / 'generic',
+ language: all_languages)
+
sparse = find_program('cgcc', required: get_option('sparse'))
if sparse.found()
run_target('sparse',
# __alignof(unsigned __int128) for the host.
atomic_test_128 = '''
int main(int ac, char **av) {
- unsigned __int128 *p = __builtin_assume_aligned(av[ac - 1], sizeof(16));
+ unsigned __int128 *p = __builtin_assume_aligned(av[ac - 1], 16);
p[1] = __atomic_load_n(&p[0], __ATOMIC_RELAXED);
__atomic_store_n(&p[2], p[3], __ATOMIC_RELAXED);
__atomic_compare_exchange_n(&p[4], &p[5], p[6], 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
'qemu-file.c',
'yank_functions.c',
)
-softmmu_ss.add(migration_files)
softmmu_ss.add(files(
'block-dirty-bitmap.c',
#define RAM_SAVE_FLAG_MULTIFD_FLUSH 0x200
/* We can't use any flag that is bigger than 0x200 */
-int (*xbzrle_encode_buffer_func)(uint8_t *, uint8_t *, int,
- uint8_t *, int) = xbzrle_encode_buffer;
-#if defined(CONFIG_AVX512BW_OPT)
-#include "qemu/cpuid.h"
-static void __attribute__((constructor)) init_cpu_flag(void)
-{
- unsigned max = __get_cpuid_max(0, NULL);
- int a, b, c, d;
- if (max >= 1) {
- __cpuid(1, a, b, c, d);
- /* We must check that AVX is not just available, but usable. */
- if ((c & bit_OSXSAVE) && (c & bit_AVX) && max >= 7) {
- int bv;
- __asm("xgetbv" : "=a"(bv), "=d"(d) : "c"(0));
- __cpuid_count(7, 0, a, b, c, d);
- /* 0xe6:
- * XCR0[7:5] = 111b (OPMASK state, upper 256-bit of ZMM0-ZMM15
- * and ZMM16-ZMM31 state are enabled by OS)
- * XCR0[2:1] = 11b (XMM state and YMM state are enabled by OS)
- */
- if ((bv & 0xe6) == 0xe6 && (b & bit_AVX512BW)) {
- xbzrle_encode_buffer_func = xbzrle_encode_buffer_avx512;
- }
- }
- }
-}
-#endif
-
XBZRLECacheStats xbzrle_counters;
/* used by the search for pages to send */
memcpy(XBZRLE.current_buf, *current_data, TARGET_PAGE_SIZE);
/* XBZRLE encoding (if there is no overflow) */
- encoded_len = xbzrle_encode_buffer_func(prev_cached_page, XBZRLE.current_buf,
- TARGET_PAGE_SIZE, XBZRLE.encoded_buf,
- TARGET_PAGE_SIZE);
+ encoded_len = xbzrle_encode_buffer(prev_cached_page, XBZRLE.current_buf,
+ TARGET_PAGE_SIZE, XBZRLE.encoded_buf,
+ TARGET_PAGE_SIZE);
/*
* Update the cache contents, so that it corresponds to the data
#include "qemu/host-utils.h"
#include "xbzrle.h"
+#if defined(CONFIG_AVX512BW_OPT)
+#include <immintrin.h>
+#include "host/cpuinfo.h"
+
+static int __attribute__((target("avx512bw")))
+xbzrle_encode_buffer_avx512(uint8_t *old_buf, uint8_t *new_buf, int slen,
+ uint8_t *dst, int dlen)
+{
+ uint32_t zrun_len = 0, nzrun_len = 0;
+ int d = 0, i = 0, num = 0;
+ uint8_t *nzrun_start = NULL;
+ /* add 1 to include residual part in main loop */
+ uint32_t count512s = (slen >> 6) + 1;
+ /* countResidual is tail of data, i.e., countResidual = slen % 64 */
+ uint32_t count_residual = slen & 0b111111;
+ bool never_same = true;
+ uint64_t mask_residual = 1;
+ mask_residual <<= count_residual;
+ mask_residual -= 1;
+ __m512i r = _mm512_set1_epi32(0);
+
+ while (count512s) {
+ int bytes_to_check = 64;
+ uint64_t mask = 0xffffffffffffffff;
+ if (count512s == 1) {
+ bytes_to_check = count_residual;
+ mask = mask_residual;
+ }
+ __m512i old_data = _mm512_mask_loadu_epi8(r,
+ mask, old_buf + i);
+ __m512i new_data = _mm512_mask_loadu_epi8(r,
+ mask, new_buf + i);
+ uint64_t comp = _mm512_cmpeq_epi8_mask(old_data, new_data);
+ count512s--;
+
+ bool is_same = (comp & 0x1);
+ while (bytes_to_check) {
+ if (d + 2 > dlen) {
+ return -1;
+ }
+ if (is_same) {
+ if (nzrun_len) {
+ d += uleb128_encode_small(dst + d, nzrun_len);
+ if (d + nzrun_len > dlen) {
+ return -1;
+ }
+ nzrun_start = new_buf + i - nzrun_len;
+ memcpy(dst + d, nzrun_start, nzrun_len);
+ d += nzrun_len;
+ nzrun_len = 0;
+ }
+ /* 64 data at a time for speed */
+ if (count512s && (comp == 0xffffffffffffffff)) {
+ i += 64;
+ zrun_len += 64;
+ break;
+ }
+ never_same = false;
+ num = ctz64(~comp);
+ num = (num < bytes_to_check) ? num : bytes_to_check;
+ zrun_len += num;
+ bytes_to_check -= num;
+ comp >>= num;
+ i += num;
+ if (bytes_to_check) {
+ /* still has different data after same data */
+ d += uleb128_encode_small(dst + d, zrun_len);
+ zrun_len = 0;
+ } else {
+ break;
+ }
+ }
+ if (never_same || zrun_len) {
+ /*
+ * never_same only acts if
+ * data begins with diff in first count512s
+ */
+ d += uleb128_encode_small(dst + d, zrun_len);
+ zrun_len = 0;
+ never_same = false;
+ }
+ /* has diff, 64 data at a time for speed */
+ if ((bytes_to_check == 64) && (comp == 0x0)) {
+ i += 64;
+ nzrun_len += 64;
+ break;
+ }
+ num = ctz64(comp);
+ num = (num < bytes_to_check) ? num : bytes_to_check;
+ nzrun_len += num;
+ bytes_to_check -= num;
+ comp >>= num;
+ i += num;
+ if (bytes_to_check) {
+ /* mask like 111000 */
+ d += uleb128_encode_small(dst + d, nzrun_len);
+ /* overflow */
+ if (d + nzrun_len > dlen) {
+ return -1;
+ }
+ nzrun_start = new_buf + i - nzrun_len;
+ memcpy(dst + d, nzrun_start, nzrun_len);
+ d += nzrun_len;
+ nzrun_len = 0;
+ is_same = true;
+ }
+ }
+ }
+
+ if (nzrun_len != 0) {
+ d += uleb128_encode_small(dst + d, nzrun_len);
+ /* overflow */
+ if (d + nzrun_len > dlen) {
+ return -1;
+ }
+ nzrun_start = new_buf + i - nzrun_len;
+ memcpy(dst + d, nzrun_start, nzrun_len);
+ d += nzrun_len;
+ }
+ return d;
+}
+
+static int xbzrle_encode_buffer_int(uint8_t *old_buf, uint8_t *new_buf,
+ int slen, uint8_t *dst, int dlen);
+
+static int (*accel_func)(uint8_t *, uint8_t *, int, uint8_t *, int);
+
+static void __attribute__((constructor)) init_accel(void)
+{
+ unsigned info = cpuinfo_init();
+ if (info & CPUINFO_AVX512BW) {
+ accel_func = xbzrle_encode_buffer_avx512;
+ } else {
+ accel_func = xbzrle_encode_buffer_int;
+ }
+}
+
+int xbzrle_encode_buffer(uint8_t *old_buf, uint8_t *new_buf, int slen,
+ uint8_t *dst, int dlen)
+{
+ return accel_func(old_buf, new_buf, slen, dst, dlen);
+}
+
+#define xbzrle_encode_buffer xbzrle_encode_buffer_int
+#endif
+
/*
page = zrun nzrun
| zrun nzrun page
return d;
}
-
-#if defined(CONFIG_AVX512BW_OPT)
-#include <immintrin.h>
-
-int __attribute__((target("avx512bw")))
-xbzrle_encode_buffer_avx512(uint8_t *old_buf, uint8_t *new_buf, int slen,
- uint8_t *dst, int dlen)
-{
- uint32_t zrun_len = 0, nzrun_len = 0;
- int d = 0, i = 0, num = 0;
- uint8_t *nzrun_start = NULL;
- /* add 1 to include residual part in main loop */
- uint32_t count512s = (slen >> 6) + 1;
- /* countResidual is tail of data, i.e., countResidual = slen % 64 */
- uint32_t count_residual = slen & 0b111111;
- bool never_same = true;
- uint64_t mask_residual = 1;
- mask_residual <<= count_residual;
- mask_residual -= 1;
- __m512i r = _mm512_set1_epi32(0);
-
- while (count512s) {
- int bytes_to_check = 64;
- uint64_t mask = 0xffffffffffffffff;
- if (count512s == 1) {
- bytes_to_check = count_residual;
- mask = mask_residual;
- }
- __m512i old_data = _mm512_mask_loadu_epi8(r,
- mask, old_buf + i);
- __m512i new_data = _mm512_mask_loadu_epi8(r,
- mask, new_buf + i);
- uint64_t comp = _mm512_cmpeq_epi8_mask(old_data, new_data);
- count512s--;
-
- bool is_same = (comp & 0x1);
- while (bytes_to_check) {
- if (d + 2 > dlen) {
- return -1;
- }
- if (is_same) {
- if (nzrun_len) {
- d += uleb128_encode_small(dst + d, nzrun_len);
- if (d + nzrun_len > dlen) {
- return -1;
- }
- nzrun_start = new_buf + i - nzrun_len;
- memcpy(dst + d, nzrun_start, nzrun_len);
- d += nzrun_len;
- nzrun_len = 0;
- }
- /* 64 data at a time for speed */
- if (count512s && (comp == 0xffffffffffffffff)) {
- i += 64;
- zrun_len += 64;
- break;
- }
- never_same = false;
- num = ctz64(~comp);
- num = (num < bytes_to_check) ? num : bytes_to_check;
- zrun_len += num;
- bytes_to_check -= num;
- comp >>= num;
- i += num;
- if (bytes_to_check) {
- /* still has different data after same data */
- d += uleb128_encode_small(dst + d, zrun_len);
- zrun_len = 0;
- } else {
- break;
- }
- }
- if (never_same || zrun_len) {
- /*
- * never_same only acts if
- * data begins with diff in first count512s
- */
- d += uleb128_encode_small(dst + d, zrun_len);
- zrun_len = 0;
- never_same = false;
- }
- /* has diff, 64 data at a time for speed */
- if ((bytes_to_check == 64) && (comp == 0x0)) {
- i += 64;
- nzrun_len += 64;
- break;
- }
- num = ctz64(comp);
- num = (num < bytes_to_check) ? num : bytes_to_check;
- nzrun_len += num;
- bytes_to_check -= num;
- comp >>= num;
- i += num;
- if (bytes_to_check) {
- /* mask like 111000 */
- d += uleb128_encode_small(dst + d, nzrun_len);
- /* overflow */
- if (d + nzrun_len > dlen) {
- return -1;
- }
- nzrun_start = new_buf + i - nzrun_len;
- memcpy(dst + d, nzrun_start, nzrun_len);
- d += nzrun_len;
- nzrun_len = 0;
- is_same = true;
- }
- }
- }
-
- if (nzrun_len != 0) {
- d += uleb128_encode_small(dst + d, nzrun_len);
- /* overflow */
- if (d + nzrun_len > dlen) {
- return -1;
- }
- nzrun_start = new_buf + i - nzrun_len;
- memcpy(dst + d, nzrun_start, nzrun_len);
- d += nzrun_len;
- }
- return d;
-}
-#endif
uint8_t *dst, int dlen);
int xbzrle_decode_buffer(uint8_t *src, int slen, uint8_t *dst, int dlen);
-#if defined(CONFIG_AVX512BW_OPT)
-int xbzrle_encode_buffer_avx512(uint8_t *old_buf, uint8_t *new_buf, int slen,
- uint8_t *dst, int dlen);
-#endif
+
#endif
#include "net/checksum.h"
#include "net/tap.h"
-void eth_setup_vlan_headers_ex(struct eth_header *ehdr, uint16_t vlan_tag,
- uint16_t vlan_ethtype, bool *is_new)
+void eth_setup_vlan_headers(struct eth_header *ehdr, size_t *ehdr_size,
+ uint16_t vlan_tag, uint16_t vlan_ethtype)
{
struct vlan_header *vhdr = PKT_GET_VLAN_HDR(ehdr);
- switch (be16_to_cpu(ehdr->h_proto)) {
- case ETH_P_VLAN:
- case ETH_P_DVLAN:
- /* vlan hdr exists */
- *is_new = false;
- break;
-
- default:
- /* No VLAN header, put a new one */
- vhdr->h_proto = ehdr->h_proto;
- ehdr->h_proto = cpu_to_be16(vlan_ethtype);
- *is_new = true;
- break;
- }
+ memmove(vhdr + 1, vhdr, *ehdr_size - ETH_HLEN);
vhdr->h_tci = cpu_to_be16(vlan_tag);
+ vhdr->h_proto = ehdr->h_proto;
+ ehdr->h_proto = cpu_to_be16(vlan_ethtype);
+ *ehdr_size += sizeof(*vhdr);
}
uint8_t
return l4len > TCP_HEADER_DATA_OFFSET(tcp);
}
-void eth_get_protocols(const struct iovec *iov, int iovcnt,
+void eth_get_protocols(const struct iovec *iov, size_t iovcnt, size_t iovoff,
bool *hasip4, bool *hasip6,
size_t *l3hdr_off,
size_t *l4hdr_off,
{
int proto;
bool fragment = false;
- size_t l2hdr_len = eth_get_l2_hdr_length_iov(iov, iovcnt);
size_t input_size = iov_size(iov, iovcnt);
size_t copied;
uint8_t ip_p;
*hasip4 = *hasip6 = false;
+ *l3hdr_off = iovoff + eth_get_l2_hdr_length_iov(iov, iovcnt, iovoff);
l4hdr_info->proto = ETH_L4_HDR_PROTO_INVALID;
- proto = eth_get_l3_proto(iov, iovcnt, l2hdr_len);
-
- *l3hdr_off = l2hdr_len;
+ proto = eth_get_l3_proto(iov, iovcnt, *l3hdr_off);
if (proto == ETH_P_IP) {
struct ip_header *iphdr = &ip4hdr_info->ip4_hdr;
- if (input_size < l2hdr_len) {
+ if (input_size < *l3hdr_off) {
return;
}
- copied = iov_to_buf(iov, iovcnt, l2hdr_len, iphdr, sizeof(*iphdr));
+ copied = iov_to_buf(iov, iovcnt, *l3hdr_off, iphdr, sizeof(*iphdr));
if (copied < sizeof(*iphdr) ||
IP_HEADER_VERSION(iphdr) != IP_HEADER_VERSION_4) {
return;
*hasip4 = true;
ip_p = iphdr->ip_p;
ip4hdr_info->fragment = IP4_IS_FRAGMENT(iphdr);
- *l4hdr_off = l2hdr_len + IP_HDR_GET_LEN(iphdr);
+ *l4hdr_off = *l3hdr_off + IP_HDR_GET_LEN(iphdr);
fragment = ip4hdr_info->fragment;
} else if (proto == ETH_P_IPV6) {
- if (!eth_parse_ipv6_hdr(iov, iovcnt, l2hdr_len, ip6hdr_info)) {
+ if (!eth_parse_ipv6_hdr(iov, iovcnt, *l3hdr_off, ip6hdr_info)) {
return;
}
*hasip6 = true;
ip_p = ip6hdr_info->l4proto;
- *l4hdr_off = l2hdr_len + ip6hdr_info->full_hdr_len;
+ *l4hdr_off = *l3hdr_off + ip6hdr_info->full_hdr_len;
fragment = ip6hdr_info->fragment;
} else {
return;
*l5hdr_off = *l4hdr_off + sizeof(l4hdr_info->hdr.udp);
}
break;
+
+ case IP_PROTO_SCTP:
+ l4hdr_info->proto = ETH_L4_HDR_PROTO_SCTP;
+ break;
}
}
size_t
eth_strip_vlan(const struct iovec *iov, int iovcnt, size_t iovoff,
- uint8_t *new_ehdr_buf,
+ void *new_ehdr_buf,
uint16_t *payload_offset, uint16_t *tci)
{
struct vlan_header vlan_hdr;
- struct eth_header *new_ehdr = (struct eth_header *) new_ehdr_buf;
+ struct eth_header *new_ehdr = new_ehdr_buf;
size_t copied = iov_to_buf(iov, iovcnt, iovoff,
new_ehdr, sizeof(*new_ehdr));
}
size_t
-eth_strip_vlan_ex(const struct iovec *iov, int iovcnt, size_t iovoff,
- uint16_t vet, uint8_t *new_ehdr_buf,
+eth_strip_vlan_ex(const struct iovec *iov, int iovcnt, size_t iovoff, int index,
+ uint16_t vet, uint16_t vet_ext, void *new_ehdr_buf,
uint16_t *payload_offset, uint16_t *tci)
{
struct vlan_header vlan_hdr;
- struct eth_header *new_ehdr = (struct eth_header *) new_ehdr_buf;
-
- size_t copied = iov_to_buf(iov, iovcnt, iovoff,
- new_ehdr, sizeof(*new_ehdr));
-
- if (copied < sizeof(*new_ehdr)) {
- return 0;
- }
+ uint16_t *new_ehdr_proto;
+ size_t new_ehdr_size;
+ size_t copied;
- if (be16_to_cpu(new_ehdr->h_proto) == vet) {
- copied = iov_to_buf(iov, iovcnt, iovoff + sizeof(*new_ehdr),
- &vlan_hdr, sizeof(vlan_hdr));
+ switch (index) {
+ case 0:
+ new_ehdr_proto = &PKT_GET_ETH_HDR(new_ehdr_buf)->h_proto;
+ new_ehdr_size = sizeof(struct eth_header);
+ copied = iov_to_buf(iov, iovcnt, iovoff, new_ehdr_buf, new_ehdr_size);
+ break;
- if (copied < sizeof(vlan_hdr)) {
+ case 1:
+ new_ehdr_proto = &PKT_GET_VLAN_HDR(new_ehdr_buf)->h_proto;
+ new_ehdr_size = sizeof(struct eth_header) + sizeof(struct vlan_header);
+ copied = iov_to_buf(iov, iovcnt, iovoff, new_ehdr_buf, new_ehdr_size);
+ if (be16_to_cpu(PKT_GET_ETH_HDR(new_ehdr_buf)->h_proto) != vet_ext) {
return 0;
}
+ break;
- new_ehdr->h_proto = vlan_hdr.h_proto;
+ default:
+ return 0;
+ }
- *tci = be16_to_cpu(vlan_hdr.h_tci);
- *payload_offset = iovoff + sizeof(*new_ehdr) + sizeof(vlan_hdr);
- return sizeof(struct eth_header);
+ if (copied < new_ehdr_size || be16_to_cpu(*new_ehdr_proto) != vet) {
+ return 0;
}
- return 0;
+ copied = iov_to_buf(iov, iovcnt, iovoff + new_ehdr_size,
+ &vlan_hdr, sizeof(vlan_hdr));
+ if (copied < sizeof(vlan_hdr)) {
+ return 0;
+ }
+
+ *new_ehdr_proto = vlan_hdr.h_proto;
+ *payload_offset = iovoff + new_ehdr_size + sizeof(vlan_hdr);
+ *tci = be16_to_cpu(vlan_hdr.h_tci);
+
+ return new_ehdr_size;
}
void
# selects a default alignment (currently the page size).
# (default: 0)
#
+# @offset: the offset into the target file that the region starts at. You
+# can use this option to back multiple regions with a single file. Must
+# be a multiple of the page size. (default: 0) (since 8.1)
+#
# @discard-data: if true, the file contents can be destroyed when QEMU
# exits, to avoid unnecessarily flushing data to the backing file.
# Note that @discard-data is only an optimization, and QEMU might
{ 'struct': 'MemoryBackendFileProperties',
'base': 'MemoryBackendProperties',
'data': { '*align': 'size',
+ '*offset': 'size',
'*discard-data': 'bool',
'mem-path': 'str',
'*pmem': { 'type': 'bool', 'if': 'CONFIG_LIBPMEM' },
they are specified. Note that the 'id' property must be set. These
objects are placed in the '/objects' path.
- ``-object memory-backend-file,id=id,size=size,mem-path=dir,share=on|off,discard-data=on|off,merge=on|off,dump=on|off,prealloc=on|off,host-nodes=host-nodes,policy=default|preferred|bind|interleave,align=align,readonly=on|off``
+ ``-object memory-backend-file,id=id,size=size,mem-path=dir,share=on|off,discard-data=on|off,merge=on|off,dump=on|off,prealloc=on|off,host-nodes=host-nodes,policy=default|preferred|bind|interleave,align=align,offset=offset,readonly=on|off``
Creates a memory file backend object, which can be used to back
the guest RAM with huge pages.
such cases, users can specify the required alignment via this
option.
+ The ``offset`` option specifies the offset into the target file
+ that the region starts at. You can use this parameter to back
+ multiple regions with a single file.
+
The ``pmem`` option specifies whether the backing file specified
by ``mem-path`` is in host persistent memory that can be
accessed using the SNIA NVM programming model (e.g. Intel
--- /dev/null
+gitlabUrl: "https://gitlab.com/"
+runnerRegistrationToken: ""
+rbac:
+ create: true
+concurrent: 200
+runners:
+ privileged: true
+ config: |
+ [[runners]]
+ limit = 100
+ environment = [
+ "DOCKER_HOST=tcp://docker:2376",
+ "DOCKER_TLS_CERTDIR=/certs",
+ "DOCKER_TLS_VERIFY=1",
+ "DOCKER_CERT_PATH=/certs/client"
+ ]
+ [runners.kubernetes]
+ poll_timeout = 1200
+ image = "ubuntu:20.04"
+ cpu_request = "0.5"
+ service_cpu_request = "0.5"
+ helper_cpu_request = "0.25"
+ cpu_request_overwrite_max_allowed = "7"
+ memory_request_overwrite_max_allowed = "30Gi"
+ [[runners.kubernetes.volumes.empty_dir]]
+ name = "docker-certs"
+ mount_path = "/certs/client"
+ medium = "Memory"
+ [runners.kubernetes.node_selector]
+ agentpool = "jobs"
tests/avocado/cpu_queries.py:QueryCPUModelExpansion.test \
tests/avocado/empty_cpu_model.py:EmptyCPUModel.test \
tests/avocado/hotplug_cpu.py:HotPlugCPU.test \
- tests/avocado/igb.py:IGB.test \
+ tests/avocado/netdev-ethtool.py:NetDevEthtool.test_igb \
+ tests/avocado/netdev-ethtool.py:NetDevEthtool.test_igb_nomsi \
tests/avocado/info_usernet.py:InfoUsernet.test_hostfwd \
tests/avocado/intel_iommu.py:IntelIOMMU.test_intel_iommu \
tests/avocado/intel_iommu.py:IntelIOMMU.test_intel_iommu_pt \
uint64_t align,
uint32_t ram_flags,
const char *path,
+ ram_addr_t offset,
bool readonly,
Error **errp)
{
mr->destructor = memory_region_destructor_ram;
mr->align = align;
mr->ram_block = qemu_ram_alloc_from_file(size, mr, ram_flags, path,
- readonly, &err);
+ offset, readonly, &err);
if (err) {
mr->size = int128_zero();
object_unparent(OBJECT(mr));
error_setg(errp, "alignment 0x%" PRIx64
" must be a power of two", block->mr->align);
return NULL;
+ } else if (offset % block->page_size) {
+ error_setg(errp, "offset 0x%" PRIx64
+ " must be multiples of page size 0x%zx",
+ offset, block->page_size);
+ return NULL;
}
block->mr->align = MAX(block->page_size, block->mr->align);
#if defined(__s390x__)
* those labels. Therefore, extending the non-empty backend file
* is disabled as well.
*/
- if (truncate && ftruncate(fd, memory)) {
+ if (truncate && ftruncate(fd, offset + memory)) {
perror("ftruncate");
}
}
block->fd = fd;
+ block->fd_offset = offset;
return area;
}
#endif
size = HOST_PAGE_ALIGN(size);
file_size = get_file_size(fd);
- if (file_size > 0 && file_size < size) {
+ if (file_size > offset && file_size < (offset + size)) {
error_setg(errp, "backing store size 0x%" PRIx64
" does not match 'size' option 0x" RAM_ADDR_FMT,
file_size, size);
RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
uint32_t ram_flags, const char *mem_path,
- bool readonly, Error **errp)
+ off_t offset, bool readonly, Error **errp)
{
int fd;
bool created;
return NULL;
}
- block = qemu_ram_alloc_from_fd(size, mr, ram_flags, fd, 0, readonly, errp);
+ block = qemu_ram_alloc_from_fd(size, mr, ram_flags, fd, offset, readonly,
+ errp);
if (!block) {
if (created) {
unlink(mem_path);
flags |= block->flags & RAM_NORESERVE ? MAP_NORESERVE : 0;
if (block->fd >= 0) {
area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
- flags, block->fd, offset);
+ flags, block->fd, offset + block->fd_offset);
} else {
flags |= MAP_ANONYMOUS;
area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
*/
mmu_idx = arm_v7m_mmu_idx_for_secstate(env, true);
oi = make_memop_idx(MO_LEUL, arm_to_core_mmu_idx(mmu_idx));
- newpc = cpu_ldl_le_mmu(env, frameptr, oi, 0);
- newpsr = cpu_ldl_le_mmu(env, frameptr + 4, oi, 0);
+ newpc = cpu_ldl_mmu(env, frameptr, oi, 0);
+ newpsr = cpu_ldl_mmu(env, frameptr + 4, oi, 0);
/* Consistency checks on new IPSR */
newpsr_exc = newpsr & XPSR_EXCP;
/* used to speed-up TLB assist handlers */
target_ulong nip; /* next instruction pointer */
- uint64_t retxh; /* high part of 128-bit helper return */
/* when a memory exception occurs, the access type is stored here */
int access_type;
DEF_HELPER_1(tbegin, void, env)
DEF_HELPER_FLAGS_1(fixup_thrm, TCG_CALL_NO_RWG, void, env)
-
-#ifdef TARGET_PPC64
-DEF_HELPER_FLAGS_3(lq_le_parallel, TCG_CALL_NO_WG, i64, env, tl, i32)
-DEF_HELPER_FLAGS_3(lq_be_parallel, TCG_CALL_NO_WG, i64, env, tl, i32)
-DEF_HELPER_FLAGS_5(stq_le_parallel, TCG_CALL_NO_WG,
- void, env, tl, i64, i64, i32)
-DEF_HELPER_FLAGS_5(stq_be_parallel, TCG_CALL_NO_WG,
- void, env, tl, i64, i64, i32)
-#endif
return i;
}
-#ifdef TARGET_PPC64
-uint64_t helper_lq_le_parallel(CPUPPCState *env, target_ulong addr,
- uint32_t opidx)
-{
- Int128 ret;
-
- /* We will have raised EXCP_ATOMIC from the translator. */
- assert(HAVE_ATOMIC128);
- ret = cpu_atomic_ldo_le_mmu(env, addr, opidx, GETPC());
- env->retxh = int128_gethi(ret);
- return int128_getlo(ret);
-}
-
-uint64_t helper_lq_be_parallel(CPUPPCState *env, target_ulong addr,
- uint32_t opidx)
-{
- Int128 ret;
-
- /* We will have raised EXCP_ATOMIC from the translator. */
- assert(HAVE_ATOMIC128);
- ret = cpu_atomic_ldo_be_mmu(env, addr, opidx, GETPC());
- env->retxh = int128_gethi(ret);
- return int128_getlo(ret);
-}
-
-void helper_stq_le_parallel(CPUPPCState *env, target_ulong addr,
- uint64_t lo, uint64_t hi, uint32_t opidx)
-{
- Int128 val;
-
- /* We will have raised EXCP_ATOMIC from the translator. */
- assert(HAVE_ATOMIC128);
- val = int128_make128(lo, hi);
- cpu_atomic_sto_le_mmu(env, addr, val, opidx, GETPC());
-}
-
-void helper_stq_be_parallel(CPUPPCState *env, target_ulong addr,
- uint64_t lo, uint64_t hi, uint32_t opidx)
-{
- Int128 val;
-
- /* We will have raised EXCP_ATOMIC from the translator. */
- assert(HAVE_ATOMIC128);
- val = int128_make128(lo, hi);
- cpu_atomic_sto_be_mmu(env, addr, val, opidx, GETPC());
-}
-#endif
-
/*****************************************************************************/
/* Altivec extension helpers */
#if HOST_BIG_ENDIAN
{
int rd = rD(ctx->opcode);
TCGv EA, hi, lo;
+ TCGv_i128 t16;
if (unlikely((rd & 1) || (rd == rA(ctx->opcode)) ||
(rd == rB(ctx->opcode)))) {
lo = cpu_gpr[rd + 1];
hi = cpu_gpr[rd];
- if (tb_cflags(ctx->base.tb) & CF_PARALLEL) {
- if (HAVE_ATOMIC128) {
- TCGv_i32 oi = tcg_temp_new_i32();
- if (ctx->le_mode) {
- tcg_gen_movi_i32(oi, make_memop_idx(MO_LE | MO_128 | MO_ALIGN,
- ctx->mem_idx));
- gen_helper_lq_le_parallel(lo, cpu_env, EA, oi);
- } else {
- tcg_gen_movi_i32(oi, make_memop_idx(MO_BE | MO_128 | MO_ALIGN,
- ctx->mem_idx));
- gen_helper_lq_be_parallel(lo, cpu_env, EA, oi);
- }
- tcg_gen_ld_i64(hi, cpu_env, offsetof(CPUPPCState, retxh));
- } else {
- /* Restart with exclusive lock. */
- gen_helper_exit_atomic(cpu_env);
- ctx->base.is_jmp = DISAS_NORETURN;
- return;
- }
- } else if (ctx->le_mode) {
- tcg_gen_qemu_ld_i64(lo, EA, ctx->mem_idx, MO_LEUQ | MO_ALIGN_16);
- tcg_gen_mov_tl(cpu_reserve, EA);
- gen_addr_add(ctx, EA, EA, 8);
- tcg_gen_qemu_ld_i64(hi, EA, ctx->mem_idx, MO_LEUQ);
- } else {
- tcg_gen_qemu_ld_i64(hi, EA, ctx->mem_idx, MO_BEUQ | MO_ALIGN_16);
- tcg_gen_mov_tl(cpu_reserve, EA);
- gen_addr_add(ctx, EA, EA, 8);
- tcg_gen_qemu_ld_i64(lo, EA, ctx->mem_idx, MO_BEUQ);
- }
+ t16 = tcg_temp_new_i128();
+ tcg_gen_qemu_ld_i128(t16, EA, ctx->mem_idx, DEF_MEMOP(MO_128 | MO_ALIGN));
+ tcg_gen_extr_i128_i64(lo, hi, t16);
tcg_gen_st_tl(hi, cpu_env, offsetof(CPUPPCState, reserve_val));
tcg_gen_st_tl(lo, cpu_env, offsetof(CPUPPCState, reserve_val2));
#if defined(TARGET_PPC64)
TCGv ea;
TCGv_i64 low_addr_gpr, high_addr_gpr;
- MemOp mop;
+ TCGv_i128 t16;
REQUIRE_INSNS_FLAGS(ctx, 64BX);
low_addr_gpr = cpu_gpr[a->rt + 1];
high_addr_gpr = cpu_gpr[a->rt];
}
+ t16 = tcg_temp_new_i128();
- if (tb_cflags(ctx->base.tb) & CF_PARALLEL) {
- if (HAVE_ATOMIC128) {
- mop = DEF_MEMOP(MO_128);
- TCGv_i32 oi = tcg_constant_i32(make_memop_idx(mop, ctx->mem_idx));
- if (store) {
- if (ctx->le_mode) {
- gen_helper_stq_le_parallel(cpu_env, ea, low_addr_gpr,
- high_addr_gpr, oi);
- } else {
- gen_helper_stq_be_parallel(cpu_env, ea, high_addr_gpr,
- low_addr_gpr, oi);
-
- }
- } else {
- if (ctx->le_mode) {
- gen_helper_lq_le_parallel(low_addr_gpr, cpu_env, ea, oi);
- tcg_gen_ld_i64(high_addr_gpr, cpu_env,
- offsetof(CPUPPCState, retxh));
- } else {
- gen_helper_lq_be_parallel(high_addr_gpr, cpu_env, ea, oi);
- tcg_gen_ld_i64(low_addr_gpr, cpu_env,
- offsetof(CPUPPCState, retxh));
- }
- }
- } else {
- /* Restart with exclusive lock. */
- gen_helper_exit_atomic(cpu_env);
- ctx->base.is_jmp = DISAS_NORETURN;
- }
+ if (store) {
+ tcg_gen_concat_i64_i128(t16, low_addr_gpr, high_addr_gpr);
+ tcg_gen_qemu_st_i128(t16, ea, ctx->mem_idx, DEF_MEMOP(MO_128));
} else {
- mop = DEF_MEMOP(MO_UQ);
- if (store) {
- tcg_gen_qemu_st_i64(low_addr_gpr, ea, ctx->mem_idx, mop);
- } else {
- tcg_gen_qemu_ld_i64(low_addr_gpr, ea, ctx->mem_idx, mop);
- }
-
- gen_addr_add(ctx, ea, ea, 8);
-
- if (store) {
- tcg_gen_qemu_st_i64(high_addr_gpr, ea, ctx->mem_idx, mop);
- } else {
- tcg_gen_qemu_ld_i64(high_addr_gpr, ea, ctx->mem_idx, mop);
- }
+ tcg_gen_qemu_ld_i128(t16, ea, ctx->mem_idx, DEF_MEMOP(MO_128));
+ tcg_gen_extr_i128_i64(low_addr_gpr, high_addr_gpr, t16);
}
#else
qemu_build_not_reached();
float_status fpu_status; /* passed to softfloat lib */
- /* The low part of a 128-bit return, or remainder of a divide. */
- uint64_t retxl;
-
PSW psw;
S390CrashReason crash_reason;
DEF_HELPER_FLAGS_2(srnm, TCG_CALL_NO_WG, void, env, i64)
DEF_HELPER_FLAGS_1(popcnt, TCG_CALL_NO_RWG_SE, i64, i64)
DEF_HELPER_2(stfle, i32, env, i64)
-DEF_HELPER_FLAGS_2(lpq, TCG_CALL_NO_WG, i64, env, i64)
-DEF_HELPER_FLAGS_2(lpq_parallel, TCG_CALL_NO_WG, i64, env, i64)
-DEF_HELPER_FLAGS_4(stpq, TCG_CALL_NO_WG, void, env, i64, i64, i64)
-DEF_HELPER_FLAGS_4(stpq_parallel, TCG_CALL_NO_WG, void, env, i64, i64, i64)
DEF_HELPER_4(mvcos, i32, env, i64, i64, i64)
DEF_HELPER_4(cu12, i32, env, i32, i32, i32)
DEF_HELPER_4(cu14, i32, env, i32, i32, i32)
D(0xc804, LPD, SSF, ILA, 0, 0, new_P, r3_P32, lpd, 0, MO_TEUL)
D(0xc805, LPDG, SSF, ILA, 0, 0, new_P, r3_P64, lpd, 0, MO_TEUQ)
/* LOAD PAIR FROM QUADWORD */
- C(0xe38f, LPQ, RXY_a, Z, 0, a2, r1_P, 0, lpq, 0)
+ C(0xe38f, LPQ, RXY_a, Z, 0, a2, 0, r1_D64, lpq, 0)
/* LOAD POSITIVE */
C(0x1000, LPR, RR_a, Z, 0, r2_32s, new, r1_32, abs, abs32)
C(0xb900, LPGR, RRE, Z, 0, r2, r1, 0, abs, abs64)
uint64_t a2, bool parallel)
{
uint32_t mem_idx = cpu_mmu_index(env, false);
+ MemOpIdx oi16 = make_memop_idx(MO_TE | MO_128, mem_idx);
+ MemOpIdx oi8 = make_memop_idx(MO_TE | MO_64, mem_idx);
+ MemOpIdx oi4 = make_memop_idx(MO_TE | MO_32, mem_idx);
+ MemOpIdx oi2 = make_memop_idx(MO_TE | MO_16, mem_idx);
+ MemOpIdx oi1 = make_memop_idx(MO_8, mem_idx);
uintptr_t ra = GETPC();
uint32_t fc = extract32(env->regs[0], 0, 8);
uint32_t sc = extract32(env->regs[0], 8, 8);
max = 3;
#endif
if ((HAVE_CMPXCHG128 ? 0 : fc + 2 > max) ||
- (HAVE_ATOMIC128 ? 0 : sc > max)) {
+ (HAVE_ATOMIC128_RW ? 0 : sc > max)) {
cpu_loop_exit_atomic(env_cpu(env), ra);
}
}
- /* All loads happen before all stores. For simplicity, load the entire
- store value area from the parameter list. */
- svh = cpu_ldq_data_ra(env, pl + 16, ra);
- svl = cpu_ldq_data_ra(env, pl + 24, ra);
+ /*
+ * All loads happen before all stores. For simplicity, load the entire
+ * store value area from the parameter list.
+ */
+ svh = cpu_ldq_mmu(env, pl + 16, oi8, ra);
+ svl = cpu_ldq_mmu(env, pl + 24, oi8, ra);
switch (fc) {
case 0:
{
- uint32_t nv = cpu_ldl_data_ra(env, pl, ra);
+ uint32_t nv = cpu_ldl_mmu(env, pl, oi4, ra);
uint32_t cv = env->regs[r3];
uint32_t ov;
if (parallel) {
-#ifdef CONFIG_USER_ONLY
- uint32_t *haddr = g2h(env_cpu(env), a1);
- ov = qatomic_cmpxchg__nocheck(haddr, cv, nv);
-#else
- MemOpIdx oi = make_memop_idx(MO_TEUL | MO_ALIGN, mem_idx);
- ov = cpu_atomic_cmpxchgl_be_mmu(env, a1, cv, nv, oi, ra);
-#endif
+ ov = cpu_atomic_cmpxchgl_be_mmu(env, a1, cv, nv, oi4, ra);
} else {
- ov = cpu_ldl_data_ra(env, a1, ra);
- cpu_stl_data_ra(env, a1, (ov == cv ? nv : ov), ra);
+ ov = cpu_ldl_mmu(env, a1, oi4, ra);
+ cpu_stl_mmu(env, a1, (ov == cv ? nv : ov), oi4, ra);
}
cc = (ov != cv);
env->regs[r3] = deposit64(env->regs[r3], 32, 32, ov);
case 1:
{
- uint64_t nv = cpu_ldq_data_ra(env, pl, ra);
+ uint64_t nv = cpu_ldq_mmu(env, pl, oi8, ra);
uint64_t cv = env->regs[r3];
uint64_t ov;
if (parallel) {
#ifdef CONFIG_ATOMIC64
- MemOpIdx oi = make_memop_idx(MO_TEUQ | MO_ALIGN, mem_idx);
- ov = cpu_atomic_cmpxchgq_be_mmu(env, a1, cv, nv, oi, ra);
+ ov = cpu_atomic_cmpxchgq_be_mmu(env, a1, cv, nv, oi8, ra);
#else
/* Note that we asserted !parallel above. */
g_assert_not_reached();
#endif
} else {
- ov = cpu_ldq_data_ra(env, a1, ra);
- cpu_stq_data_ra(env, a1, (ov == cv ? nv : ov), ra);
+ ov = cpu_ldq_mmu(env, a1, oi8, ra);
+ cpu_stq_mmu(env, a1, (ov == cv ? nv : ov), oi8, ra);
}
cc = (ov != cv);
env->regs[r3] = ov;
case 2:
{
- uint64_t nvh = cpu_ldq_data_ra(env, pl, ra);
- uint64_t nvl = cpu_ldq_data_ra(env, pl + 8, ra);
- Int128 nv = int128_make128(nvl, nvh);
+ Int128 nv = cpu_ld16_mmu(env, pl, oi16, ra);
Int128 cv = int128_make128(env->regs[r3 + 1], env->regs[r3]);
Int128 ov;
if (!parallel) {
- uint64_t oh = cpu_ldq_data_ra(env, a1 + 0, ra);
- uint64_t ol = cpu_ldq_data_ra(env, a1 + 8, ra);
-
- ov = int128_make128(ol, oh);
+ ov = cpu_ld16_mmu(env, a1, oi16, ra);
cc = !int128_eq(ov, cv);
if (cc) {
nv = ov;
}
-
- cpu_stq_data_ra(env, a1 + 0, int128_gethi(nv), ra);
- cpu_stq_data_ra(env, a1 + 8, int128_getlo(nv), ra);
+ cpu_st16_mmu(env, a1, nv, oi16, ra);
} else if (HAVE_CMPXCHG128) {
- MemOpIdx oi = make_memop_idx(MO_TE | MO_128 | MO_ALIGN, mem_idx);
- ov = cpu_atomic_cmpxchgo_be_mmu(env, a1, cv, nv, oi, ra);
+ ov = cpu_atomic_cmpxchgo_be_mmu(env, a1, cv, nv, oi16, ra);
cc = !int128_eq(ov, cv);
} else {
/* Note that we asserted !parallel above. */
if (cc == 0) {
switch (sc) {
case 0:
- cpu_stb_data_ra(env, a2, svh >> 56, ra);
+ cpu_stb_mmu(env, a2, svh >> 56, oi1, ra);
break;
case 1:
- cpu_stw_data_ra(env, a2, svh >> 48, ra);
+ cpu_stw_mmu(env, a2, svh >> 48, oi2, ra);
break;
case 2:
- cpu_stl_data_ra(env, a2, svh >> 32, ra);
+ cpu_stl_mmu(env, a2, svh >> 32, oi4, ra);
break;
case 3:
- cpu_stq_data_ra(env, a2, svh, ra);
+ cpu_stq_mmu(env, a2, svh, oi8, ra);
break;
case 4:
- if (!parallel) {
- cpu_stq_data_ra(env, a2 + 0, svh, ra);
- cpu_stq_data_ra(env, a2 + 8, svl, ra);
- } else if (HAVE_ATOMIC128) {
- MemOpIdx oi = make_memop_idx(MO_TEUQ | MO_ALIGN_16, mem_idx);
- Int128 sv = int128_make128(svl, svh);
- cpu_atomic_sto_be_mmu(env, a2, sv, oi, ra);
- } else {
- /* Note that we asserted !parallel above. */
- g_assert_not_reached();
- }
+ cpu_st16_mmu(env, a2, int128_make128(svl, svh), oi16, ra);
break;
default:
g_assert_not_reached();
}
#endif
-/* load pair from quadword */
-uint64_t HELPER(lpq)(CPUS390XState *env, uint64_t addr)
-{
- uintptr_t ra = GETPC();
- uint64_t hi, lo;
-
- check_alignment(env, addr, 16, ra);
- hi = cpu_ldq_data_ra(env, addr + 0, ra);
- lo = cpu_ldq_data_ra(env, addr + 8, ra);
-
- env->retxl = lo;
- return hi;
-}
-
-uint64_t HELPER(lpq_parallel)(CPUS390XState *env, uint64_t addr)
-{
- uintptr_t ra = GETPC();
- uint64_t hi, lo;
- int mem_idx;
- MemOpIdx oi;
- Int128 v;
-
- assert(HAVE_ATOMIC128);
-
- mem_idx = cpu_mmu_index(env, false);
- oi = make_memop_idx(MO_TEUQ | MO_ALIGN_16, mem_idx);
- v = cpu_atomic_ldo_be_mmu(env, addr, oi, ra);
- hi = int128_gethi(v);
- lo = int128_getlo(v);
-
- env->retxl = lo;
- return hi;
-}
-
-/* store pair to quadword */
-void HELPER(stpq)(CPUS390XState *env, uint64_t addr,
- uint64_t low, uint64_t high)
-{
- uintptr_t ra = GETPC();
-
- check_alignment(env, addr, 16, ra);
- cpu_stq_data_ra(env, addr + 0, high, ra);
- cpu_stq_data_ra(env, addr + 8, low, ra);
-}
-
-void HELPER(stpq_parallel)(CPUS390XState *env, uint64_t addr,
- uint64_t low, uint64_t high)
-{
- uintptr_t ra = GETPC();
- int mem_idx;
- MemOpIdx oi;
- Int128 v;
-
- assert(HAVE_ATOMIC128);
-
- mem_idx = cpu_mmu_index(env, false);
- oi = make_memop_idx(MO_TEUQ | MO_ALIGN_16, mem_idx);
- v = int128_make128(low, high);
- cpu_atomic_sto_be_mmu(env, addr, v, oi, ra);
-}
-
/* Execute instruction. This instruction executes an insn modified with
the contents of r1. It does not change the executed instruction in memory;
it does not change the program counter.
tcg_gen_st32_i64(v, cpu_env, freg32_offset(reg));
}
-static void return_low128(TCGv_i64 dest)
-{
- tcg_gen_ld_i64(dest, cpu_env, offsetof(CPUS390XState, retxl));
-}
-
static void update_psw_addr(DisasContext *s)
{
/* psw.addr */
static DisasJumpType op_lpq(DisasContext *s, DisasOps *o)
{
- if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) {
- gen_helper_lpq(o->out, cpu_env, o->in2);
- } else if (HAVE_ATOMIC128) {
- gen_helper_lpq_parallel(o->out, cpu_env, o->in2);
- } else {
- gen_helper_exit_atomic(cpu_env);
- return DISAS_NORETURN;
- }
- return_low128(o->out2);
+ o->out_128 = tcg_temp_new_i128();
+ tcg_gen_qemu_ld_i128(o->out_128, o->in2, get_mem_index(s),
+ MO_TE | MO_128 | MO_ALIGN);
return DISAS_NEXT;
}
static DisasJumpType op_stpq(DisasContext *s, DisasOps *o)
{
- if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) {
- gen_helper_stpq(cpu_env, o->in2, o->out2, o->out);
- } else if (HAVE_ATOMIC128) {
- gen_helper_stpq_parallel(cpu_env, o->in2, o->out2, o->out);
- } else {
- gen_helper_exit_atomic(cpu_env);
- return DISAS_NORETURN;
- }
+ TCGv_i128 t16 = tcg_temp_new_i128();
+
+ tcg_gen_concat_i64_i128(t16, o->out2, o->out);
+ tcg_gen_qemu_st_i128(t16, o->in2, get_mem_index(s),
+ MO_TE | MO_128 | MO_ALIGN);
return DISAS_NEXT;
}
* License along with this library; if not, see <http://www.gnu.org/licenses/>.
*/
-#define DEBUG_DISAS
-
#include "qemu/osdep.h"
#include "cpu.h"
#include "disas/disas.h"
ret = cpu_ldb_mmu(env, addr, oi, GETPC());
break;
case 2:
- if (asi & 8) {
- ret = cpu_ldw_le_mmu(env, addr, oi, GETPC());
- } else {
- ret = cpu_ldw_be_mmu(env, addr, oi, GETPC());
- }
+ ret = cpu_ldw_mmu(env, addr, oi, GETPC());
break;
case 4:
- if (asi & 8) {
- ret = cpu_ldl_le_mmu(env, addr, oi, GETPC());
- } else {
- ret = cpu_ldl_be_mmu(env, addr, oi, GETPC());
- }
+ ret = cpu_ldl_mmu(env, addr, oi, GETPC());
break;
case 8:
- if (asi & 8) {
- ret = cpu_ldq_le_mmu(env, addr, oi, GETPC());
- } else {
- ret = cpu_ldq_be_mmu(env, addr, oi, GETPC());
- }
+ ret = cpu_ldq_mmu(env, addr, oi, GETPC());
break;
default:
g_assert_not_reached();
#include "asi.h"
-#define DEBUG_DISAS
-
#define DYNAMIC_PC 1 /* dynamic pc value */
#define JUMP_PC 2 /* dynamic pc value which takes only two values
according to jump_pc[T2] */
#include "../tcg-ldst.c.inc"
#include "../tcg-pool.c.inc"
#include "qemu/bitops.h"
-#ifdef __linux__
-#include <asm/hwcap.h>
-#endif
-#ifdef CONFIG_DARWIN
-#include <sys/sysctl.h>
-#endif
/* We're going to re-use TCGType in setting of the SF bit, which controls
the size of the operation performed. If we know the values match, it
return TCG_REG_X0 + slot;
}
-bool have_lse;
-bool have_lse2;
-
#define TCG_REG_TMP TCG_REG_X30
#define TCG_VEC_TMP TCG_REG_V31
}
}
-#ifdef CONFIG_DARWIN
-static bool sysctl_for_bool(const char *name)
-{
- int val = 0;
- size_t len = sizeof(val);
-
- if (sysctlbyname(name, &val, &len, NULL, 0) == 0) {
- return val != 0;
- }
-
- /*
- * We might in the future ask for properties not present in older kernels,
- * but we're only asking about static properties, all of which should be
- * 'int'. So we shouln't see ENOMEM (val too small), or any of the other
- * more exotic errors.
- */
- assert(errno == ENOENT);
- return false;
-}
-#endif
-
static void tcg_target_init(TCGContext *s)
{
-#ifdef __linux__
- unsigned long hwcap = qemu_getauxval(AT_HWCAP);
- have_lse = hwcap & HWCAP_ATOMICS;
- have_lse2 = hwcap & HWCAP_USCAT;
-#endif
-#ifdef CONFIG_DARWIN
- have_lse = sysctl_for_bool("hw.optional.arm.FEAT_LSE");
- have_lse2 = sysctl_for_bool("hw.optional.arm.FEAT_LSE2");
-#endif
-
tcg_target_available_regs[TCG_TYPE_I32] = 0xffffffffu;
tcg_target_available_regs[TCG_TYPE_I64] = 0xffffffffu;
tcg_target_available_regs[TCG_TYPE_V64] = 0xffffffff00000000ull;
#ifndef AARCH64_TCG_TARGET_H
#define AARCH64_TCG_TARGET_H
+#include "host/cpuinfo.h"
+
#define TCG_TARGET_INSN_UNIT_SIZE 4
#define TCG_TARGET_TLB_DISPLACEMENT_BITS 24
#define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
#define TCG_TARGET_CALL_ARG_I128 TCG_CALL_ARG_EVEN
#define TCG_TARGET_CALL_RET_I128 TCG_CALL_RET_NORMAL
-extern bool have_lse;
-extern bool have_lse2;
+#define have_lse (cpuinfo & CPUINFO_LSE)
+#define have_lse2 (cpuinfo & CPUINFO_LSE2)
/* optional instructions */
#define TCG_TARGET_HAS_div_i32 1
# define SOFTMMU_RESERVE_REGS 0
#endif
-/* The host compiler should supply <cpuid.h> to enable runtime features
- detection, as we're not going to go so far as our own inline assembly.
- If not available, default values will be assumed. */
-#if defined(CONFIG_CPUID_H)
-#include "qemu/cpuid.h"
-#endif
-
/* For 64-bit, we always know that CMOV is available. */
#if TCG_TARGET_REG_BITS == 64
-# define have_cmov 1
-#elif defined(CONFIG_CPUID_H)
-static bool have_cmov;
-#else
-# define have_cmov 0
-#endif
-
-/* We need these symbols in tcg-target.h, and we can't properly conditionalize
- it there. Therefore we always define the variable. */
-bool have_bmi1;
-bool have_popcnt;
-bool have_avx1;
-bool have_avx2;
-bool have_avx512bw;
-bool have_avx512dq;
-bool have_avx512vbmi2;
-bool have_avx512vl;
-bool have_movbe;
-bool have_atomic16;
-
-#ifdef CONFIG_CPUID_H
-static bool have_bmi2;
-static bool have_lzcnt;
+# define have_cmov true
#else
-# define have_bmi2 0
-# define have_lzcnt 0
+# define have_cmov (cpuinfo & CPUINFO_CMOV)
#endif
+#define have_bmi2 (cpuinfo & CPUINFO_BMI2)
+#define have_lzcnt (cpuinfo & CPUINFO_LZCNT)
static const tcg_insn_unit *tb_ret_addr;
static void tcg_target_init(TCGContext *s)
{
-#ifdef CONFIG_CPUID_H
- unsigned a, b, c, d, b7 = 0, c7 = 0;
- unsigned max = __get_cpuid_max(0, 0);
-
- if (max >= 7) {
- /* BMI1 is available on AMD Piledriver and Intel Haswell CPUs. */
- __cpuid_count(7, 0, a, b7, c7, d);
- have_bmi1 = (b7 & bit_BMI) != 0;
- have_bmi2 = (b7 & bit_BMI2) != 0;
- }
-
- if (max >= 1) {
- __cpuid(1, a, b, c, d);
-#ifndef have_cmov
- /* For 32-bit, 99% certainty that we're running on hardware that
- supports cmov, but we still need to check. In case cmov is not
- available, we'll use a small forward branch. */
- have_cmov = (d & bit_CMOV) != 0;
-#endif
-
- /* MOVBE is only available on Intel Atom and Haswell CPUs, so we
- need to probe for it. */
- have_movbe = (c & bit_MOVBE) != 0;
- have_popcnt = (c & bit_POPCNT) != 0;
-
- /* There are a number of things we must check before we can be
- sure of not hitting invalid opcode. */
- if (c & bit_OSXSAVE) {
- unsigned bv = xgetbv_low(0);
-
- if ((bv & 6) == 6) {
- have_avx1 = (c & bit_AVX) != 0;
- have_avx2 = (b7 & bit_AVX2) != 0;
-
- /*
- * There are interesting instructions in AVX512, so long
- * as we have AVX512VL, which indicates support for EVEX
- * on sizes smaller than 512 bits. We are required to
- * check that OPMASK and all extended ZMM state are enabled
- * even if we're not using them -- the insns will fault.
- */
- if ((bv & 0xe0) == 0xe0
- && (b7 & bit_AVX512F)
- && (b7 & bit_AVX512VL)) {
- have_avx512vl = true;
- have_avx512bw = (b7 & bit_AVX512BW) != 0;
- have_avx512dq = (b7 & bit_AVX512DQ) != 0;
- have_avx512vbmi2 = (c7 & bit_AVX512VBMI2) != 0;
- }
-
- /*
- * The Intel SDM has added:
- * Processors that enumerate support for Intel® AVX
- * (by setting the feature flag CPUID.01H:ECX.AVX[bit 28])
- * guarantee that the 16-byte memory operations performed
- * by the following instructions will always be carried
- * out atomically:
- * - MOVAPD, MOVAPS, and MOVDQA.
- * - VMOVAPD, VMOVAPS, and VMOVDQA when encoded with VEX.128.
- * - VMOVAPD, VMOVAPS, VMOVDQA32, and VMOVDQA64 when encoded
- * with EVEX.128 and k0 (masking disabled).
- * Note that these instructions require the linear addresses
- * of their memory operands to be 16-byte aligned.
- *
- * AMD has provided an even stronger guarantee that processors
- * with AVX provide 16-byte atomicity for all cachable,
- * naturally aligned single loads and stores, e.g. MOVDQU.
- *
- * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104688
- */
- if (have_avx1) {
- __cpuid(0, a, b, c, d);
- have_atomic16 = (c == signature_INTEL_ecx ||
- c == signature_AMD_ecx);
- }
- }
- }
- }
-
- max = __get_cpuid_max(0x8000000, 0);
- if (max >= 1) {
- __cpuid(0x80000001, a, b, c, d);
- /* LZCNT was introduced with AMD Barcelona and Intel Haswell CPUs. */
- have_lzcnt = (c & bit_LZCNT) != 0;
- }
-#endif /* CONFIG_CPUID_H */
-
tcg_target_available_regs[TCG_TYPE_I32] = ALL_GENERAL_REGS;
if (TCG_TARGET_REG_BITS == 64) {
tcg_target_available_regs[TCG_TYPE_I64] = ALL_GENERAL_REGS;
#ifndef I386_TCG_TARGET_H
#define I386_TCG_TARGET_H
+#include "host/cpuinfo.h"
+
#define TCG_TARGET_INSN_UNIT_SIZE 1
#define TCG_TARGET_TLB_DISPLACEMENT_BITS 31
# define TCG_TARGET_CALL_RET_I128 TCG_CALL_RET_BY_REF
#endif
-extern bool have_bmi1;
-extern bool have_popcnt;
-extern bool have_avx1;
-extern bool have_avx2;
-extern bool have_avx512bw;
-extern bool have_avx512dq;
-extern bool have_avx512vbmi2;
-extern bool have_avx512vl;
-extern bool have_movbe;
-extern bool have_atomic16;
+#define have_bmi1 (cpuinfo & CPUINFO_BMI1)
+#define have_popcnt (cpuinfo & CPUINFO_POPCNT)
+#define have_avx1 (cpuinfo & CPUINFO_AVX1)
+#define have_avx2 (cpuinfo & CPUINFO_AVX2)
+#define have_movbe (cpuinfo & CPUINFO_MOVBE)
+#define have_atomic16 (cpuinfo & CPUINFO_ATOMIC_VMOVDQA)
+
+/*
+ * There are interesting instructions in AVX512, so long as we have AVX512VL,
+ * which indicates support for EVEX on sizes smaller than 512 bits.
+ */
+#define have_avx512vl ((cpuinfo & CPUINFO_AVX512VL) && \
+ (cpuinfo & CPUINFO_AVX512F))
+#define have_avx512bw ((cpuinfo & CPUINFO_AVX512BW) && have_avx512vl)
+#define have_avx512dq ((cpuinfo & CPUINFO_AVX512DQ) && have_avx512vl)
+#define have_avx512vbmi2 ((cpuinfo & CPUINFO_AVX512VBMI2) && have_avx512vl)
/* optional instructions */
#define TCG_TARGET_HAS_div2_i32 1
{
if (TCG_TARGET_REG_BITS == 32) {
/* Inline expansion below is simply too large for 32-bit hosts. */
- gen_atomic_cx_i128 gen = ((memop & MO_BSWAP) == MO_LE
- ? gen_helper_nonatomic_cmpxchgo_le
- : gen_helper_nonatomic_cmpxchgo_be);
MemOpIdx oi = make_memop_idx(memop, idx);
TCGv_i64 a64 = maybe_extend_addr64(addr);
- gen(retv, cpu_env, a64, cmpv, newv, tcg_constant_i32(oi));
+ gen_helper_nonatomic_cmpxchgo(retv, cpu_env, a64, cmpv, newv,
+ tcg_constant_i32(oi));
maybe_free_addr64(a64);
} else {
TCGv_i128 oldv = tcg_temp_ebb_new_i128();
* THE SOFTWARE.
*/
-/* define it to use liveness analysis (better code) */
-#define USE_TCG_OPTIMIZATIONS
-
#include "qemu/osdep.h"
/* Define to jump the ELF file used to communicate with GDB. */
(uintptr_t)s->code_buf, prologue_size);
#endif
-#ifdef DEBUG_DISAS
if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
FILE *logfile = qemu_log_trylock();
if (logfile) {
qemu_log_unlock(logfile);
}
}
-#endif
#ifndef CONFIG_TCG_INTERPRETER
/*
}
#endif
-#ifdef DEBUG_DISAS
if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
&& qemu_log_in_addr_range(pc_start))) {
FILE *logfile = qemu_log_trylock();
qemu_log_unlock(logfile);
}
}
-#endif
#ifdef CONFIG_DEBUG_TCG
/* Ensure all labels referenced have been emitted. */
qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock());
#endif
-#ifdef USE_TCG_OPTIMIZATIONS
tcg_optimize(s);
-#endif
#ifdef CONFIG_PROFILER
qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock());
liveness_pass_1(s);
if (s->nb_indirects > 0) {
-#ifdef DEBUG_DISAS
if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND)
&& qemu_log_in_addr_range(pc_start))) {
FILE *logfile = qemu_log_trylock();
qemu_log_unlock(logfile);
}
}
-#endif
+
/* Replace indirect temps with direct temps. */
if (liveness_pass_2(s)) {
/* If changes were made, re-run liveness. */
qatomic_set(&prof->la_time, prof->la_time + profile_getclock());
#endif
-#ifdef DEBUG_DISAS
if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
&& qemu_log_in_addr_range(pc_start))) {
FILE *logfile = qemu_log_trylock();
qemu_log_unlock(logfile);
}
}
-#endif
/* Initialize goto_tb jump offsets. */
tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID;
from avocado import skip
from avocado_qemu import QemuSystemTest
-from avocado_qemu import exec_command, exec_command_and_wait_for_pattern
from avocado_qemu import wait_for_console_pattern
class NetDevEthtool(QemuSystemTest):
# URL into a unique one
return self.fetch_asset(name=name, locations=(url), asset_hash=sha1)
- def common_test_code(self, netdev, extra_args=None, kvm=False):
+ def common_test_code(self, netdev, extra_args=None):
# This custom kernel has drivers for all the supported network
# devices we can emulate in QEMU
'-drive', drive,
'-device', netdev)
- if kvm:
- self.vm.add_args('-accel', 'kvm')
-
self.vm.set_console(console_index=0)
self.vm.launch()
# no need to gracefully shutdown, just finish
self.vm.kill()
- # Skip testing for MSI for now. Allegedly it was fixed by:
- # 28e96556ba (igb: Allocate MSI-X vector when testing)
- # but I'm seeing oops in the kernel
- @skip("Kernel bug with MSI enabled")
def test_igb(self):
"""
:avocado: tags=device:igb
"""
self.common_test_code("igb", "pci=nomsi")
- def test_igb_nomsi_kvm(self):
- """
- :avocado: tags=device:igb
- """
- self.require_accelerator('kvm')
- self.common_test_code("igb", "pci=nomsi", True)
-
# It seems the other popular cards we model in QEMU currently fail
# the pattern test with:
#
sources: 'qht-bench.c',
dependencies: [qemuutil])
-if have_system
-xbzrle_bench = executable('xbzrle-bench',
- sources: 'xbzrle-bench.c',
- dependencies: [qemuutil,migration])
-endif
-
qtree_bench = executable('qtree-bench',
sources: 'qtree-bench.c',
dependencies: [qemuutil])
+++ /dev/null
-/*
- * Xor Based Zero Run Length Encoding unit tests.
- *
- * Copyright 2013 Red Hat, Inc. and/or its affiliates
- *
- * Authors:
- * Orit Wasserman <owasserm@redhat.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2 or later.
- * See the COPYING file in the top-level directory.
- *
- */
-#include "qemu/osdep.h"
-#include "qemu/cutils.h"
-#include "../migration/xbzrle.h"
-
-#if defined(CONFIG_AVX512BW_OPT)
-#define XBZRLE_PAGE_SIZE 4096
-static bool is_cpu_support_avx512bw;
-#include "qemu/cpuid.h"
-static void __attribute__((constructor)) init_cpu_flag(void)
-{
- unsigned max = __get_cpuid_max(0, NULL);
- int a, b, c, d;
- is_cpu_support_avx512bw = false;
- if (max >= 1) {
- __cpuid(1, a, b, c, d);
- /* We must check that AVX is not just available, but usable. */
- if ((c & bit_OSXSAVE) && (c & bit_AVX) && max >= 7) {
- int bv;
- __asm("xgetbv" : "=a"(bv), "=d"(d) : "c"(0));
- __cpuid_count(7, 0, a, b, c, d);
- /* 0xe6:
- * XCR0[7:5] = 111b (OPMASK state, upper 256-bit of ZMM0-ZMM15
- * and ZMM16-ZMM31 state are enabled by OS)
- * XCR0[2:1] = 11b (XMM state and YMM state are enabled by OS)
- */
- if ((bv & 0xe6) == 0xe6 && (b & bit_AVX512BW)) {
- is_cpu_support_avx512bw = true;
- }
- }
- }
- return ;
-}
-
-struct ResTime {
- float t_raw;
- float t_512;
-};
-
-
-/* Function prototypes
-int xbzrle_encode_buffer_avx512(uint8_t *old_buf, uint8_t *new_buf, int slen,
- uint8_t *dst, int dlen);
-*/
-static void encode_decode_zero(struct ResTime *res)
-{
- uint8_t *buffer = g_malloc0(XBZRLE_PAGE_SIZE);
- uint8_t *compressed = g_malloc0(XBZRLE_PAGE_SIZE);
- uint8_t *buffer512 = g_malloc0(XBZRLE_PAGE_SIZE);
- uint8_t *compressed512 = g_malloc0(XBZRLE_PAGE_SIZE);
- int i = 0;
- int dlen = 0, dlen512 = 0;
- int diff_len = g_test_rand_int_range(0, XBZRLE_PAGE_SIZE - 1006);
-
- for (i = diff_len; i > 0; i--) {
- buffer[1000 + i] = i;
- buffer512[1000 + i] = i;
- }
-
- buffer[1000 + diff_len + 3] = 103;
- buffer[1000 + diff_len + 5] = 105;
-
- buffer512[1000 + diff_len + 3] = 103;
- buffer512[1000 + diff_len + 5] = 105;
-
- /* encode zero page */
- time_t t_start, t_end, t_start512, t_end512;
- t_start = clock();
- dlen = xbzrle_encode_buffer(buffer, buffer, XBZRLE_PAGE_SIZE, compressed,
- XBZRLE_PAGE_SIZE);
- t_end = clock();
- float time_val = difftime(t_end, t_start);
- g_assert(dlen == 0);
-
- t_start512 = clock();
- dlen512 = xbzrle_encode_buffer_avx512(buffer512, buffer512, XBZRLE_PAGE_SIZE,
- compressed512, XBZRLE_PAGE_SIZE);
- t_end512 = clock();
- float time_val512 = difftime(t_end512, t_start512);
- g_assert(dlen512 == 0);
-
- res->t_raw = time_val;
- res->t_512 = time_val512;
-
- g_free(buffer);
- g_free(compressed);
- g_free(buffer512);
- g_free(compressed512);
-
-}
-
-static void test_encode_decode_zero_avx512(void)
-{
- int i;
- float time_raw = 0.0, time_512 = 0.0;
- struct ResTime res;
- for (i = 0; i < 10000; i++) {
- encode_decode_zero(&res);
- time_raw += res.t_raw;
- time_512 += res.t_512;
- }
- printf("Zero test:\n");
- printf("Raw xbzrle_encode time is %f ms\n", time_raw);
- printf("512 xbzrle_encode time is %f ms\n", time_512);
-}
-
-static void encode_decode_unchanged(struct ResTime *res)
-{
- uint8_t *compressed = g_malloc0(XBZRLE_PAGE_SIZE);
- uint8_t *test = g_malloc0(XBZRLE_PAGE_SIZE);
- uint8_t *compressed512 = g_malloc0(XBZRLE_PAGE_SIZE);
- uint8_t *test512 = g_malloc0(XBZRLE_PAGE_SIZE);
- int i = 0;
- int dlen = 0, dlen512 = 0;
- int diff_len = g_test_rand_int_range(0, XBZRLE_PAGE_SIZE - 1006);
-
- for (i = diff_len; i > 0; i--) {
- test[1000 + i] = i + 4;
- test512[1000 + i] = i + 4;
- }
-
- test[1000 + diff_len + 3] = 107;
- test[1000 + diff_len + 5] = 109;
-
- test512[1000 + diff_len + 3] = 107;
- test512[1000 + diff_len + 5] = 109;
-
- /* test unchanged buffer */
- time_t t_start, t_end, t_start512, t_end512;
- t_start = clock();
- dlen = xbzrle_encode_buffer(test, test, XBZRLE_PAGE_SIZE, compressed,
- XBZRLE_PAGE_SIZE);
- t_end = clock();
- float time_val = difftime(t_end, t_start);
- g_assert(dlen == 0);
-
- t_start512 = clock();
- dlen512 = xbzrle_encode_buffer_avx512(test512, test512, XBZRLE_PAGE_SIZE,
- compressed512, XBZRLE_PAGE_SIZE);
- t_end512 = clock();
- float time_val512 = difftime(t_end512, t_start512);
- g_assert(dlen512 == 0);
-
- res->t_raw = time_val;
- res->t_512 = time_val512;
-
- g_free(test);
- g_free(compressed);
- g_free(test512);
- g_free(compressed512);
-
-}
-
-static void test_encode_decode_unchanged_avx512(void)
-{
- int i;
- float time_raw = 0.0, time_512 = 0.0;
- struct ResTime res;
- for (i = 0; i < 10000; i++) {
- encode_decode_unchanged(&res);
- time_raw += res.t_raw;
- time_512 += res.t_512;
- }
- printf("Unchanged test:\n");
- printf("Raw xbzrle_encode time is %f ms\n", time_raw);
- printf("512 xbzrle_encode time is %f ms\n", time_512);
-}
-
-static void encode_decode_1_byte(struct ResTime *res)
-{
- uint8_t *buffer = g_malloc0(XBZRLE_PAGE_SIZE);
- uint8_t *test = g_malloc0(XBZRLE_PAGE_SIZE);
- uint8_t *compressed = g_malloc(XBZRLE_PAGE_SIZE);
- uint8_t *buffer512 = g_malloc0(XBZRLE_PAGE_SIZE);
- uint8_t *test512 = g_malloc0(XBZRLE_PAGE_SIZE);
- uint8_t *compressed512 = g_malloc(XBZRLE_PAGE_SIZE);
- int dlen = 0, rc = 0, dlen512 = 0, rc512 = 0;
- uint8_t buf[2];
- uint8_t buf512[2];
-
- test[XBZRLE_PAGE_SIZE - 1] = 1;
- test512[XBZRLE_PAGE_SIZE - 1] = 1;
-
- time_t t_start, t_end, t_start512, t_end512;
- t_start = clock();
- dlen = xbzrle_encode_buffer(buffer, test, XBZRLE_PAGE_SIZE, compressed,
- XBZRLE_PAGE_SIZE);
- t_end = clock();
- float time_val = difftime(t_end, t_start);
- g_assert(dlen == (uleb128_encode_small(&buf[0], 4095) + 2));
-
- rc = xbzrle_decode_buffer(compressed, dlen, buffer, XBZRLE_PAGE_SIZE);
- g_assert(rc == XBZRLE_PAGE_SIZE);
- g_assert(memcmp(test, buffer, XBZRLE_PAGE_SIZE) == 0);
-
- t_start512 = clock();
- dlen512 = xbzrle_encode_buffer_avx512(buffer512, test512, XBZRLE_PAGE_SIZE,
- compressed512, XBZRLE_PAGE_SIZE);
- t_end512 = clock();
- float time_val512 = difftime(t_end512, t_start512);
- g_assert(dlen512 == (uleb128_encode_small(&buf512[0], 4095) + 2));
-
- rc512 = xbzrle_decode_buffer(compressed512, dlen512, buffer512,
- XBZRLE_PAGE_SIZE);
- g_assert(rc512 == XBZRLE_PAGE_SIZE);
- g_assert(memcmp(test512, buffer512, XBZRLE_PAGE_SIZE) == 0);
-
- res->t_raw = time_val;
- res->t_512 = time_val512;
-
- g_free(buffer);
- g_free(compressed);
- g_free(test);
- g_free(buffer512);
- g_free(compressed512);
- g_free(test512);
-
-}
-
-static void test_encode_decode_1_byte_avx512(void)
-{
- int i;
- float time_raw = 0.0, time_512 = 0.0;
- struct ResTime res;
- for (i = 0; i < 10000; i++) {
- encode_decode_1_byte(&res);
- time_raw += res.t_raw;
- time_512 += res.t_512;
- }
- printf("1 byte test:\n");
- printf("Raw xbzrle_encode time is %f ms\n", time_raw);
- printf("512 xbzrle_encode time is %f ms\n", time_512);
-}
-
-static void encode_decode_overflow(struct ResTime *res)
-{
- uint8_t *compressed = g_malloc0(XBZRLE_PAGE_SIZE);
- uint8_t *test = g_malloc0(XBZRLE_PAGE_SIZE);
- uint8_t *buffer = g_malloc0(XBZRLE_PAGE_SIZE);
- uint8_t *compressed512 = g_malloc0(XBZRLE_PAGE_SIZE);
- uint8_t *test512 = g_malloc0(XBZRLE_PAGE_SIZE);
- uint8_t *buffer512 = g_malloc0(XBZRLE_PAGE_SIZE);
- int i = 0, rc = 0, rc512 = 0;
-
- for (i = 0; i < XBZRLE_PAGE_SIZE / 2 - 1; i++) {
- test[i * 2] = 1;
- test512[i * 2] = 1;
- }
-
- /* encode overflow */
- time_t t_start, t_end, t_start512, t_end512;
- t_start = clock();
- rc = xbzrle_encode_buffer(buffer, test, XBZRLE_PAGE_SIZE, compressed,
- XBZRLE_PAGE_SIZE);
- t_end = clock();
- float time_val = difftime(t_end, t_start);
- g_assert(rc == -1);
-
- t_start512 = clock();
- rc512 = xbzrle_encode_buffer_avx512(buffer512, test512, XBZRLE_PAGE_SIZE,
- compressed512, XBZRLE_PAGE_SIZE);
- t_end512 = clock();
- float time_val512 = difftime(t_end512, t_start512);
- g_assert(rc512 == -1);
-
- res->t_raw = time_val;
- res->t_512 = time_val512;
-
- g_free(buffer);
- g_free(compressed);
- g_free(test);
- g_free(buffer512);
- g_free(compressed512);
- g_free(test512);
-
-}
-
-static void test_encode_decode_overflow_avx512(void)
-{
- int i;
- float time_raw = 0.0, time_512 = 0.0;
- struct ResTime res;
- for (i = 0; i < 10000; i++) {
- encode_decode_overflow(&res);
- time_raw += res.t_raw;
- time_512 += res.t_512;
- }
- printf("Overflow test:\n");
- printf("Raw xbzrle_encode time is %f ms\n", time_raw);
- printf("512 xbzrle_encode time is %f ms\n", time_512);
-}
-
-static void encode_decode_range_avx512(struct ResTime *res)
-{
- uint8_t *buffer = g_malloc0(XBZRLE_PAGE_SIZE);
- uint8_t *compressed = g_malloc(XBZRLE_PAGE_SIZE);
- uint8_t *test = g_malloc0(XBZRLE_PAGE_SIZE);
- uint8_t *buffer512 = g_malloc0(XBZRLE_PAGE_SIZE);
- uint8_t *compressed512 = g_malloc(XBZRLE_PAGE_SIZE);
- uint8_t *test512 = g_malloc0(XBZRLE_PAGE_SIZE);
- int i = 0, rc = 0, rc512 = 0;
- int dlen = 0, dlen512 = 0;
-
- int diff_len = g_test_rand_int_range(0, XBZRLE_PAGE_SIZE - 1006);
-
- for (i = diff_len; i > 0; i--) {
- buffer[1000 + i] = i;
- test[1000 + i] = i + 4;
- buffer512[1000 + i] = i;
- test512[1000 + i] = i + 4;
- }
-
- buffer[1000 + diff_len + 3] = 103;
- test[1000 + diff_len + 3] = 107;
-
- buffer[1000 + diff_len + 5] = 105;
- test[1000 + diff_len + 5] = 109;
-
- buffer512[1000 + diff_len + 3] = 103;
- test512[1000 + diff_len + 3] = 107;
-
- buffer512[1000 + diff_len + 5] = 105;
- test512[1000 + diff_len + 5] = 109;
-
- /* test encode/decode */
- time_t t_start, t_end, t_start512, t_end512;
- t_start = clock();
- dlen = xbzrle_encode_buffer(test, buffer, XBZRLE_PAGE_SIZE, compressed,
- XBZRLE_PAGE_SIZE);
- t_end = clock();
- float time_val = difftime(t_end, t_start);
- rc = xbzrle_decode_buffer(compressed, dlen, test, XBZRLE_PAGE_SIZE);
- g_assert(rc < XBZRLE_PAGE_SIZE);
- g_assert(memcmp(test, buffer, XBZRLE_PAGE_SIZE) == 0);
-
- t_start512 = clock();
- dlen512 = xbzrle_encode_buffer_avx512(test512, buffer512, XBZRLE_PAGE_SIZE,
- compressed512, XBZRLE_PAGE_SIZE);
- t_end512 = clock();
- float time_val512 = difftime(t_end512, t_start512);
- rc512 = xbzrle_decode_buffer(compressed512, dlen512, test512, XBZRLE_PAGE_SIZE);
- g_assert(rc512 < XBZRLE_PAGE_SIZE);
- g_assert(memcmp(test512, buffer512, XBZRLE_PAGE_SIZE) == 0);
-
- res->t_raw = time_val;
- res->t_512 = time_val512;
-
- g_free(buffer);
- g_free(compressed);
- g_free(test);
- g_free(buffer512);
- g_free(compressed512);
- g_free(test512);
-
-}
-
-static void test_encode_decode_avx512(void)
-{
- int i;
- float time_raw = 0.0, time_512 = 0.0;
- struct ResTime res;
- for (i = 0; i < 10000; i++) {
- encode_decode_range_avx512(&res);
- time_raw += res.t_raw;
- time_512 += res.t_512;
- }
- printf("Encode decode test:\n");
- printf("Raw xbzrle_encode time is %f ms\n", time_raw);
- printf("512 xbzrle_encode time is %f ms\n", time_512);
-}
-
-static void encode_decode_random(struct ResTime *res)
-{
- uint8_t *buffer = g_malloc0(XBZRLE_PAGE_SIZE);
- uint8_t *compressed = g_malloc(XBZRLE_PAGE_SIZE);
- uint8_t *test = g_malloc0(XBZRLE_PAGE_SIZE);
- uint8_t *buffer512 = g_malloc0(XBZRLE_PAGE_SIZE);
- uint8_t *compressed512 = g_malloc(XBZRLE_PAGE_SIZE);
- uint8_t *test512 = g_malloc0(XBZRLE_PAGE_SIZE);
- int i = 0, rc = 0, rc512 = 0;
- int dlen = 0, dlen512 = 0;
-
- int diff_len = g_test_rand_int_range(0, XBZRLE_PAGE_SIZE - 1);
- /* store the index of diff */
- int dirty_index[diff_len];
- for (int j = 0; j < diff_len; j++) {
- dirty_index[j] = g_test_rand_int_range(0, XBZRLE_PAGE_SIZE - 1);
- }
- for (i = diff_len - 1; i >= 0; i--) {
- buffer[dirty_index[i]] = i;
- test[dirty_index[i]] = i + 4;
- buffer512[dirty_index[i]] = i;
- test512[dirty_index[i]] = i + 4;
- }
-
- time_t t_start, t_end, t_start512, t_end512;
- t_start = clock();
- dlen = xbzrle_encode_buffer(test, buffer, XBZRLE_PAGE_SIZE, compressed,
- XBZRLE_PAGE_SIZE);
- t_end = clock();
- float time_val = difftime(t_end, t_start);
- rc = xbzrle_decode_buffer(compressed, dlen, test, XBZRLE_PAGE_SIZE);
- g_assert(rc < XBZRLE_PAGE_SIZE);
-
- t_start512 = clock();
- dlen512 = xbzrle_encode_buffer_avx512(test512, buffer512, XBZRLE_PAGE_SIZE,
- compressed512, XBZRLE_PAGE_SIZE);
- t_end512 = clock();
- float time_val512 = difftime(t_end512, t_start512);
- rc512 = xbzrle_decode_buffer(compressed512, dlen512, test512, XBZRLE_PAGE_SIZE);
- g_assert(rc512 < XBZRLE_PAGE_SIZE);
-
- res->t_raw = time_val;
- res->t_512 = time_val512;
-
- g_free(buffer);
- g_free(compressed);
- g_free(test);
- g_free(buffer512);
- g_free(compressed512);
- g_free(test512);
-
-}
-
-static void test_encode_decode_random_avx512(void)
-{
- int i;
- float time_raw = 0.0, time_512 = 0.0;
- struct ResTime res;
- for (i = 0; i < 10000; i++) {
- encode_decode_random(&res);
- time_raw += res.t_raw;
- time_512 += res.t_512;
- }
- printf("Random test:\n");
- printf("Raw xbzrle_encode time is %f ms\n", time_raw);
- printf("512 xbzrle_encode time is %f ms\n", time_512);
-}
-#endif
-
-int main(int argc, char **argv)
-{
- g_test_init(&argc, &argv, NULL);
- g_test_rand_int();
- #if defined(CONFIG_AVX512BW_OPT)
- if (likely(is_cpu_support_avx512bw)) {
- g_test_add_func("/xbzrle/encode_decode_zero", test_encode_decode_zero_avx512);
- g_test_add_func("/xbzrle/encode_decode_unchanged",
- test_encode_decode_unchanged_avx512);
- g_test_add_func("/xbzrle/encode_decode_1_byte", test_encode_decode_1_byte_avx512);
- g_test_add_func("/xbzrle/encode_decode_overflow",
- test_encode_decode_overflow_avx512);
- g_test_add_func("/xbzrle/encode_decode", test_encode_decode_avx512);
- g_test_add_func("/xbzrle/encode_decode_random", test_encode_decode_random_avx512);
- }
- #endif
- return g_test_run();
-}
e1000e_macreg_write(&d->e1000e, E1000_RCTL, E1000_RCTL_EN);
/* Enable all interrupts */
+ e1000e_macreg_write(&d->e1000e, E1000_GPIE, E1000_GPIE_MSIX_MODE);
e1000e_macreg_write(&d->e1000e, E1000_IMS, 0xFFFFFFFF);
e1000e_macreg_write(&d->e1000e, E1000_EIMS, 0xFFFFFFFF);
#define XBZRLE_PAGE_SIZE 4096
-int (*xbzrle_encode_buffer_func)(uint8_t *, uint8_t *, int,
- uint8_t *, int) = xbzrle_encode_buffer;
-#if defined(CONFIG_AVX512BW_OPT)
-#include "qemu/cpuid.h"
-static void __attribute__((constructor)) init_cpu_flag(void)
-{
- unsigned max = __get_cpuid_max(0, NULL);
- int a, b, c, d;
- if (max >= 1) {
- __cpuid(1, a, b, c, d);
- /* We must check that AVX is not just available, but usable. */
- if ((c & bit_OSXSAVE) && (c & bit_AVX) && max >= 7) {
- int bv;
- __asm("xgetbv" : "=a"(bv), "=d"(d) : "c"(0));
- __cpuid_count(7, 0, a, b, c, d);
- /* 0xe6:
- * XCR0[7:5] = 111b (OPMASK state, upper 256-bit of ZMM0-ZMM15
- * and ZMM16-ZMM31 state are enabled by OS)
- * XCR0[2:1] = 11b (XMM state and YMM state are enabled by OS)
- */
- if ((bv & 0xe6) == 0xe6 && (b & bit_AVX512BW)) {
- xbzrle_encode_buffer_func = xbzrle_encode_buffer_avx512;
- }
- }
- }
- return ;
-}
-#endif
-
static void test_uleb(void)
{
uint32_t i, val;
buffer[1000 + diff_len + 5] = 105;
/* encode zero page */
- dlen = xbzrle_encode_buffer_func(buffer, buffer, XBZRLE_PAGE_SIZE, compressed,
- XBZRLE_PAGE_SIZE);
+ dlen = xbzrle_encode_buffer(buffer, buffer, XBZRLE_PAGE_SIZE,
+ compressed, XBZRLE_PAGE_SIZE);
g_assert(dlen == 0);
g_free(buffer);
test[1000 + diff_len + 5] = 109;
/* test unchanged buffer */
- dlen = xbzrle_encode_buffer_func(test, test, XBZRLE_PAGE_SIZE, compressed,
- XBZRLE_PAGE_SIZE);
+ dlen = xbzrle_encode_buffer(test, test, XBZRLE_PAGE_SIZE,
+ compressed, XBZRLE_PAGE_SIZE);
g_assert(dlen == 0);
g_free(test);
test[XBZRLE_PAGE_SIZE - 1] = 1;
- dlen = xbzrle_encode_buffer_func(buffer, test, XBZRLE_PAGE_SIZE, compressed,
- XBZRLE_PAGE_SIZE);
+ dlen = xbzrle_encode_buffer(buffer, test, XBZRLE_PAGE_SIZE,
+ compressed, XBZRLE_PAGE_SIZE);
g_assert(dlen == (uleb128_encode_small(&buf[0], 4095) + 2));
rc = xbzrle_decode_buffer(compressed, dlen, buffer, XBZRLE_PAGE_SIZE);
}
/* encode overflow */
- rc = xbzrle_encode_buffer_func(buffer, test, XBZRLE_PAGE_SIZE, compressed,
- XBZRLE_PAGE_SIZE);
+ rc = xbzrle_encode_buffer(buffer, test, XBZRLE_PAGE_SIZE,
+ compressed, XBZRLE_PAGE_SIZE);
g_assert(rc == -1);
g_free(buffer);
test[1000 + diff_len + 5] = 109;
/* test encode/decode */
- dlen = xbzrle_encode_buffer_func(test, buffer, XBZRLE_PAGE_SIZE, compressed,
- XBZRLE_PAGE_SIZE);
+ dlen = xbzrle_encode_buffer(test, buffer, XBZRLE_PAGE_SIZE,
+ compressed, XBZRLE_PAGE_SIZE);
rc = xbzrle_decode_buffer(compressed, dlen, test, XBZRLE_PAGE_SIZE);
g_assert(rc < XBZRLE_PAGE_SIZE);
#include "qemu/osdep.h"
#include "qemu/cutils.h"
#include "qemu/bswap.h"
+#include "host/cpuinfo.h"
static bool
buffer_zero_int(const void *buf, size_t len)
}
#endif /* CONFIG_AVX512F_OPT */
-
-/* Note that for test_buffer_is_zero_next_accel, the most preferred
- * ISA must have the least significant bit.
- */
-#define CACHE_AVX512F 1
-#define CACHE_AVX2 2
-#define CACHE_SSE4 4
-#define CACHE_SSE2 8
-
-/* Make sure that these variables are appropriately initialized when
+/*
+ * Make sure that these variables are appropriately initialized when
* SSE2 is enabled on the compiler command-line, but the compiler is
* too old to support CONFIG_AVX2_OPT.
*/
#if defined(CONFIG_AVX512F_OPT) || defined(CONFIG_AVX2_OPT)
-# define INIT_CACHE 0
-# define INIT_ACCEL buffer_zero_int
+# define INIT_USED 0
+# define INIT_LENGTH 0
+# define INIT_ACCEL buffer_zero_int
#else
# ifndef __SSE2__
# error "ISA selection confusion"
# endif
-# define INIT_CACHE CACHE_SSE2
-# define INIT_ACCEL buffer_zero_sse2
+# define INIT_USED CPUINFO_SSE2
+# define INIT_LENGTH 64
+# define INIT_ACCEL buffer_zero_sse2
#endif
-static unsigned cpuid_cache = INIT_CACHE;
+static unsigned used_accel = INIT_USED;
+static unsigned length_to_accel = INIT_LENGTH;
static bool (*buffer_accel)(const void *, size_t) = INIT_ACCEL;
-static int length_to_accel = 64;
-static void init_accel(unsigned cache)
+static unsigned __attribute__((noinline))
+select_accel_cpuinfo(unsigned info)
{
- bool (*fn)(const void *, size_t) = buffer_zero_int;
- if (cache & CACHE_SSE2) {
- fn = buffer_zero_sse2;
- length_to_accel = 64;
- }
+ /* Array is sorted in order of algorithm preference. */
+ static const struct {
+ unsigned bit;
+ unsigned len;
+ bool (*fn)(const void *, size_t);
+ } all[] = {
+#ifdef CONFIG_AVX512F_OPT
+ { CPUINFO_AVX512F, 256, buffer_zero_avx512 },
+#endif
#ifdef CONFIG_AVX2_OPT
- if (cache & CACHE_SSE4) {
- fn = buffer_zero_sse4;
- length_to_accel = 64;
- }
- if (cache & CACHE_AVX2) {
- fn = buffer_zero_avx2;
- length_to_accel = 128;
- }
+ { CPUINFO_AVX2, 128, buffer_zero_avx2 },
+ { CPUINFO_SSE4, 64, buffer_zero_sse4 },
#endif
-#ifdef CONFIG_AVX512F_OPT
- if (cache & CACHE_AVX512F) {
- fn = buffer_zero_avx512;
- length_to_accel = 256;
+ { CPUINFO_SSE2, 64, buffer_zero_sse2 },
+ { CPUINFO_ALWAYS, 0, buffer_zero_int },
+ };
+
+ for (unsigned i = 0; i < ARRAY_SIZE(all); ++i) {
+ if (info & all[i].bit) {
+ length_to_accel = all[i].len;
+ buffer_accel = all[i].fn;
+ return all[i].bit;
+ }
}
-#endif
- buffer_accel = fn;
+ return 0;
}
#if defined(CONFIG_AVX512F_OPT) || defined(CONFIG_AVX2_OPT)
-#include "qemu/cpuid.h"
-
-static void __attribute__((constructor)) init_cpuid_cache(void)
+static void __attribute__((constructor)) init_accel(void)
{
- unsigned max = __get_cpuid_max(0, NULL);
- int a, b, c, d;
- unsigned cache = 0;
-
- if (max >= 1) {
- __cpuid(1, a, b, c, d);
- if (d & bit_SSE2) {
- cache |= CACHE_SSE2;
- }
- if (c & bit_SSE4_1) {
- cache |= CACHE_SSE4;
- }
-
- /* We must check that AVX is not just available, but usable. */
- if ((c & bit_OSXSAVE) && (c & bit_AVX) && max >= 7) {
- unsigned bv = xgetbv_low(0);
- __cpuid_count(7, 0, a, b, c, d);
- if ((bv & 0x6) == 0x6 && (b & bit_AVX2)) {
- cache |= CACHE_AVX2;
- }
- /* 0xe6:
- * XCR0[7:5] = 111b (OPMASK state, upper 256-bit of ZMM0-ZMM15
- * and ZMM16-ZMM31 state are enabled by OS)
- * XCR0[2:1] = 11b (XMM state and YMM state are enabled by OS)
- */
- if ((bv & 0xe6) == 0xe6 && (b & bit_AVX512F)) {
- cache |= CACHE_AVX512F;
- }
- }
- }
- cpuid_cache = cache;
- init_accel(cache);
+ used_accel = select_accel_cpuinfo(cpuinfo_init());
}
#endif /* CONFIG_AVX2_OPT */
bool test_buffer_is_zero_next_accel(void)
{
- /* If no bits set, we just tested buffer_zero_int, and there
- are no more acceleration options to test. */
- if (cpuid_cache == 0) {
- return false;
- }
- /* Disable the accelerator we used before and select a new one. */
- cpuid_cache &= cpuid_cache - 1;
- init_accel(cpuid_cache);
- return true;
+ /*
+ * Accumulate the accelerators that we've already tested, and
+ * remove them from the set to test this round. We'll get back
+ * a zero from select_accel_cpuinfo when there are no more.
+ */
+ unsigned used = select_accel_cpuinfo(cpuinfo & ~used_accel);
+ used_accel |= used;
+ return used;
}
static bool select_accel_fn(const void *buf, size_t len)
--- /dev/null
+/*
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ * Host specific cpu indentification for AArch64.
+ */
+
+#include "qemu/osdep.h"
+#include "host/cpuinfo.h"
+
+#ifdef CONFIG_LINUX
+# ifdef CONFIG_GETAUXVAL
+# include <sys/auxv.h>
+# else
+# include <asm/hwcap.h>
+# include "elf.h"
+# endif
+#endif
+#ifdef CONFIG_DARWIN
+# include <sys/sysctl.h>
+#endif
+
+unsigned cpuinfo;
+
+#ifdef CONFIG_DARWIN
+static bool sysctl_for_bool(const char *name)
+{
+ int val = 0;
+ size_t len = sizeof(val);
+
+ if (sysctlbyname(name, &val, &len, NULL, 0) == 0) {
+ return val != 0;
+ }
+
+ /*
+ * We might in the future ask for properties not present in older kernels,
+ * but we're only asking about static properties, all of which should be
+ * 'int'. So we shouln't see ENOMEM (val too small), or any of the other
+ * more exotic errors.
+ */
+ assert(errno == ENOENT);
+ return false;
+}
+#endif
+
+/* Called both as constructor and (possibly) via other constructors. */
+unsigned __attribute__((constructor)) cpuinfo_init(void)
+{
+ unsigned info = cpuinfo;
+
+ if (info) {
+ return info;
+ }
+
+ info = CPUINFO_ALWAYS;
+
+#ifdef CONFIG_LINUX
+ unsigned long hwcap = qemu_getauxval(AT_HWCAP);
+ info |= (hwcap & HWCAP_ATOMICS ? CPUINFO_LSE : 0);
+ info |= (hwcap & HWCAP_USCAT ? CPUINFO_LSE2 : 0);
+#endif
+#ifdef CONFIG_DARWIN
+ info |= sysctl_for_bool("hw.optional.arm.FEAT_LSE") * CPUINFO_LSE;
+ info |= sysctl_for_bool("hw.optional.arm.FEAT_LSE2") * CPUINFO_LSE2;
+#endif
+
+ cpuinfo = info;
+ return info;
+}
--- /dev/null
+/*
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ * Host specific cpu indentification for x86.
+ */
+
+#include "qemu/osdep.h"
+#include "host/cpuinfo.h"
+#ifdef CONFIG_CPUID_H
+# include "qemu/cpuid.h"
+#endif
+
+unsigned cpuinfo;
+
+/* Called both as constructor and (possibly) via other constructors. */
+unsigned __attribute__((constructor)) cpuinfo_init(void)
+{
+ unsigned info = cpuinfo;
+
+ if (info) {
+ return info;
+ }
+
+#ifdef CONFIG_CPUID_H
+ unsigned max, a, b, c, d, b7 = 0, c7 = 0;
+
+ max = __get_cpuid_max(0, 0);
+
+ if (max >= 7) {
+ __cpuid_count(7, 0, a, b7, c7, d);
+ info |= (b7 & bit_BMI ? CPUINFO_BMI1 : 0);
+ info |= (b7 & bit_BMI2 ? CPUINFO_BMI2 : 0);
+ }
+
+ if (max >= 1) {
+ __cpuid(1, a, b, c, d);
+
+ info |= (d & bit_CMOV ? CPUINFO_CMOV : 0);
+ info |= (d & bit_SSE2 ? CPUINFO_SSE2 : 0);
+ info |= (c & bit_SSE4_1 ? CPUINFO_SSE4 : 0);
+ info |= (c & bit_MOVBE ? CPUINFO_MOVBE : 0);
+ info |= (c & bit_POPCNT ? CPUINFO_POPCNT : 0);
+
+ /* For AVX features, we must check available and usable. */
+ if ((c & bit_AVX) && (c & bit_OSXSAVE)) {
+ unsigned bv = xgetbv_low(0);
+
+ if ((bv & 6) == 6) {
+ info |= CPUINFO_AVX1;
+ info |= (b7 & bit_AVX2 ? CPUINFO_AVX2 : 0);
+
+ if ((bv & 0xe0) == 0xe0) {
+ info |= (b7 & bit_AVX512F ? CPUINFO_AVX512F : 0);
+ info |= (b7 & bit_AVX512VL ? CPUINFO_AVX512VL : 0);
+ info |= (b7 & bit_AVX512BW ? CPUINFO_AVX512BW : 0);
+ info |= (b7 & bit_AVX512DQ ? CPUINFO_AVX512DQ : 0);
+ info |= (c7 & bit_AVX512VBMI2 ? CPUINFO_AVX512VBMI2 : 0);
+ }
+
+ /*
+ * The Intel SDM has added:
+ * Processors that enumerate support for Intel® AVX
+ * (by setting the feature flag CPUID.01H:ECX.AVX[bit 28])
+ * guarantee that the 16-byte memory operations performed
+ * by the following instructions will always be carried
+ * out atomically:
+ * - MOVAPD, MOVAPS, and MOVDQA.
+ * - VMOVAPD, VMOVAPS, and VMOVDQA when encoded with VEX.128.
+ * - VMOVAPD, VMOVAPS, VMOVDQA32, and VMOVDQA64 when encoded
+ * with EVEX.128 and k0 (masking disabled).
+ * Note that these instructions require the linear addresses
+ * of their memory operands to be 16-byte aligned.
+ *
+ * AMD has provided an even stronger guarantee that processors
+ * with AVX provide 16-byte atomicity for all cachable,
+ * naturally aligned single loads and stores, e.g. MOVDQU.
+ *
+ * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104688
+ */
+ __cpuid(0, a, b, c, d);
+ if (c == signature_INTEL_ecx) {
+ info |= CPUINFO_ATOMIC_VMOVDQA;
+ } else if (c == signature_AMD_ecx) {
+ info |= CPUINFO_ATOMIC_VMOVDQA | CPUINFO_ATOMIC_VMOVDQU;
+ }
+ }
+ }
+ }
+
+ max = __get_cpuid_max(0x8000000, 0);
+ if (max >= 1) {
+ __cpuid(0x80000001, a, b, c, d);
+ info |= (c & bit_LZCNT ? CPUINFO_LZCNT : 0);
+ }
+#endif
+
+ info |= CPUINFO_ALWAYS;
+ cpuinfo = info;
+ return info;
+}
return crc^0xffffffff;
}
+uint32_t iov_crc32c(uint32_t crc, const struct iovec *iov, size_t iov_cnt)
+{
+ while (iov_cnt--) {
+ crc = crc32c(crc, iov->iov_base, iov->iov_len) ^ 0xffffffff;
+ iov++;
+ }
+ return crc ^ 0xffffffff;
+}
endif
util_ss.add(when: 'CONFIG_LINUX', if_true: files('vfio-helpers.c'))
endif
+
+if cpu == 'aarch64'
+ util_ss.add(files('cpuinfo-aarch64.c'))
+elif cpu in ['x86', 'x86_64']
+ util_ss.add(files('cpuinfo-i386.c'))
+endif
*/
static char *sysfs_find_group_file(const char *device, Error **errp)
{
+ g_autoptr(GError) gerr = NULL;
char *sysfs_link;
char *sysfs_group;
char *p;
char *path = NULL;
sysfs_link = g_strdup_printf("/sys/bus/pci/devices/%s/iommu_group", device);
- sysfs_group = g_malloc0(PATH_MAX);
- if (readlink(sysfs_link, sysfs_group, PATH_MAX - 1) == -1) {
- error_setg_errno(errp, errno, "Failed to find iommu group sysfs path");
+ sysfs_group = g_file_read_link(sysfs_link, &gerr);
+ if (gerr) {
+ error_setg(errp, "Failed to find iommu group sysfs path: %s",
+ gerr->message);
goto out;
}
p = strrchr(sysfs_group, '/');
projects / mirror_qemu.git / commitdiff