[mirror_qemu.git] / target / s390x / vec_fpu_helper.c

/*
 * QEMU TCG support -- s390x vector floating point instruction support
 *
 * Copyright (C) 2019 Red Hat Inc
 *
 * Authors:
 *   David Hildenbrand <david@redhat.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2 or later.
 * See the COPYING file in the top-level directory.
 */
#include "qemu/osdep.h"
#include "qemu-common.h"
#include "cpu.h"
#include "internal.h"
#include "vec.h"
#include "tcg_s390x.h"
#include "tcg/tcg-gvec-desc.h"
#include "exec/exec-all.h"
#include "exec/helper-proto.h"
#include "fpu/softfloat.h"

#define VIC_INVALID         0x1
#define VIC_DIVBYZERO       0x2
#define VIC_OVERFLOW        0x3
#define VIC_UNDERFLOW       0x4
#define VIC_INEXACT         0x5

/* returns the VEX. If the VEX is 0, there is no trap */
static uint8_t check_ieee_exc(CPUS390XState *env, uint8_t enr, bool XxC,
                              uint8_t *vec_exc)
{
    uint8_t vece_exc = 0, trap_exc;
    unsigned qemu_exc;

    /* Retrieve and clear the softfloat exceptions */
    qemu_exc = env->fpu_status.float_exception_flags;
    if (qemu_exc == 0) {
        return 0;
    }
    env->fpu_status.float_exception_flags = 0;

    vece_exc = s390_softfloat_exc_to_ieee(qemu_exc);

    /* Add them to the vector-wide s390x exception bits */
    *vec_exc |= vece_exc;

    /* Check for traps and construct the VXC */
    trap_exc = vece_exc & env->fpc >> 24;
    if (trap_exc) {
        if (trap_exc & S390_IEEE_MASK_INVALID) {
            return enr << 4 | VIC_INVALID;
        } else if (trap_exc & S390_IEEE_MASK_DIVBYZERO) {
            return enr << 4 | VIC_DIVBYZERO;
        } else if (trap_exc & S390_IEEE_MASK_OVERFLOW) {
            return enr << 4 | VIC_OVERFLOW;
        } else if (trap_exc & S390_IEEE_MASK_UNDERFLOW) {
            return enr << 4 | VIC_UNDERFLOW;
        } else if (!XxC) {
            g_assert(trap_exc & S390_IEEE_MASK_INEXACT);
            /* inexact has lowest priority on traps */
            return enr << 4 | VIC_INEXACT;
        }
    }
    return 0;
}

static void handle_ieee_exc(CPUS390XState *env, uint8_t vxc, uint8_t vec_exc,
                            uintptr_t retaddr)
{
    if (vxc) {
        /* on traps, the fpc flags are not updated, instruction is suppressed */
        tcg_s390_vector_exception(env, vxc, retaddr);
    }
    if (vec_exc) {
        /* indicate exceptions for all elements combined */
        env->fpc |= vec_exc << 16;
    }
}

static float32 s390_vec_read_float32(const S390Vector *v, uint8_t enr)
{
    return make_float32(s390_vec_read_element32(v, enr));
}

static float64 s390_vec_read_float64(const S390Vector *v, uint8_t enr)
{
    return make_float64(s390_vec_read_element64(v, enr));
}

static float128 s390_vec_read_float128(const S390Vector *v)
{
    return make_float128(s390_vec_read_element64(v, 0),
                         s390_vec_read_element64(v, 1));
}

static void s390_vec_write_float32(S390Vector *v, uint8_t enr, float32 data)
{
    return s390_vec_write_element32(v, enr, data);
}

static void s390_vec_write_float64(S390Vector *v, uint8_t enr, float64 data)
{
    return s390_vec_write_element64(v, enr, data);
}

static void s390_vec_write_float128(S390Vector *v, float128 data)
{
    s390_vec_write_element64(v, 0, data.high);
    s390_vec_write_element64(v, 1, data.low);
}

typedef float32 (*vop32_2_fn)(float32 a, float_status *s);
static void vop32_2(S390Vector *v1, const S390Vector *v2, CPUS390XState *env,
                    bool s, bool XxC, uint8_t erm, vop32_2_fn fn,
                    uintptr_t retaddr)
{
    uint8_t vxc, vec_exc = 0;
    S390Vector tmp = {};
    int i, old_mode;

    old_mode = s390_swap_bfp_rounding_mode(env, erm);
    for (i = 0; i < 4; i++) {
        const float32 a = s390_vec_read_float32(v2, i);

        s390_vec_write_float32(&tmp, i, fn(a, &env->fpu_status));
        vxc = check_ieee_exc(env, i, XxC, &vec_exc);
        if (s || vxc) {
            break;
        }
    }
    s390_restore_bfp_rounding_mode(env, old_mode);
    handle_ieee_exc(env, vxc, vec_exc, retaddr);
    *v1 = tmp;
}

typedef float64 (*vop64_2_fn)(float64 a, float_status *s);
static void vop64_2(S390Vector *v1, const S390Vector *v2, CPUS390XState *env,
                    bool s, bool XxC, uint8_t erm, vop64_2_fn fn,
                    uintptr_t retaddr)
{
    uint8_t vxc, vec_exc = 0;
    S390Vector tmp = {};
    int i, old_mode;

    old_mode = s390_swap_bfp_rounding_mode(env, erm);
    for (i = 0; i < 2; i++) {
        const float64 a = s390_vec_read_float64(v2, i);

        s390_vec_write_float64(&tmp, i, fn(a, &env->fpu_status));
        vxc = check_ieee_exc(env, i, XxC, &vec_exc);
        if (s || vxc) {
            break;
        }
    }
    s390_restore_bfp_rounding_mode(env, old_mode);
    handle_ieee_exc(env, vxc, vec_exc, retaddr);
    *v1 = tmp;
}

typedef float128 (*vop128_2_fn)(float128 a, float_status *s);
static void vop128_2(S390Vector *v1, const S390Vector *v2, CPUS390XState *env,
                    bool s, bool XxC, uint8_t erm, vop128_2_fn fn,
                    uintptr_t retaddr)
{
    const float128 a = s390_vec_read_float128(v2);
    uint8_t vxc, vec_exc = 0;
    S390Vector tmp = {};
    int old_mode;

    old_mode = s390_swap_bfp_rounding_mode(env, erm);
    s390_vec_write_float128(&tmp, fn(a, &env->fpu_status));
    vxc = check_ieee_exc(env, 0, XxC, &vec_exc);
    s390_restore_bfp_rounding_mode(env, old_mode);
    handle_ieee_exc(env, vxc, vec_exc, retaddr);
    *v1 = tmp;
}

static float64 vcdg64(float64 a, float_status *s)
{
    return int64_to_float64(a, s);
}

static float64 vcdlg64(float64 a, float_status *s)
{
    return uint64_to_float64(a, s);
}

static float64 vcgd64(float64 a, float_status *s)
{
    const float64 tmp = float64_to_int64(a, s);

    return float64_is_any_nan(a) ? INT64_MIN : tmp;
}

static float64 vclgd64(float64 a, float_status *s)
{
    const float64 tmp = float64_to_uint64(a, s);

    return float64_is_any_nan(a) ? 0 : tmp;
}

#define DEF_GVEC_VOP2_FN(NAME, FN, BITS)                                       \
void HELPER(gvec_##NAME##BITS)(void *v1, const void *v2, CPUS390XState *env,   \
                               uint32_t desc)                                  \
{                                                                              \
    const uint8_t erm = extract32(simd_data(desc), 4, 4);                      \
    const bool se = extract32(simd_data(desc), 3, 1);                          \
    const bool XxC = extract32(simd_data(desc), 2, 1);                         \
                                                                               \
    vop##BITS##_2(v1, v2, env, se, XxC, erm, FN, GETPC());                     \
}

#define DEF_GVEC_VOP2_64(NAME)                                                 \
DEF_GVEC_VOP2_FN(NAME, NAME##64, 64)

#define DEF_GVEC_VOP2(NAME, OP)                                                \
DEF_GVEC_VOP2_FN(NAME, float32_##OP, 32)                                       \
DEF_GVEC_VOP2_FN(NAME, float64_##OP, 64)                                       \
DEF_GVEC_VOP2_FN(NAME, float128_##OP, 128)

DEF_GVEC_VOP2_64(vcdg)
DEF_GVEC_VOP2_64(vcdlg)
DEF_GVEC_VOP2_64(vcgd)
DEF_GVEC_VOP2_64(vclgd)
DEF_GVEC_VOP2(vfi, round_to_int)
DEF_GVEC_VOP2(vfsq, sqrt)

typedef float32 (*vop32_3_fn)(float32 a, float32 b, float_status *s);
static void vop32_3(S390Vector *v1, const S390Vector *v2, const S390Vector *v3,
                    CPUS390XState *env, bool s, vop32_3_fn fn,
                    uintptr_t retaddr)
{
    uint8_t vxc, vec_exc = 0;
    S390Vector tmp = {};
    int i;

    for (i = 0; i < 4; i++) {
        const float32 a = s390_vec_read_float32(v2, i);
        const float32 b = s390_vec_read_float32(v3, i);

        s390_vec_write_float32(&tmp, i, fn(a, b, &env->fpu_status));
        vxc = check_ieee_exc(env, i, false, &vec_exc);
        if (s || vxc) {
            break;
        }
    }
    handle_ieee_exc(env, vxc, vec_exc, retaddr);
    *v1 = tmp;
}

typedef float64 (*vop64_3_fn)(float64 a, float64 b, float_status *s);
static void vop64_3(S390Vector *v1, const S390Vector *v2, const S390Vector *v3,
                    CPUS390XState *env, bool s, vop64_3_fn fn,
                    uintptr_t retaddr)
{
    uint8_t vxc, vec_exc = 0;
    S390Vector tmp = {};
    int i;

    for (i = 0; i < 2; i++) {
        const float64 a = s390_vec_read_float64(v2, i);
        const float64 b = s390_vec_read_float64(v3, i);

        s390_vec_write_float64(&tmp, i, fn(a, b, &env->fpu_status));
        vxc = check_ieee_exc(env, i, false, &vec_exc);
        if (s || vxc) {
            break;
        }
    }
    handle_ieee_exc(env, vxc, vec_exc, retaddr);
    *v1 = tmp;
}

typedef float128 (*vop128_3_fn)(float128 a, float128 b, float_status *s);
static void vop128_3(S390Vector *v1, const S390Vector *v2, const S390Vector *v3,
                     CPUS390XState *env, bool s, vop128_3_fn fn,
                     uintptr_t retaddr)
{
    const float128 a = s390_vec_read_float128(v2);
    const float128 b = s390_vec_read_float128(v3);
    uint8_t vxc, vec_exc = 0;
    S390Vector tmp = {};

    s390_vec_write_float128(&tmp, fn(a, b, &env->fpu_status));
    vxc = check_ieee_exc(env, 0, false, &vec_exc);
    handle_ieee_exc(env, vxc, vec_exc, retaddr);
    *v1 = tmp;
}

#define DEF_GVEC_VOP3_B(NAME, OP, BITS)                                        \
void HELPER(gvec_##NAME##BITS)(void *v1, const void *v2, const void *v3,       \
                              CPUS390XState *env, uint32_t desc)               \
{                                                                              \
    const bool se = extract32(simd_data(desc), 3, 1);                          \
                                                                               \
    vop##BITS##_3(v1, v2, v3, env, se, float##BITS##_##OP, GETPC());           \
}

#define DEF_GVEC_VOP3(NAME, OP)                                                \
DEF_GVEC_VOP3_B(NAME, OP, 32)                                                  \
DEF_GVEC_VOP3_B(NAME, OP, 64)                                                  \
DEF_GVEC_VOP3_B(NAME, OP, 128)

DEF_GVEC_VOP3(vfa, add)
DEF_GVEC_VOP3(vfs, sub)
DEF_GVEC_VOP3(vfd, div)
DEF_GVEC_VOP3(vfm, mul)

static int wfc32(const S390Vector *v1, const S390Vector *v2,
                 CPUS390XState *env, bool signal, uintptr_t retaddr)
{
    /* only the zero-indexed elements are compared */
    const float32 a = s390_vec_read_float32(v1, 0);
    const float32 b = s390_vec_read_float32(v2, 0);
    uint8_t vxc, vec_exc = 0;
    int cmp;

    if (signal) {
        cmp = float32_compare(a, b, &env->fpu_status);
    } else {
        cmp = float32_compare_quiet(a, b, &env->fpu_status);
    }
    vxc = check_ieee_exc(env, 0, false, &vec_exc);
    handle_ieee_exc(env, vxc, vec_exc, retaddr);

    return float_comp_to_cc(env, cmp);
}

static int wfc64(const S390Vector *v1, const S390Vector *v2,
                 CPUS390XState *env, bool signal, uintptr_t retaddr)
{
    /* only the zero-indexed elements are compared */
    const float64 a = s390_vec_read_float64(v1, 0);
    const float64 b = s390_vec_read_float64(v2, 0);
    uint8_t vxc, vec_exc = 0;
    int cmp;

    if (signal) {
        cmp = float64_compare(a, b, &env->fpu_status);
    } else {
        cmp = float64_compare_quiet(a, b, &env->fpu_status);
    }
    vxc = check_ieee_exc(env, 0, false, &vec_exc);
    handle_ieee_exc(env, vxc, vec_exc, retaddr);

    return float_comp_to_cc(env, cmp);
}

static int wfc128(const S390Vector *v1, const S390Vector *v2,
                  CPUS390XState *env, bool signal, uintptr_t retaddr)
{
    /* only the zero-indexed elements are compared */
    const float128 a = s390_vec_read_float128(v1);
    const float128 b = s390_vec_read_float128(v2);
    uint8_t vxc, vec_exc = 0;
    int cmp;

    if (signal) {
        cmp = float128_compare(a, b, &env->fpu_status);
    } else {
        cmp = float128_compare_quiet(a, b, &env->fpu_status);
    }
    vxc = check_ieee_exc(env, 0, false, &vec_exc);
    handle_ieee_exc(env, vxc, vec_exc, retaddr);

    return float_comp_to_cc(env, cmp);
}

#define DEF_GVEC_WFC_B(NAME, SIGNAL, BITS)                                     \
void HELPER(gvec_##NAME##BITS)(const void *v1, const void *v2,                 \
                               CPUS390XState *env, uint32_t desc)              \
{                                                                              \
    env->cc_op = wfc##BITS(v1, v2, env, SIGNAL, GETPC());                      \
}

#define DEF_GVEC_WFC(NAME, SIGNAL)                                             \
     DEF_GVEC_WFC_B(NAME, SIGNAL, 32)                                          \
     DEF_GVEC_WFC_B(NAME, SIGNAL, 64)                                          \
     DEF_GVEC_WFC_B(NAME, SIGNAL, 128)

DEF_GVEC_WFC(wfc, false)
DEF_GVEC_WFC(wfk, true)

typedef bool (*vfc32_fn)(float32 a, float32 b, float_status *status);
static int vfc32(S390Vector *v1, const S390Vector *v2, const S390Vector *v3,
                 CPUS390XState *env, bool s, vfc32_fn fn, uintptr_t retaddr)
{
    uint8_t vxc, vec_exc = 0;
    S390Vector tmp = {};
    int match = 0;
    int i;

    for (i = 0; i < 4; i++) {
        const float32 a = s390_vec_read_float32(v2, i);
        const float32 b = s390_vec_read_float32(v3, i);

        /* swap the order of the parameters, so we can use existing functions */
        if (fn(b, a, &env->fpu_status)) {
            match++;
            s390_vec_write_element32(&tmp, i, -1u);
        }
        vxc = check_ieee_exc(env, i, false, &vec_exc);
        if (s || vxc) {
            break;
        }
    }

    handle_ieee_exc(env, vxc, vec_exc, retaddr);
    *v1 = tmp;
    if (match) {
        return s || match == 4 ? 0 : 1;
    }
    return 3;
}

typedef bool (*vfc64_fn)(float64 a, float64 b, float_status *status);
static int vfc64(S390Vector *v1, const S390Vector *v2, const S390Vector *v3,
                 CPUS390XState *env, bool s, vfc64_fn fn, uintptr_t retaddr)
{
    uint8_t vxc, vec_exc = 0;
    S390Vector tmp = {};
    int match = 0;
    int i;

    for (i = 0; i < 2; i++) {
        const float64 a = s390_vec_read_float64(v2, i);
        const float64 b = s390_vec_read_float64(v3, i);

        /* swap the order of the parameters, so we can use existing functions */
        if (fn(b, a, &env->fpu_status)) {
            match++;
            s390_vec_write_element64(&tmp, i, -1ull);
        }
        vxc = check_ieee_exc(env, i, false, &vec_exc);
        if (s || vxc) {
            break;
        }
    }

    handle_ieee_exc(env, vxc, vec_exc, retaddr);
    *v1 = tmp;
    if (match) {
        return s || match == 2 ? 0 : 1;
    }
    return 3;
}

typedef bool (*vfc128_fn)(float128 a, float128 b, float_status *status);
static int vfc128(S390Vector *v1, const S390Vector *v2, const S390Vector *v3,
                 CPUS390XState *env, bool s, vfc128_fn fn, uintptr_t retaddr)
{
    const float128 a = s390_vec_read_float128(v2);
    const float128 b = s390_vec_read_float128(v3);
    uint8_t vxc, vec_exc = 0;
    S390Vector tmp = {};
    bool match = false;

    /* swap the order of the parameters, so we can use existing functions */
    if (fn(b, a, &env->fpu_status)) {
        match = true;
        s390_vec_write_element64(&tmp, 0, -1ull);
        s390_vec_write_element64(&tmp, 1, -1ull);
    }
    vxc = check_ieee_exc(env, 0, false, &vec_exc);
    handle_ieee_exc(env, vxc, vec_exc, retaddr);
    *v1 = tmp;
    return match ? 0 : 3;
}

#define DEF_GVEC_VFC_B(NAME, OP, BITS)                                         \
void HELPER(gvec_##NAME##BITS)(void *v1, const void *v2, const void *v3,       \
                               CPUS390XState *env, uint32_t desc)              \
{                                                                              \
    const bool se = extract32(simd_data(desc), 3, 1);                          \
    const bool sq = extract32(simd_data(desc), 2, 1);                          \
    vfc##BITS##_fn fn = sq ? float##BITS##_##OP : float##BITS##_##OP##_quiet;  \
                                                                               \
    vfc##BITS(v1, v2, v3, env, se, fn, GETPC());                               \
}                                                                              \
                                                                               \
void HELPER(gvec_##NAME##BITS##_cc)(void *v1, const void *v2, const void *v3,  \
                                    CPUS390XState *env, uint32_t desc)         \
{                                                                              \
    const bool se = extract32(simd_data(desc), 3, 1);                          \
    const bool sq = extract32(simd_data(desc), 2, 1);                          \
    vfc##BITS##_fn fn = sq ? float##BITS##_##OP : float##BITS##_##OP##_quiet;  \
                                                                               \
    env->cc_op = vfc##BITS(v1, v2, v3, env, se, fn, GETPC());                  \
}

#define DEF_GVEC_VFC(NAME, OP)                                                 \
DEF_GVEC_VFC_B(NAME, OP, 32)                                                   \
DEF_GVEC_VFC_B(NAME, OP, 64)                                                   \
DEF_GVEC_VFC_B(NAME, OP, 128)                                                  \

DEF_GVEC_VFC(vfce, eq)
DEF_GVEC_VFC(vfch, lt)
DEF_GVEC_VFC(vfche, le)

void HELPER(gvec_vfll32)(void *v1, const void *v2, CPUS390XState *env,
                         uint32_t desc)
{
    const bool s = extract32(simd_data(desc), 3, 1);
    uint8_t vxc, vec_exc = 0;
    S390Vector tmp = {};
    int i;

    for (i = 0; i < 2; i++) {
        /* load from even element */
        const float32 a = s390_vec_read_element32(v2, i * 2);
        const uint64_t ret = float32_to_float64(a, &env->fpu_status);

        s390_vec_write_element64(&tmp, i, ret);
        /* indicate the source element */
        vxc = check_ieee_exc(env, i * 2, false, &vec_exc);
        if (s || vxc) {
            break;
        }
    }
    handle_ieee_exc(env, vxc, vec_exc, GETPC());
    *(S390Vector *)v1 = tmp;
}

void HELPER(gvec_vfll64)(void *v1, const void *v2, CPUS390XState *env,
                         uint32_t desc)
{
    /* load from even element */
    const float128 ret = float64_to_float128(s390_vec_read_float64(v2, 0),
                                             &env->fpu_status);
    uint8_t vxc, vec_exc = 0;

    vxc = check_ieee_exc(env, 0, false, &vec_exc);
    handle_ieee_exc(env, vxc, vec_exc, GETPC());
    s390_vec_write_float128(v1, ret);
}

void HELPER(gvec_vflr64)(void *v1, const void *v2, CPUS390XState *env,
                         uint32_t desc)
{
    const uint8_t erm = extract32(simd_data(desc), 4, 4);
    const bool s = extract32(simd_data(desc), 3, 1);
    const bool XxC = extract32(simd_data(desc), 2, 1);
    uint8_t vxc, vec_exc = 0;
    S390Vector tmp = {};
    int i, old_mode;

    old_mode = s390_swap_bfp_rounding_mode(env, erm);
    for (i = 0; i < 2; i++) {
        float64 a = s390_vec_read_element64(v2, i);
        uint32_t ret = float64_to_float32(a, &env->fpu_status);

        /* place at even element */
        s390_vec_write_element32(&tmp, i * 2, ret);
        /* indicate the source element */
        vxc = check_ieee_exc(env, i, XxC, &vec_exc);
        if (s || vxc) {
            break;
        }
    }
    s390_restore_bfp_rounding_mode(env, old_mode);
    handle_ieee_exc(env, vxc, vec_exc, GETPC());
    *(S390Vector *)v1 = tmp;
}

void HELPER(gvec_vflr128)(void *v1, const void *v2, CPUS390XState *env,
                          uint32_t desc)
{
    const uint8_t erm = extract32(simd_data(desc), 4, 4);
    const bool XxC = extract32(simd_data(desc), 2, 1);
    uint8_t vxc, vec_exc = 0;
    int old_mode;
    float64 ret;

    old_mode = s390_swap_bfp_rounding_mode(env, erm);
    ret = float128_to_float64(s390_vec_read_float128(v2), &env->fpu_status);
    vxc = check_ieee_exc(env, 0, XxC, &vec_exc);
    s390_restore_bfp_rounding_mode(env, old_mode);
    handle_ieee_exc(env, vxc, vec_exc, GETPC());

    /* place at even element, odd element is unpredictable */
    s390_vec_write_float64(v1, 0, ret);
}

static void vfma64(S390Vector *v1, const S390Vector *v2, const S390Vector *v3,
                   const S390Vector *v4, CPUS390XState *env, bool s, int flags,
                   uintptr_t retaddr)
{
    uint8_t vxc, vec_exc = 0;
    S390Vector tmp = {};
    int i;

    for (i = 0; i < 2; i++) {
        const float64 a = s390_vec_read_float64(v2, i);
        const float64 b = s390_vec_read_float64(v3, i);
        const float64 c = s390_vec_read_float64(v4, i);
        const float64 ret = float64_muladd(a, b, c, flags, &env->fpu_status);

        s390_vec_write_float64(&tmp, i, ret);
        vxc = check_ieee_exc(env, i, false, &vec_exc);
        if (s || vxc) {
            break;
        }
    }
    handle_ieee_exc(env, vxc, vec_exc, retaddr);
    *v1 = tmp;
}

#define DEF_GVEC_VFMA_B(NAME, FLAGS, BITS)                                     \
void HELPER(gvec_##NAME##BITS)(void *v1, const void *v2, const void *v3,       \
                               const void *v4, CPUS390XState *env,             \
                               uint32_t desc)                                  \
{                                                                              \
    const bool se = extract32(simd_data(desc), 3, 1);                          \
                                                                               \
    vfma##BITS(v1, v2, v3, v4, env, se, FLAGS, GETPC());                       \
}

#define DEF_GVEC_VFMA(NAME, FLAGS)                                             \
    DEF_GVEC_VFMA_B(NAME, FLAGS, 64)

DEF_GVEC_VFMA(vfma, 0)
DEF_GVEC_VFMA(vfms, float_muladd_negate_c)

void HELPER(gvec_vftci64)(void *v1, const void *v2, CPUS390XState *env,
                          uint32_t desc)
{
    const uint16_t i3 = extract32(simd_data(desc), 4, 12);
    const bool s = extract32(simd_data(desc), 3, 1);
    int i, match = 0;

    for (i = 0; i < 2; i++) {
        const float64 a = s390_vec_read_float64(v2, i);

        if (float64_dcmask(env, a) & i3) {
            match++;
            s390_vec_write_element64(v1, i, -1ull);
        } else {
            s390_vec_write_element64(v1, i, 0);
        }
        if (s) {
            break;
        }
    }

    if (match == 2 || (s && match)) {
        env->cc_op = 0;
    } else if (match) {
        env->cc_op = 1;
    } else {
        env->cc_op = 3;
    }
}
Commit	Line	Data
3a0eae85 DH	1	/*
	2	* QEMU TCG support -- s390x vector floating point instruction support
	3	*
	4	* Copyright (C) 2019 Red Hat Inc
	5	*
	6	* Authors:
	7	* David Hildenbrand <david@redhat.com>
	8	*
	9	* This work is licensed under the terms of the GNU GPL, version 2 or later.
	10	* See the COPYING file in the top-level directory.
	11	*/
	12	#include "qemu/osdep.h"
	13	#include "qemu-common.h"
	14	#include "cpu.h"
	15	#include "internal.h"
	16	#include "vec.h"
	17	#include "tcg_s390x.h"
	18	#include "tcg/tcg-gvec-desc.h"
	19	#include "exec/exec-all.h"
	20	#include "exec/helper-proto.h"
	21	#include "fpu/softfloat.h"
	22
	23	#define VIC_INVALID 0x1
	24	#define VIC_DIVBYZERO 0x2
	25	#define VIC_OVERFLOW 0x3
	26	#define VIC_UNDERFLOW 0x4
	27	#define VIC_INEXACT 0x5
	28
	29	/* returns the VEX. If the VEX is 0, there is no trap */
	30	static uint8_t check_ieee_exc(CPUS390XState *env, uint8_t enr, bool XxC,
	31	uint8_t *vec_exc)
	32	{
	33	uint8_t vece_exc = 0, trap_exc;
	34	unsigned qemu_exc;
	35
	36	/* Retrieve and clear the softfloat exceptions */
	37	qemu_exc = env->fpu_status.float_exception_flags;
	38	if (qemu_exc == 0) {
	39	return 0;
	40	}
	41	env->fpu_status.float_exception_flags = 0;
	42
	43	vece_exc = s390_softfloat_exc_to_ieee(qemu_exc);
	44
	45	/* Add them to the vector-wide s390x exception bits */
	46	*vec_exc \|= vece_exc;
	47
	48	/* Check for traps and construct the VXC */
	49	trap_exc = vece_exc & env->fpc >> 24;
	50	if (trap_exc) {
	51	if (trap_exc & S390_IEEE_MASK_INVALID) {
	52	return enr << 4 \| VIC_INVALID;
	53	} else if (trap_exc & S390_IEEE_MASK_DIVBYZERO) {
	54	return enr << 4 \| VIC_DIVBYZERO;
	55	} else if (trap_exc & S390_IEEE_MASK_OVERFLOW) {
	56	return enr << 4 \| VIC_OVERFLOW;
	57	} else if (trap_exc & S390_IEEE_MASK_UNDERFLOW) {
	58	return enr << 4 \| VIC_UNDERFLOW;
	59	} else if (!XxC) {
	60	g_assert(trap_exc & S390_IEEE_MASK_INEXACT);
	61	/* inexact has lowest priority on traps */
	62	return enr << 4 \| VIC_INEXACT;
	63	}
	64	}
65	return 0;
66	}
67
68	static void handle_ieee_exc(CPUS390XState *env, uint8_t vxc, uint8_t vec_exc,
69	uintptr_t retaddr)
70	{
71	if (vxc) {
72	/* on traps, the fpc flags are not updated, instruction is suppressed */
73	tcg_s390_vector_exception(env, vxc, retaddr);
74	}
75	if (vec_exc) {
76	/* indicate exceptions for all elements combined */
77	env->fpc \|= vec_exc << 16;
78	}
79	}
80
0987961d DH	81	static float32 s390_vec_read_float32(const S390Vector *v, uint8_t enr)
	82	{
	83	return make_float32(s390_vec_read_element32(v, enr));
	84	}
	85
863b9507 DH	86	static float64 s390_vec_read_float64(const S390Vector *v, uint8_t enr)
	87	{
	88	return make_float64(s390_vec_read_element64(v, enr));
	89	}
	90
0987961d DH	91	static float128 s390_vec_read_float128(const S390Vector *v)
	92	{
	93	return make_float128(s390_vec_read_element64(v, 0),
	94	s390_vec_read_element64(v, 1));
	95	}
	96
	97	static void s390_vec_write_float32(S390Vector *v, uint8_t enr, float32 data)
	98	{
	99	return s390_vec_write_element32(v, enr, data);
	100	}
	101
863b9507 DH	102	static void s390_vec_write_float64(S390Vector *v, uint8_t enr, float64 data)
	103	{
	104	return s390_vec_write_element64(v, enr, data);
	105	}
	106
0987961d DH	107	static void s390_vec_write_float128(S390Vector *v, float128 data)
	108	{
	109	s390_vec_write_element64(v, 0, data.high);
	110	s390_vec_write_element64(v, 1, data.low);
	111	}
	112
acb269a4 DH	113	typedef float32 (vop32_2_fn)(float32 a, float_status s);
	114	static void vop32_2(S390Vector v1, const S390Vector v2, CPUS390XState *env,
	115	bool s, bool XxC, uint8_t erm, vop32_2_fn fn,
	116	uintptr_t retaddr)
	117	{
	118	uint8_t vxc, vec_exc = 0;
	119	S390Vector tmp = {};
	120	int i, old_mode;
	121
	122	old_mode = s390_swap_bfp_rounding_mode(env, erm);
	123	for (i = 0; i < 4; i++) {
	124	const float32 a = s390_vec_read_float32(v2, i);
	125
	126	s390_vec_write_float32(&tmp, i, fn(a, &env->fpu_status));
	127	vxc = check_ieee_exc(env, i, XxC, &vec_exc);
	128	if (s \|\| vxc) {
	129	break;
	130	}
	131	}
	132	s390_restore_bfp_rounding_mode(env, old_mode);
	133	handle_ieee_exc(env, vxc, vec_exc, retaddr);
	134	*v1 = tmp;
	135	}
	136
21bd6ea2	137	typedef float64 (vop64_2_fn)(float64 a, float_status s);
bb03fd84 DH	138	static void vop64_2(S390Vector v1, const S390Vector v2, CPUS390XState *env,
	139	bool s, bool XxC, uint8_t erm, vop64_2_fn fn,
	140	uintptr_t retaddr)
	141	{
	142	uint8_t vxc, vec_exc = 0;
	143	S390Vector tmp = {};
	144	int i, old_mode;
	145
	146	old_mode = s390_swap_bfp_rounding_mode(env, erm);
	147	for (i = 0; i < 2; i++) {
21bd6ea2	148	const float64 a = s390_vec_read_float64(v2, i);
bb03fd84	149
21bd6ea2	150	s390_vec_write_float64(&tmp, i, fn(a, &env->fpu_status));
bb03fd84 DH	151	vxc = check_ieee_exc(env, i, XxC, &vec_exc);
	152	if (s \|\| vxc) {
	153	break;
	154	}
	155	}
	156	s390_restore_bfp_rounding_mode(env, old_mode);
	157	handle_ieee_exc(env, vxc, vec_exc, retaddr);
	158	*v1 = tmp;
	159	}
	160
acb269a4 DH	161	typedef float128 (vop128_2_fn)(float128 a, float_status s);
	162	static void vop128_2(S390Vector v1, const S390Vector v2, CPUS390XState *env,
	163	bool s, bool XxC, uint8_t erm, vop128_2_fn fn,
	164	uintptr_t retaddr)
	165	{
	166	const float128 a = s390_vec_read_float128(v2);
	167	uint8_t vxc, vec_exc = 0;
	168	S390Vector tmp = {};
	169	int old_mode;
	170
	171	old_mode = s390_swap_bfp_rounding_mode(env, erm);
	172	s390_vec_write_float128(&tmp, fn(a, &env->fpu_status));
	173	vxc = check_ieee_exc(env, 0, XxC, &vec_exc);
	174	s390_restore_bfp_rounding_mode(env, old_mode);
	175	handle_ieee_exc(env, vxc, vec_exc, retaddr);
	176	*v1 = tmp;
	177	}
	178
21bd6ea2 DH	179	static float64 vcdg64(float64 a, float_status *s)
	180	{
	181	return int64_to_float64(a, s);
	182	}
	183
	184	static float64 vcdlg64(float64 a, float_status *s)
	185	{
	186	return uint64_to_float64(a, s);
	187	}
	188
	189	static float64 vcgd64(float64 a, float_status *s)
	190	{
	191	const float64 tmp = float64_to_int64(a, s);
	192
	193	return float64_is_any_nan(a) ? INT64_MIN : tmp;
	194	}
	195
	196	static float64 vclgd64(float64 a, float_status *s)
	197	{
	198	const float64 tmp = float64_to_uint64(a, s);
	199
	200	return float64_is_any_nan(a) ? 0 : tmp;
	201	}
	202
	203	#define DEF_GVEC_VOP2_FN(NAME, FN, BITS) \
	204	void HELPER(gvec_##NAME##BITS)(void v1, const void v2, CPUS390XState *env, \
	205	uint32_t desc) \
	206	{ \
	207	const uint8_t erm = extract32(simd_data(desc), 4, 4); \
	208	const bool se = extract32(simd_data(desc), 3, 1); \
	209	const bool XxC = extract32(simd_data(desc), 2, 1); \
	210	\
	211	vop##BITS##_2(v1, v2, env, se, XxC, erm, FN, GETPC()); \
	212	}
	213
	214	#define DEF_GVEC_VOP2_64(NAME) \
	215	DEF_GVEC_VOP2_FN(NAME, NAME##64, 64)
	216
	217	#define DEF_GVEC_VOP2(NAME, OP) \
acb269a4 DH	218	DEF_GVEC_VOP2_FN(NAME, float32_##OP, 32) \
	219	DEF_GVEC_VOP2_FN(NAME, float64_##OP, 64) \
	220	DEF_GVEC_VOP2_FN(NAME, float128_##OP, 128)
21bd6ea2 DH	221
	222	DEF_GVEC_VOP2_64(vcdg)
	223	DEF_GVEC_VOP2_64(vcdlg)
	224	DEF_GVEC_VOP2_64(vcgd)
	225	DEF_GVEC_VOP2_64(vclgd)
	226	DEF_GVEC_VOP2(vfi, round_to_int)
	227	DEF_GVEC_VOP2(vfsq, sqrt)
	228
0987961d DH	229	typedef float32 (vop32_3_fn)(float32 a, float32 b, float_status s);
	230	static void vop32_3(S390Vector v1, const S390Vector v2, const S390Vector *v3,
	231	CPUS390XState *env, bool s, vop32_3_fn fn,
	232	uintptr_t retaddr)
	233	{
	234	uint8_t vxc, vec_exc = 0;
	235	S390Vector tmp = {};
	236	int i;
	237
	238	for (i = 0; i < 4; i++) {
	239	const float32 a = s390_vec_read_float32(v2, i);
	240	const float32 b = s390_vec_read_float32(v3, i);
	241
	242	s390_vec_write_float32(&tmp, i, fn(a, b, &env->fpu_status));
	243	vxc = check_ieee_exc(env, i, false, &vec_exc);
	244	if (s \|\| vxc) {
	245	break;
	246	}
	247	}
	248	handle_ieee_exc(env, vxc, vec_exc, retaddr);
	249	*v1 = tmp;
	250	}
	251
863b9507	252	typedef float64 (vop64_3_fn)(float64 a, float64 b, float_status s);
3a0eae85 DH	253	static void vop64_3(S390Vector v1, const S390Vector v2, const S390Vector *v3,
	254	CPUS390XState *env, bool s, vop64_3_fn fn,
	255	uintptr_t retaddr)
	256	{
	257	uint8_t vxc, vec_exc = 0;
	258	S390Vector tmp = {};
	259	int i;
	260
	261	for (i = 0; i < 2; i++) {
863b9507 DH	262	const float64 a = s390_vec_read_float64(v2, i);
863b9507 DH	263	const float64 b = s390_vec_read_float64(v3, i);
3a0eae85	264
863b9507	265	s390_vec_write_float64(&tmp, i, fn(a, b, &env->fpu_status));
3a0eae85 DH	266	vxc = check_ieee_exc(env, i, false, &vec_exc);
	267	if (s \|\| vxc) {
	268	break;
	269	}
	270	}
	271	handle_ieee_exc(env, vxc, vec_exc, retaddr);
	272	*v1 = tmp;
	273	}
	274
0987961d DH	275	typedef float128 (vop128_3_fn)(float128 a, float128 b, float_status s);
	276	static void vop128_3(S390Vector v1, const S390Vector v2, const S390Vector *v3,
	277	CPUS390XState *env, bool s, vop128_3_fn fn,
	278	uintptr_t retaddr)
	279	{
	280	const float128 a = s390_vec_read_float128(v2);
	281	const float128 b = s390_vec_read_float128(v3);
	282	uint8_t vxc, vec_exc = 0;
	283	S390Vector tmp = {};
	284
	285	s390_vec_write_float128(&tmp, fn(a, b, &env->fpu_status));
	286	vxc = check_ieee_exc(env, 0, false, &vec_exc);
	287	handle_ieee_exc(env, vxc, vec_exc, retaddr);
	288	*v1 = tmp;
	289	}
	290
	291	#define DEF_GVEC_VOP3_B(NAME, OP, BITS) \
	292	void HELPER(gvec_##NAME##BITS)(void v1, const void v2, const void *v3, \
	293	CPUS390XState *env, uint32_t desc) \
863b9507 DH	294	{ \
	295	const bool se = extract32(simd_data(desc), 3, 1); \
	296	\
0987961d	297	vop##BITS##_3(v1, v2, v3, env, se, float##BITS##_##OP, GETPC()); \
3a0eae85 DH	298	}
3a0eae85 DH	299
0987961d DH	300	#define DEF_GVEC_VOP3(NAME, OP) \
	301	DEF_GVEC_VOP3_B(NAME, OP, 32) \
	302	DEF_GVEC_VOP3_B(NAME, OP, 64) \
	303	DEF_GVEC_VOP3_B(NAME, OP, 128)
	304
863b9507 DH	305	DEF_GVEC_VOP3(vfa, add)
	306	DEF_GVEC_VOP3(vfs, sub)
	307	DEF_GVEC_VOP3(vfd, div)
	308	DEF_GVEC_VOP3(vfm, mul)
5b89f0fb	309
1c6b5b47 DH	310	static int wfc32(const S390Vector v1, const S390Vector v2,
	311	CPUS390XState *env, bool signal, uintptr_t retaddr)
	312	{
	313	/* only the zero-indexed elements are compared */
	314	const float32 a = s390_vec_read_float32(v1, 0);
	315	const float32 b = s390_vec_read_float32(v2, 0);
	316	uint8_t vxc, vec_exc = 0;
	317	int cmp;
	318
	319	if (signal) {
	320	cmp = float32_compare(a, b, &env->fpu_status);
	321	} else {
	322	cmp = float32_compare_quiet(a, b, &env->fpu_status);
	323	}
	324	vxc = check_ieee_exc(env, 0, false, &vec_exc);
	325	handle_ieee_exc(env, vxc, vec_exc, retaddr);
	326
	327	return float_comp_to_cc(env, cmp);
	328	}
	329
5b89f0fb DH	330	static int wfc64(const S390Vector v1, const S390Vector v2,
	331	CPUS390XState *env, bool signal, uintptr_t retaddr)
	332	{
	333	/* only the zero-indexed elements are compared */
4da79375 DH	334	const float64 a = s390_vec_read_float64(v1, 0);
4da79375 DH	335	const float64 b = s390_vec_read_float64(v2, 0);
5b89f0fb DH	336	uint8_t vxc, vec_exc = 0;
	337	int cmp;
	338
	339	if (signal) {
	340	cmp = float64_compare(a, b, &env->fpu_status);
	341	} else {
	342	cmp = float64_compare_quiet(a, b, &env->fpu_status);
	343	}
	344	vxc = check_ieee_exc(env, 0, false, &vec_exc);
	345	handle_ieee_exc(env, vxc, vec_exc, retaddr);
	346
	347	return float_comp_to_cc(env, cmp);
	348	}
	349
1c6b5b47 DH	350	static int wfc128(const S390Vector v1, const S390Vector v2,
	351	CPUS390XState *env, bool signal, uintptr_t retaddr)
	352	{
	353	/* only the zero-indexed elements are compared */
	354	const float128 a = s390_vec_read_float128(v1);
	355	const float128 b = s390_vec_read_float128(v2);
	356	uint8_t vxc, vec_exc = 0;
	357	int cmp;
	358
	359	if (signal) {
	360	cmp = float128_compare(a, b, &env->fpu_status);
	361	} else {
	362	cmp = float128_compare_quiet(a, b, &env->fpu_status);
	363	}
	364	vxc = check_ieee_exc(env, 0, false, &vec_exc);
	365	handle_ieee_exc(env, vxc, vec_exc, retaddr);
	366
	367	return float_comp_to_cc(env, cmp);
	368	}
	369
4da79375 DH	370	#define DEF_GVEC_WFC_B(NAME, SIGNAL, BITS) \
	371	void HELPER(gvec_##NAME##BITS)(const void v1, const void v2, \
	372	CPUS390XState *env, uint32_t desc) \
	373	{ \
	374	env->cc_op = wfc##BITS(v1, v2, env, SIGNAL, GETPC()); \
5b89f0fb DH	375	}
5b89f0fb DH	376
4da79375	377	#define DEF_GVEC_WFC(NAME, SIGNAL) \
1c6b5b47 DH	378	DEF_GVEC_WFC_B(NAME, SIGNAL, 32) \
	379	DEF_GVEC_WFC_B(NAME, SIGNAL, 64) \
	380	DEF_GVEC_WFC_B(NAME, SIGNAL, 128)
4da79375 DH	381
	382	DEF_GVEC_WFC(wfc, false)
	383	DEF_GVEC_WFC(wfk, true)
2c806ab4	384
e384332c DH	385	typedef bool (vfc32_fn)(float32 a, float32 b, float_status status);
	386	static int vfc32(S390Vector v1, const S390Vector v2, const S390Vector *v3,
	387	CPUS390XState *env, bool s, vfc32_fn fn, uintptr_t retaddr)
	388	{
	389	uint8_t vxc, vec_exc = 0;
	390	S390Vector tmp = {};
	391	int match = 0;
	392	int i;
	393
	394	for (i = 0; i < 4; i++) {
	395	const float32 a = s390_vec_read_float32(v2, i);
	396	const float32 b = s390_vec_read_float32(v3, i);
	397
	398	/* swap the order of the parameters, so we can use existing functions */
	399	if (fn(b, a, &env->fpu_status)) {
	400	match++;
	401	s390_vec_write_element32(&tmp, i, -1u);
	402	}
	403	vxc = check_ieee_exc(env, i, false, &vec_exc);
	404	if (s \|\| vxc) {
	405	break;
	406	}
	407	}
	408
	409	handle_ieee_exc(env, vxc, vec_exc, retaddr);
	410	*v1 = tmp;
	411	if (match) {
	412	return s \|\| match == 4 ? 0 : 1;
	413	}
	414	return 3;
	415	}
	416
0673ecdf	417	typedef bool (vfc64_fn)(float64 a, float64 b, float_status status);
2c806ab4 DH	418	static int vfc64(S390Vector v1, const S390Vector v2, const S390Vector *v3,
	419	CPUS390XState *env, bool s, vfc64_fn fn, uintptr_t retaddr)
	420	{
	421	uint8_t vxc, vec_exc = 0;
	422	S390Vector tmp = {};
	423	int match = 0;
	424	int i;
	425
	426	for (i = 0; i < 2; i++) {
64deb65a DH	427	const float64 a = s390_vec_read_float64(v2, i);
64deb65a DH	428	const float64 b = s390_vec_read_float64(v3, i);
2c806ab4 DH	429
	430	/* swap the order of the parameters, so we can use existing functions */
	431	if (fn(b, a, &env->fpu_status)) {
	432	match++;
	433	s390_vec_write_element64(&tmp, i, -1ull);
	434	}
	435	vxc = check_ieee_exc(env, i, false, &vec_exc);
	436	if (s \|\| vxc) {
	437	break;
	438	}
	439	}
	440
	441	handle_ieee_exc(env, vxc, vec_exc, retaddr);
	442	*v1 = tmp;
	443	if (match) {
	444	return s \|\| match == 2 ? 0 : 1;
	445	}
	446	return 3;
	447	}
	448
e384332c DH	449	typedef bool (vfc128_fn)(float128 a, float128 b, float_status status);
	450	static int vfc128(S390Vector v1, const S390Vector v2, const S390Vector *v3,
	451	CPUS390XState *env, bool s, vfc128_fn fn, uintptr_t retaddr)
	452	{
	453	const float128 a = s390_vec_read_float128(v2);
	454	const float128 b = s390_vec_read_float128(v3);
	455	uint8_t vxc, vec_exc = 0;
	456	S390Vector tmp = {};
	457	bool match = false;
	458
	459	/* swap the order of the parameters, so we can use existing functions */
	460	if (fn(b, a, &env->fpu_status)) {
	461	match = true;
	462	s390_vec_write_element64(&tmp, 0, -1ull);
	463	s390_vec_write_element64(&tmp, 1, -1ull);
	464	}
	465	vxc = check_ieee_exc(env, 0, false, &vec_exc);
	466	handle_ieee_exc(env, vxc, vec_exc, retaddr);
	467	*v1 = tmp;
	468	return match ? 0 : 3;
	469	}
	470
64deb65a DH	471	#define DEF_GVEC_VFC_B(NAME, OP, BITS) \
	472	void HELPER(gvec_##NAME##BITS)(void v1, const void v2, const void *v3, \
	473	CPUS390XState *env, uint32_t desc) \
	474	{ \
	475	const bool se = extract32(simd_data(desc), 3, 1); \
e384332c DH	476	const bool sq = extract32(simd_data(desc), 2, 1); \
e384332c DH	477	vfc##BITS##_fn fn = sq ? float##BITS##_##OP : float##BITS##_##OP##_quiet; \
64deb65a DH	478	\
	479	vfc##BITS(v1, v2, v3, env, se, fn, GETPC()); \
	480	} \
	481	\
	482	void HELPER(gvec_##NAME##BITS##_cc)(void v1, const void v2, const void *v3, \
	483	CPUS390XState *env, uint32_t desc) \
	484	{ \
	485	const bool se = extract32(simd_data(desc), 3, 1); \
e384332c DH	486	const bool sq = extract32(simd_data(desc), 2, 1); \
e384332c DH	487	vfc##BITS##_fn fn = sq ? float##BITS##_##OP : float##BITS##_##OP##_quiet; \
64deb65a DH	488	\
64deb65a DH	489	env->cc_op = vfc##BITS(v1, v2, v3, env, se, fn, GETPC()); \
2c806ab4 DH	490	}
2c806ab4 DH	491
64deb65a	492	#define DEF_GVEC_VFC(NAME, OP) \
e384332c DH	493	DEF_GVEC_VFC_B(NAME, OP, 32) \
	494	DEF_GVEC_VFC_B(NAME, OP, 64) \
	495	DEF_GVEC_VFC_B(NAME, OP, 128) \
2c806ab4	496
64deb65a DH	497	DEF_GVEC_VFC(vfce, eq)
	498	DEF_GVEC_VFC(vfch, lt)
	499	DEF_GVEC_VFC(vfche, le)
bb03fd84	500
860b707b DH	501	void HELPER(gvec_vfll32)(void v1, const void v2, CPUS390XState *env,
860b707b DH	502	uint32_t desc)
1a76e59d	503	{
860b707b	504	const bool s = extract32(simd_data(desc), 3, 1);
1a76e59d DH	505	uint8_t vxc, vec_exc = 0;
	506	S390Vector tmp = {};
	507	int i;
	508
	509	for (i = 0; i < 2; i++) {
	510	/* load from even element */
	511	const float32 a = s390_vec_read_element32(v2, i * 2);
	512	const uint64_t ret = float32_to_float64(a, &env->fpu_status);
	513
	514	s390_vec_write_element64(&tmp, i, ret);
	515	/* indicate the source element */
	516	vxc = check_ieee_exc(env, i * 2, false, &vec_exc);
	517	if (s \|\| vxc) {
	518	break;
	519	}
	520	}
860b707b DH	521	handle_ieee_exc(env, vxc, vec_exc, GETPC());
860b707b DH	522	(S390Vector )v1 = tmp;
1a76e59d	523	}
4500ede4	524
2e96005e DH	525	void HELPER(gvec_vfll64)(void v1, const void v2, CPUS390XState *env,
	526	uint32_t desc)
	527	{
	528	/* load from even element */
	529	const float128 ret = float64_to_float128(s390_vec_read_float64(v2, 0),
	530	&env->fpu_status);
	531	uint8_t vxc, vec_exc = 0;
	532
	533	vxc = check_ieee_exc(env, 0, false, &vec_exc);
	534	handle_ieee_exc(env, vxc, vec_exc, GETPC());
	535	s390_vec_write_float128(v1, ret);
	536	}
	537
977e43d9 DH	538	void HELPER(gvec_vflr64)(void v1, const void v2, CPUS390XState *env,
977e43d9 DH	539	uint32_t desc)
4500ede4	540	{
977e43d9 DH	541	const uint8_t erm = extract32(simd_data(desc), 4, 4);
	542	const bool s = extract32(simd_data(desc), 3, 1);
	543	const bool XxC = extract32(simd_data(desc), 2, 1);
4500ede4 DH	544	uint8_t vxc, vec_exc = 0;
	545	S390Vector tmp = {};
	546	int i, old_mode;
	547
	548	old_mode = s390_swap_bfp_rounding_mode(env, erm);
	549	for (i = 0; i < 2; i++) {
	550	float64 a = s390_vec_read_element64(v2, i);
	551	uint32_t ret = float64_to_float32(a, &env->fpu_status);
	552
	553	/* place at even element */
	554	s390_vec_write_element32(&tmp, i * 2, ret);
	555	/* indicate the source element */
	556	vxc = check_ieee_exc(env, i, XxC, &vec_exc);
	557	if (s \|\| vxc) {
	558	break;
	559	}
	560	}
	561	s390_restore_bfp_rounding_mode(env, old_mode);
977e43d9 DH	562	handle_ieee_exc(env, vxc, vec_exc, GETPC());
977e43d9 DH	563	(S390Vector )v1 = tmp;
4500ede4	564	}
8d47d4d2	565
9cbc8be0 DH	566	void HELPER(gvec_vflr128)(void v1, const void v2, CPUS390XState *env,
	567	uint32_t desc)
	568	{
	569	const uint8_t erm = extract32(simd_data(desc), 4, 4);
	570	const bool XxC = extract32(simd_data(desc), 2, 1);
	571	uint8_t vxc, vec_exc = 0;
	572	int old_mode;
	573	float64 ret;
	574
	575	old_mode = s390_swap_bfp_rounding_mode(env, erm);
	576	ret = float128_to_float64(s390_vec_read_float128(v2), &env->fpu_status);
	577	vxc = check_ieee_exc(env, 0, XxC, &vec_exc);
	578	s390_restore_bfp_rounding_mode(env, old_mode);
	579	handle_ieee_exc(env, vxc, vec_exc, GETPC());
	580
	581	/* place at even element, odd element is unpredictable */
	582	s390_vec_write_float64(v1, 0, ret);
	583	}
	584
c64c5984 DH	585	static void vfma64(S390Vector v1, const S390Vector v2, const S390Vector *v3,
	586	const S390Vector v4, CPUS390XState env, bool s, int flags,
	587	uintptr_t retaddr)
	588	{
	589	uint8_t vxc, vec_exc = 0;
	590	S390Vector tmp = {};
	591	int i;
	592
	593	for (i = 0; i < 2; i++) {
34142ffd DH	594	const float64 a = s390_vec_read_float64(v2, i);
	595	const float64 b = s390_vec_read_float64(v3, i);
	596	const float64 c = s390_vec_read_float64(v4, i);
	597	const float64 ret = float64_muladd(a, b, c, flags, &env->fpu_status);
c64c5984	598
34142ffd	599	s390_vec_write_float64(&tmp, i, ret);
c64c5984 DH	600	vxc = check_ieee_exc(env, i, false, &vec_exc);
	601	if (s \|\| vxc) {
	602	break;
	603	}
	604	}
	605	handle_ieee_exc(env, vxc, vec_exc, retaddr);
	606	*v1 = tmp;
	607	}
	608
34142ffd DH	609	#define DEF_GVEC_VFMA_B(NAME, FLAGS, BITS) \
	610	void HELPER(gvec_##NAME##BITS)(void v1, const void v2, const void *v3, \
	611	const void v4, CPUS390XState env, \
	612	uint32_t desc) \
	613	{ \
	614	const bool se = extract32(simd_data(desc), 3, 1); \
	615	\
	616	vfma##BITS(v1, v2, v3, v4, env, se, FLAGS, GETPC()); \
c64c5984 DH	617	}
c64c5984 DH	618
34142ffd DH	619	#define DEF_GVEC_VFMA(NAME, FLAGS) \
34142ffd DH	620	DEF_GVEC_VFMA_B(NAME, FLAGS, 64)
c64c5984	621
34142ffd DH	622	DEF_GVEC_VFMA(vfma, 0)
34142ffd DH	623	DEF_GVEC_VFMA(vfms, float_muladd_negate_c)
5938f20c	624
622ebe64 DH	625	void HELPER(gvec_vftci64)(void v1, const void v2, CPUS390XState *env,
622ebe64 DH	626	uint32_t desc)
83b955f9	627	{
622ebe64 DH	628	const uint16_t i3 = extract32(simd_data(desc), 4, 12);
622ebe64 DH	629	const bool s = extract32(simd_data(desc), 3, 1);
83b955f9 DH	630	int i, match = 0;
	631
	632	for (i = 0; i < 2; i++) {
622ebe64	633	const float64 a = s390_vec_read_float64(v2, i);
83b955f9 DH	634
	635	if (float64_dcmask(env, a) & i3) {
	636	match++;
	637	s390_vec_write_element64(v1, i, -1ull);
	638	} else {
	639	s390_vec_write_element64(v1, i, 0);
	640	}
	641	if (s) {
	642	break;
	643	}
	644	}
	645
622ebe64 DH	646	if (match == 2 \|\| (s && match)) {
	647	env->cc_op = 0;
	648	} else if (match) {
	649	env->cc_op = 1;
	650	} else {
	651	env->cc_op = 3;
83b955f9	652	}
83b955f9	653	}