[mirror_qemu.git] / target / s390x / vec_string_helper.c

/*
 * QEMU TCG support -- s390x vector string instruction support
 *
 * Copyright (C) 2019 Red Hat Inc
 *
 * Authors:
 *   David Hildenbrand <david@redhat.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2 or later.
 * See the COPYING file in the top-level directory.
 */
#include "qemu/osdep.h"
#include "qemu-common.h"
#include "cpu.h"
#include "internal.h"
#include "vec.h"
#include "tcg/tcg.h"
#include "tcg/tcg-gvec-desc.h"
#include "exec/helper-proto.h"

/*
 * Returns a bit set in the MSB of each element that is zero,
 * as defined by the mask.
 */
static inline uint64_t zero_search(uint64_t a, uint64_t mask)
{
    return ~(((a & mask) + mask) | a | mask);
}

/*
 * Returns a bit set in the MSB of each element that is not zero,
 * as defined by the mask.
 */
static inline uint64_t nonzero_search(uint64_t a, uint64_t mask)
{
    return (((a & mask) + mask) | a) & ~mask;
}

/*
 * Returns the byte offset for the first match, or 16 for no match.
 */
static inline int match_index(uint64_t c0, uint64_t c1)
{
    return (c0 ? clz64(c0) : clz64(c1) + 64) >> 3;
}

/*
 * Returns the number of bits composing one element.
 */
static uint8_t get_element_bits(uint8_t es)
{
    return (1 << es) * BITS_PER_BYTE;
}

/*
 * Returns the bitmask for a single element.
 */
static uint64_t get_single_element_mask(uint8_t es)
{
    return -1ull >> (64 - get_element_bits(es));
}

/*
 * Returns the bitmask for a single element (excluding the MSB).
 */
static uint64_t get_single_element_lsbs_mask(uint8_t es)
{
    return -1ull >> (65 - get_element_bits(es));
}

/*
 * Returns the bitmasks for multiple elements (excluding the MSBs).
 */
static uint64_t get_element_lsbs_mask(uint8_t es)
{
    return dup_const(es, get_single_element_lsbs_mask(es));
}

static int vfae(void *v1, const void *v2, const void *v3, bool in,
                bool rt, bool zs, uint8_t es)
{
    const uint64_t mask = get_element_lsbs_mask(es);
    const int bits = get_element_bits(es);
    uint64_t a0, a1, b0, b1, e0, e1, t0, t1, z0, z1;
    uint64_t first_zero = 16;
    uint64_t first_equal;
    int i;

    a0 = s390_vec_read_element64(v2, 0);
    a1 = s390_vec_read_element64(v2, 1);
    b0 = s390_vec_read_element64(v3, 0);
    b1 = s390_vec_read_element64(v3, 1);
    e0 = 0;
    e1 = 0;
    /* compare against equality with every other element */
    for (i = 0; i < 64; i += bits) {
        t0 = rol64(b0, i);
        t1 = rol64(b1, i);
        e0 |= zero_search(a0 ^ t0, mask);
        e0 |= zero_search(a0 ^ t1, mask);
        e1 |= zero_search(a1 ^ t0, mask);
        e1 |= zero_search(a1 ^ t1, mask);
    }
    /* invert the result if requested - invert only the MSBs */
    if (in) {
        e0 = ~e0 & ~mask;
        e1 = ~e1 & ~mask;
    }
    first_equal = match_index(e0, e1);

    if (zs) {
        z0 = zero_search(a0, mask);
        z1 = zero_search(a1, mask);
        first_zero = match_index(z0, z1);
    }

    if (rt) {
        e0 = (e0 >> (bits - 1)) * get_single_element_mask(es);
        e1 = (e1 >> (bits - 1)) * get_single_element_mask(es);
        s390_vec_write_element64(v1, 0, e0);
        s390_vec_write_element64(v1, 1, e1);
    } else {
        s390_vec_write_element64(v1, 0, MIN(first_equal, first_zero));
        s390_vec_write_element64(v1, 1, 0);
    }

    if (first_zero == 16 && first_equal == 16) {
        return 3; /* no match */
    } else if (first_zero == 16) {
        return 1; /* matching elements, no match for zero */
    } else if (first_equal < first_zero) {
        return 2; /* matching elements before match for zero */
    }
    return 0; /* match for zero */
}

#define DEF_VFAE_HELPER(BITS)                                                  \
void HELPER(gvec_vfae##BITS)(void *v1, const void *v2, const void *v3,         \
                             uint32_t desc)                                    \
{                                                                              \
    const bool in = extract32(simd_data(desc), 3, 1);                          \
    const bool rt = extract32(simd_data(desc), 2, 1);                          \
    const bool zs = extract32(simd_data(desc), 1, 1);                          \
                                                                               \
    vfae(v1, v2, v3, in, rt, zs, MO_##BITS);                                   \
}
DEF_VFAE_HELPER(8)
DEF_VFAE_HELPER(16)
DEF_VFAE_HELPER(32)

#define DEF_VFAE_CC_HELPER(BITS)                                               \
void HELPER(gvec_vfae_cc##BITS)(void *v1, const void *v2, const void *v3,      \
                                CPUS390XState *env, uint32_t desc)             \
{                                                                              \
    const bool in = extract32(simd_data(desc), 3, 1);                          \
    const bool rt = extract32(simd_data(desc), 2, 1);                          \
    const bool zs = extract32(simd_data(desc), 1, 1);                          \
                                                                               \
    env->cc_op = vfae(v1, v2, v3, in, rt, zs, MO_##BITS);                      \
}
DEF_VFAE_CC_HELPER(8)
DEF_VFAE_CC_HELPER(16)
DEF_VFAE_CC_HELPER(32)

static int vfee(void *v1, const void *v2, const void *v3, bool zs, uint8_t es)
{
    const uint64_t mask = get_element_lsbs_mask(es);
    uint64_t a0, a1, b0, b1, e0, e1, z0, z1;
    uint64_t first_zero = 16;
    uint64_t first_equal;

    a0 = s390_vec_read_element64(v2, 0);
    a1 = s390_vec_read_element64(v2, 1);
    b0 = s390_vec_read_element64(v3, 0);
    b1 = s390_vec_read_element64(v3, 1);
    e0 = zero_search(a0 ^ b0, mask);
    e1 = zero_search(a1 ^ b1, mask);
    first_equal = match_index(e0, e1);

    if (zs) {
        z0 = zero_search(a0, mask);
        z1 = zero_search(a1, mask);
        first_zero = match_index(z0, z1);
    }

    s390_vec_write_element64(v1, 0, MIN(first_equal, first_zero));
    s390_vec_write_element64(v1, 1, 0);
    if (first_zero == 16 && first_equal == 16) {
        return 3; /* no match */
    } else if (first_zero == 16) {
        return 1; /* matching elements, no match for zero */
    } else if (first_equal < first_zero) {
        return 2; /* matching elements before match for zero */
    }
    return 0; /* match for zero */
}

#define DEF_VFEE_HELPER(BITS)                                                  \
void HELPER(gvec_vfee##BITS)(void *v1, const void *v2, const void *v3,         \
                             uint32_t desc)                                    \
{                                                                              \
    const bool zs = extract32(simd_data(desc), 1, 1);                          \
                                                                               \
    vfee(v1, v2, v3, zs, MO_##BITS);                                           \
}
DEF_VFEE_HELPER(8)
DEF_VFEE_HELPER(16)
DEF_VFEE_HELPER(32)

#define DEF_VFEE_CC_HELPER(BITS)                                               \
void HELPER(gvec_vfee_cc##BITS)(void *v1, const void *v2, const void *v3,      \
                                CPUS390XState *env, uint32_t desc)             \
{                                                                              \
    const bool zs = extract32(simd_data(desc), 1, 1);                          \
                                                                               \
    env->cc_op = vfee(v1, v2, v3, zs, MO_##BITS);                              \
}
DEF_VFEE_CC_HELPER(8)
DEF_VFEE_CC_HELPER(16)
DEF_VFEE_CC_HELPER(32)

static int vfene(void *v1, const void *v2, const void *v3, bool zs, uint8_t es)
{
    const uint64_t mask = get_element_lsbs_mask(es);
    uint64_t a0, a1, b0, b1, e0, e1, z0, z1;
    uint64_t first_zero = 16;
    uint64_t first_inequal;
    bool smaller = false;

    a0 = s390_vec_read_element64(v2, 0);
    a1 = s390_vec_read_element64(v2, 1);
    b0 = s390_vec_read_element64(v3, 0);
    b1 = s390_vec_read_element64(v3, 1);
    e0 = nonzero_search(a0 ^ b0, mask);
    e1 = nonzero_search(a1 ^ b1, mask);
    first_inequal = match_index(e0, e1);

    /* identify the smaller element */
    if (first_inequal < 16) {
        uint8_t enr = first_inequal / (1 << es);
        uint32_t a = s390_vec_read_element(v2, enr, es);
        uint32_t b = s390_vec_read_element(v3, enr, es);

        smaller = a < b;
    }

    if (zs) {
        z0 = zero_search(a0, mask);
        z1 = zero_search(a1, mask);
        first_zero = match_index(z0, z1);
    }

    s390_vec_write_element64(v1, 0, MIN(first_inequal, first_zero));
    s390_vec_write_element64(v1, 1, 0);
    if (first_zero == 16 && first_inequal == 16) {
        return 3;
    } else if (first_zero < first_inequal) {
        return 0;
    }
    return smaller ? 1 : 2;
}

#define DEF_VFENE_HELPER(BITS)                                                 \
void HELPER(gvec_vfene##BITS)(void *v1, const void *v2, const void *v3,        \
                              uint32_t desc)                                   \
{                                                                              \
    const bool zs = extract32(simd_data(desc), 1, 1);                          \
                                                                               \
    vfene(v1, v2, v3, zs, MO_##BITS);                                          \
}
DEF_VFENE_HELPER(8)
DEF_VFENE_HELPER(16)
DEF_VFENE_HELPER(32)

#define DEF_VFENE_CC_HELPER(BITS)                                              \
void HELPER(gvec_vfene_cc##BITS)(void *v1, const void *v2, const void *v3,     \
                                 CPUS390XState *env, uint32_t desc)            \
{                                                                              \
    const bool zs = extract32(simd_data(desc), 1, 1);                          \
                                                                               \
    env->cc_op = vfene(v1, v2, v3, zs, MO_##BITS);                             \
}
DEF_VFENE_CC_HELPER(8)
DEF_VFENE_CC_HELPER(16)
DEF_VFENE_CC_HELPER(32)
Commit	Line	Data
1fd28638 DH	1	/*
	2	* QEMU TCG support -- s390x vector string instruction support
	3	*
	4	* Copyright (C) 2019 Red Hat Inc
	5	*
	6	* Authors:
	7	* David Hildenbrand <david@redhat.com>
	8	*
	9	* This work is licensed under the terms of the GNU GPL, version 2 or later.
	10	* See the COPYING file in the top-level directory.
	11	*/
	12	#include "qemu/osdep.h"
	13	#include "qemu-common.h"
	14	#include "cpu.h"
	15	#include "internal.h"
	16	#include "vec.h"
	17	#include "tcg/tcg.h"
	18	#include "tcg/tcg-gvec-desc.h"
	19	#include "exec/helper-proto.h"
	20
	21	/*
	22	* Returns a bit set in the MSB of each element that is zero,
	23	* as defined by the mask.
	24	*/
	25	static inline uint64_t zero_search(uint64_t a, uint64_t mask)
	26	{
	27	return ~(((a & mask) + mask) \| a \| mask);
	28	}
	29
074e99b3 DH	30	/*
	31	* Returns a bit set in the MSB of each element that is not zero,
	32	* as defined by the mask.
	33	*/
	34	static inline uint64_t nonzero_search(uint64_t a, uint64_t mask)
	35	{
	36	return (((a & mask) + mask) \| a) & ~mask;
	37	}
	38
1fd28638 DH	39	/*
	40	* Returns the byte offset for the first match, or 16 for no match.
	41	*/
	42	static inline int match_index(uint64_t c0, uint64_t c1)
	43	{
	44	return (c0 ? clz64(c0) : clz64(c1) + 64) >> 3;
	45	}
	46
	47	/*
	48	* Returns the number of bits composing one element.
	49	*/
	50	static uint8_t get_element_bits(uint8_t es)
	51	{
	52	return (1 << es) * BITS_PER_BYTE;
	53	}
	54
	55	/*
	56	* Returns the bitmask for a single element.
	57	*/
	58	static uint64_t get_single_element_mask(uint8_t es)
	59	{
	60	return -1ull >> (64 - get_element_bits(es));
	61	}
	62
	63	/*
	64	* Returns the bitmask for a single element (excluding the MSB).
	65	*/
	66	static uint64_t get_single_element_lsbs_mask(uint8_t es)
	67	{
	68	return -1ull >> (65 - get_element_bits(es));
	69	}
	70
	71	/*
	72	* Returns the bitmasks for multiple elements (excluding the MSBs).
	73	*/
	74	static uint64_t get_element_lsbs_mask(uint8_t es)
	75	{
	76	return dup_const(es, get_single_element_lsbs_mask(es));
	77	}
	78
	79	static int vfae(void v1, const void v2, const void *v3, bool in,
	80	bool rt, bool zs, uint8_t es)
	81	{
	82	const uint64_t mask = get_element_lsbs_mask(es);
	83	const int bits = get_element_bits(es);
	84	uint64_t a0, a1, b0, b1, e0, e1, t0, t1, z0, z1;
	85	uint64_t first_zero = 16;
	86	uint64_t first_equal;
	87	int i;
	88
	89	a0 = s390_vec_read_element64(v2, 0);
	90	a1 = s390_vec_read_element64(v2, 1);
	91	b0 = s390_vec_read_element64(v3, 0);
	92	b1 = s390_vec_read_element64(v3, 1);
	93	e0 = 0;
	94	e1 = 0;
	95	/* compare against equality with every other element */
	96	for (i = 0; i < 64; i += bits) {
	97	t0 = rol64(b0, i);
	98	t1 = rol64(b1, i);
	99	e0 \|= zero_search(a0 ^ t0, mask);
	100	e0 \|= zero_search(a0 ^ t1, mask);
	101	e1 \|= zero_search(a1 ^ t0, mask);
	102	e1 \|= zero_search(a1 ^ t1, mask);
103	}
104	/* invert the result if requested - invert only the MSBs */
105	if (in) {
106	e0 = ~e0 & ~mask;
107	e1 = ~e1 & ~mask;
108	}
109	first_equal = match_index(e0, e1);
110
111	if (zs) {
112	z0 = zero_search(a0, mask);
113	z1 = zero_search(a1, mask);
114	first_zero = match_index(z0, z1);
115	}
116
117	if (rt) {
118	e0 = (e0 >> (bits - 1)) * get_single_element_mask(es);
119	e1 = (e1 >> (bits - 1)) * get_single_element_mask(es);
120	s390_vec_write_element64(v1, 0, e0);
121	s390_vec_write_element64(v1, 1, e1);
122	} else {
123	s390_vec_write_element64(v1, 0, MIN(first_equal, first_zero));
124	s390_vec_write_element64(v1, 1, 0);
125	}
126
127	if (first_zero == 16 && first_equal == 16) {
128	return 3; /* no match */
129	} else if (first_zero == 16) {
130	return 1; /* matching elements, no match for zero */
131	} else if (first_equal < first_zero) {
132	return 2; /* matching elements before match for zero */
133	}
134	return 0; /* match for zero */
135	}
136
137	#define DEF_VFAE_HELPER(BITS) \
138	void HELPER(gvec_vfae##BITS)(void v1, const void v2, const void *v3, \
139	uint32_t desc) \
140	{ \
141	const bool in = extract32(simd_data(desc), 3, 1); \
142	const bool rt = extract32(simd_data(desc), 2, 1); \
143	const bool zs = extract32(simd_data(desc), 1, 1); \
144	\
145	vfae(v1, v2, v3, in, rt, zs, MO_##BITS); \
146	}
147	DEF_VFAE_HELPER(8)
148	DEF_VFAE_HELPER(16)
149	DEF_VFAE_HELPER(32)
150
151	#define DEF_VFAE_CC_HELPER(BITS) \
152	void HELPER(gvec_vfae_cc##BITS)(void v1, const void v2, const void *v3, \
153	CPUS390XState *env, uint32_t desc) \
154	{ \
155	const bool in = extract32(simd_data(desc), 3, 1); \
156	const bool rt = extract32(simd_data(desc), 2, 1); \
157	const bool zs = extract32(simd_data(desc), 1, 1); \
158	\
159	env->cc_op = vfae(v1, v2, v3, in, rt, zs, MO_##BITS); \
160	}
161	DEF_VFAE_CC_HELPER(8)
162	DEF_VFAE_CC_HELPER(16)
163	DEF_VFAE_CC_HELPER(32)
8c0e1e58 DH	164
	165	static int vfee(void v1, const void v2, const void *v3, bool zs, uint8_t es)
	166	{
	167	const uint64_t mask = get_element_lsbs_mask(es);
	168	uint64_t a0, a1, b0, b1, e0, e1, z0, z1;
	169	uint64_t first_zero = 16;
	170	uint64_t first_equal;
	171
	172	a0 = s390_vec_read_element64(v2, 0);
	173	a1 = s390_vec_read_element64(v2, 1);
	174	b0 = s390_vec_read_element64(v3, 0);
	175	b1 = s390_vec_read_element64(v3, 1);
	176	e0 = zero_search(a0 ^ b0, mask);
	177	e1 = zero_search(a1 ^ b1, mask);
	178	first_equal = match_index(e0, e1);
	179
	180	if (zs) {
	181	z0 = zero_search(a0, mask);
	182	z1 = zero_search(a1, mask);
	183	first_zero = match_index(z0, z1);
	184	}
	185
	186	s390_vec_write_element64(v1, 0, MIN(first_equal, first_zero));
	187	s390_vec_write_element64(v1, 1, 0);
	188	if (first_zero == 16 && first_equal == 16) {
	189	return 3; /* no match */
	190	} else if (first_zero == 16) {
	191	return 1; /* matching elements, no match for zero */
	192	} else if (first_equal < first_zero) {
	193	return 2; /* matching elements before match for zero */
	194	}
	195	return 0; /* match for zero */
	196	}
	197
	198	#define DEF_VFEE_HELPER(BITS) \
	199	void HELPER(gvec_vfee##BITS)(void v1, const void v2, const void *v3, \
	200	uint32_t desc) \
	201	{ \
	202	const bool zs = extract32(simd_data(desc), 1, 1); \
	203	\
	204	vfee(v1, v2, v3, zs, MO_##BITS); \
	205	}
	206	DEF_VFEE_HELPER(8)
	207	DEF_VFEE_HELPER(16)
	208	DEF_VFEE_HELPER(32)
	209
	210	#define DEF_VFEE_CC_HELPER(BITS) \
	211	void HELPER(gvec_vfee_cc##BITS)(void v1, const void v2, const void *v3, \
	212	CPUS390XState *env, uint32_t desc) \
	213	{ \
	214	const bool zs = extract32(simd_data(desc), 1, 1); \
	215	\
	216	env->cc_op = vfee(v1, v2, v3, zs, MO_##BITS); \
	217	}
	218	DEF_VFEE_CC_HELPER(8)
	219	DEF_VFEE_CC_HELPER(16)
	220	DEF_VFEE_CC_HELPER(32)
074e99b3 DH	221
	222	static int vfene(void v1, const void v2, const void *v3, bool zs, uint8_t es)
	223	{
	224	const uint64_t mask = get_element_lsbs_mask(es);
	225	uint64_t a0, a1, b0, b1, e0, e1, z0, z1;
	226	uint64_t first_zero = 16;
	227	uint64_t first_inequal;
	228	bool smaller = false;
	229
	230	a0 = s390_vec_read_element64(v2, 0);
	231	a1 = s390_vec_read_element64(v2, 1);
	232	b0 = s390_vec_read_element64(v3, 0);
	233	b1 = s390_vec_read_element64(v3, 1);
	234	e0 = nonzero_search(a0 ^ b0, mask);
	235	e1 = nonzero_search(a1 ^ b1, mask);
	236	first_inequal = match_index(e0, e1);
	237
	238	/* identify the smaller element */
	239	if (first_inequal < 16) {
	240	uint8_t enr = first_inequal / (1 << es);
	241	uint32_t a = s390_vec_read_element(v2, enr, es);
	242	uint32_t b = s390_vec_read_element(v3, enr, es);
	243
	244	smaller = a < b;
	245	}
	246
	247	if (zs) {
	248	z0 = zero_search(a0, mask);
	249	z1 = zero_search(a1, mask);
	250	first_zero = match_index(z0, z1);
	251	}
	252
	253	s390_vec_write_element64(v1, 0, MIN(first_inequal, first_zero));
	254	s390_vec_write_element64(v1, 1, 0);
	255	if (first_zero == 16 && first_inequal == 16) {
	256	return 3;
	257	} else if (first_zero < first_inequal) {
	258	return 0;
	259	}
	260	return smaller ? 1 : 2;
	261	}
	262
	263	#define DEF_VFENE_HELPER(BITS) \
	264	void HELPER(gvec_vfene##BITS)(void v1, const void v2, const void *v3, \
	265	uint32_t desc) \
	266	{ \
	267	const bool zs = extract32(simd_data(desc), 1, 1); \
	268	\
	269	vfene(v1, v2, v3, zs, MO_##BITS); \
	270	}
	271	DEF_VFENE_HELPER(8)
	272	DEF_VFENE_HELPER(16)
	273	DEF_VFENE_HELPER(32)
	274
	275	#define DEF_VFENE_CC_HELPER(BITS) \
	276	void HELPER(gvec_vfene_cc##BITS)(void v1, const void v2, const void *v3, \
	277	CPUS390XState *env, uint32_t desc) \
	278	{ \
	279	const bool zs = extract32(simd_data(desc), 1, 1); \
	280	\
	281	env->cc_op = vfene(v1, v2, v3, zs, MO_##BITS); \
	282	}
	283	DEF_VFENE_CC_HELPER(8)
	284	DEF_VFENE_CC_HELPER(16)
285	DEF_VFENE_CC_HELPER(32)