target/hexagon/op_helper.c

   1 /*
   2  *  Copyright(c) 2019-2023 Qualcomm Innovation Center, Inc. All Rights Reserved.
   3  *
   4  *  This program is free software; you can redistribute it and/or modify
   5  *  it under the terms of the GNU General Public License as published by
   6  *  the Free Software Foundation; either version 2 of the License, or
   7  *  (at your option) any later version.
   8  *
   9  *  This program is distributed in the hope that it will be useful,
  10  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  *  GNU General Public License for more details.
  13  *
  14  *  You should have received a copy of the GNU General Public License
  15  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 #include "qemu/osdep.h"
  19 #include "qemu/log.h"
  20 #include "exec/exec-all.h"
  21 #include "exec/cpu_ldst.h"
  22 #include "exec/helper-proto.h"
  23 #include "fpu/softfloat.h"
  24 #include "cpu.h"
  25 #include "internal.h"
  26 #include "macros.h"
  27 #include "arch.h"
  28 #include "hex_arch_types.h"
  29 #include "fma_emu.h"
  30 #include "mmvec/mmvec.h"
  31 #include "mmvec/macros.h"
  32 #include "op_helper.h"
  33 #include "translate.h"
  34
  35 #define SF_BIAS        127
  36 #define SF_MANTBITS    23
  37
  38 /* Exceptions processing helpers */
  39 static G_NORETURN
  40 void do_raise_exception_err(CPUHexagonState *env,
  41                             uint32_t exception,
  42                             uintptr_t pc)
  43 {
  44     CPUState *cs = env_cpu(env);
  45     qemu_log_mask(CPU_LOG_INT, "%s: %d\n", __func__, exception);
  46     cs->exception_index = exception;
  47     cpu_loop_exit_restore(cs, pc);
  48 }
  49
  50 G_NORETURN void HELPER(raise_exception)(CPUHexagonState *env, uint32_t excp)
  51 {
  52     do_raise_exception_err(env, excp, 0);
  53 }
  54
  55 void log_reg_write(CPUHexagonState *env, int rnum,
  56                    target_ulong val, uint32_t slot)
  57 {
  58     HEX_DEBUG_LOG("log_reg_write[%d] = " TARGET_FMT_ld " (0x" TARGET_FMT_lx ")",
  59                   rnum, val, val);
  60     if (val == env->gpr[rnum]) {
  61         HEX_DEBUG_LOG(" NO CHANGE");
  62     }
  63     HEX_DEBUG_LOG("\n");
  64
  65     env->new_value[rnum] = val;
  66     if (HEX_DEBUG) {
  67         /* Do this so HELPER(debug_commit_end) will know */
  68         env->reg_written[rnum] = 1;
  69     }
  70 }
  71
  72 static void log_pred_write(CPUHexagonState *env, int pnum, target_ulong val)
  73 {
  74     HEX_DEBUG_LOG("log_pred_write[%d] = " TARGET_FMT_ld
  75                   " (0x" TARGET_FMT_lx ")\n",
  76                   pnum, val, val);
  77
  78     /* Multiple writes to the same preg are and'ed together */
  79     if (env->pred_written & (1 << pnum)) {
  80         env->new_pred_value[pnum] &= val & 0xff;
  81     } else {
  82         env->new_pred_value[pnum] = val & 0xff;
  83         env->pred_written |= 1 << pnum;
  84     }
  85 }
  86
  87 void log_store32(CPUHexagonState *env, target_ulong addr,
  88                  target_ulong val, int width, int slot)
  89 {
  90     HEX_DEBUG_LOG("log_store%d(0x" TARGET_FMT_lx
  91                   ", %" PRId32 " [0x08%" PRIx32 "])\n",
  92                   width, addr, val, val);
  93     env->mem_log_stores[slot].va = addr;
  94     env->mem_log_stores[slot].width = width;
  95     env->mem_log_stores[slot].data32 = val;
  96 }
  97
  98 void log_store64(CPUHexagonState *env, target_ulong addr,
  99                  int64_t val, int width, int slot)
 100 {
 101     HEX_DEBUG_LOG("log_store%d(0x" TARGET_FMT_lx
 102                   ", %" PRId64 " [0x016%" PRIx64 "])\n",
 103                    width, addr, val, val);
 104     env->mem_log_stores[slot].va = addr;
 105     env->mem_log_stores[slot].width = width;
 106     env->mem_log_stores[slot].data64 = val;
 107 }
 108
 109 /* Handy place to set a breakpoint */
 110 void HELPER(debug_start_packet)(CPUHexagonState *env)
 111 {
 112     HEX_DEBUG_LOG("Start packet: pc = 0x" TARGET_FMT_lx "\n",
 113                   env->gpr[HEX_REG_PC]);
 114
 115     for (int i = 0; i < TOTAL_PER_THREAD_REGS; i++) {
 116         env->reg_written[i] = 0;
 117     }
 118 }
 119
 120 /* Checks for bookkeeping errors between disassembly context and runtime */
 121 void HELPER(debug_check_store_width)(CPUHexagonState *env, int slot, int check)
 122 {
 123     if (env->mem_log_stores[slot].width != check) {
 124         HEX_DEBUG_LOG("ERROR: %d != %d\n",
 125                       env->mem_log_stores[slot].width, check);
 126         g_assert_not_reached();
 127     }
 128 }
 129
 130 void HELPER(commit_store)(CPUHexagonState *env, int slot_num)
 131 {
 132     uintptr_t ra = GETPC();
 133     uint8_t width = env->mem_log_stores[slot_num].width;
 134     target_ulong va = env->mem_log_stores[slot_num].va;
 135
 136     switch (width) {
 137     case 1:
 138         cpu_stb_data_ra(env, va, env->mem_log_stores[slot_num].data32, ra);
 139         break;
 140     case 2:
 141         cpu_stw_data_ra(env, va, env->mem_log_stores[slot_num].data32, ra);
 142         break;
 143     case 4:
 144         cpu_stl_data_ra(env, va, env->mem_log_stores[slot_num].data32, ra);
 145         break;
 146     case 8:
 147         cpu_stq_data_ra(env, va, env->mem_log_stores[slot_num].data64, ra);
 148         break;
 149     default:
 150         g_assert_not_reached();
 151     }
 152 }
 153
 154 void HELPER(gather_store)(CPUHexagonState *env, uint32_t addr, int slot)
 155 {
 156     mem_gather_store(env, addr, slot);
 157 }
 158
 159 void HELPER(commit_hvx_stores)(CPUHexagonState *env)
 160 {
 161     uintptr_t ra = GETPC();
 162     int i;
 163
 164     /* Normal (possibly masked) vector store */
 165     for (i = 0; i < VSTORES_MAX; i++) {
 166         if (env->vstore_pending[i]) {
 167             env->vstore_pending[i] = 0;
 168             target_ulong va = env->vstore[i].va;
 169             int size = env->vstore[i].size;
 170             for (int j = 0; j < size; j++) {
 171                 if (test_bit(j, env->vstore[i].mask)) {
 172                     cpu_stb_data_ra(env, va + j, env->vstore[i].data.ub[j], ra);
 173                 }
 174             }
 175         }
 176     }
 177
 178     /* Scatter store */
 179     if (env->vtcm_pending) {
 180         env->vtcm_pending = false;
 181         if (env->vtcm_log.op) {
 182             /* Need to perform the scatter read/modify/write at commit time */
 183             if (env->vtcm_log.op_size == 2) {
 184                 SCATTER_OP_WRITE_TO_MEM(uint16_t);
 185             } else if (env->vtcm_log.op_size == 4) {
 186                 /* Word Scatter += */
 187                 SCATTER_OP_WRITE_TO_MEM(uint32_t);
 188             } else {
 189                 g_assert_not_reached();
 190             }
 191         } else {
 192             for (i = 0; i < sizeof(MMVector); i++) {
 193                 if (test_bit(i, env->vtcm_log.mask)) {
 194                     cpu_stb_data_ra(env, env->vtcm_log.va[i],
 195                                     env->vtcm_log.data.ub[i], ra);
 196                     clear_bit(i, env->vtcm_log.mask);
 197                     env->vtcm_log.data.ub[i] = 0;
 198                 }
 199
 200             }
 201         }
 202     }
 203 }
 204
 205 static void print_store(CPUHexagonState *env, int slot)
 206 {
 207     if (!(env->slot_cancelled & (1 << slot))) {
 208         uint8_t width = env->mem_log_stores[slot].width;
 209         if (width == 1) {
 210             uint32_t data = env->mem_log_stores[slot].data32 & 0xff;
 211             HEX_DEBUG_LOG("\tmemb[0x" TARGET_FMT_lx "] = %" PRId32
 212                           " (0x%02" PRIx32 ")\n",
 213                           env->mem_log_stores[slot].va, data, data);
 214         } else if (width == 2) {
 215             uint32_t data = env->mem_log_stores[slot].data32 & 0xffff;
 216             HEX_DEBUG_LOG("\tmemh[0x" TARGET_FMT_lx "] = %" PRId32
 217                           " (0x%04" PRIx32 ")\n",
 218                           env->mem_log_stores[slot].va, data, data);
 219         } else if (width == 4) {
 220             uint32_t data = env->mem_log_stores[slot].data32;
 221             HEX_DEBUG_LOG("\tmemw[0x" TARGET_FMT_lx "] = %" PRId32
 222                           " (0x%08" PRIx32 ")\n",
 223                           env->mem_log_stores[slot].va, data, data);
 224         } else if (width == 8) {
 225             HEX_DEBUG_LOG("\tmemd[0x" TARGET_FMT_lx "] = %" PRId64
 226                           " (0x%016" PRIx64 ")\n",
 227                           env->mem_log_stores[slot].va,
 228                           env->mem_log_stores[slot].data64,
 229                           env->mem_log_stores[slot].data64);
 230         } else {
 231             HEX_DEBUG_LOG("\tBad store width %d\n", width);
 232             g_assert_not_reached();
 233         }
 234     }
 235 }
 236
 237 /* This function is a handy place to set a breakpoint */
 238 void HELPER(debug_commit_end)(CPUHexagonState *env, int has_st0, int has_st1)
 239 {
 240     bool reg_printed = false;
 241     bool pred_printed = false;
 242     int i;
 243
 244     HEX_DEBUG_LOG("Packet committed: pc = 0x" TARGET_FMT_lx "\n",
 245                   env->this_PC);
 246     HEX_DEBUG_LOG("slot_cancelled = %d\n", env->slot_cancelled);
 247
 248     for (i = 0; i < TOTAL_PER_THREAD_REGS; i++) {
 249         if (env->reg_written[i]) {
 250             if (!reg_printed) {
 251                 HEX_DEBUG_LOG("Regs written\n");
 252                 reg_printed = true;
 253             }
 254             HEX_DEBUG_LOG("\tr%d = " TARGET_FMT_ld " (0x" TARGET_FMT_lx ")\n",
 255                           i, env->new_value[i], env->new_value[i]);
 256         }
 257     }
 258
 259     for (i = 0; i < NUM_PREGS; i++) {
 260         if (env->pred_written & (1 << i)) {
 261             if (!pred_printed) {
 262                 HEX_DEBUG_LOG("Predicates written\n");
 263                 pred_printed = true;
 264             }
 265             HEX_DEBUG_LOG("\tp%d = 0x" TARGET_FMT_lx "\n",
 266                           i, env->new_pred_value[i]);
 267         }
 268     }
 269
 270     if (has_st0 || has_st1) {
 271         HEX_DEBUG_LOG("Stores\n");
 272         if (has_st0) {
 273             print_store(env, 0);
 274         }
 275         if (has_st1) {
 276             print_store(env, 1);
 277         }
 278     }
 279
 280     HEX_DEBUG_LOG("Next PC = " TARGET_FMT_lx "\n", env->gpr[HEX_REG_PC]);
 281     HEX_DEBUG_LOG("Exec counters: pkt = " TARGET_FMT_lx
 282                   ", insn = " TARGET_FMT_lx
 283                   ", hvx = " TARGET_FMT_lx "\n",
 284                   env->gpr[HEX_REG_QEMU_PKT_CNT],
 285                   env->gpr[HEX_REG_QEMU_INSN_CNT],
 286                   env->gpr[HEX_REG_QEMU_HVX_CNT]);
 287
 288 }
 289
 290 int32_t HELPER(fcircadd)(int32_t RxV, int32_t offset, int32_t M, int32_t CS)
 291 {
 292     uint32_t K_const = extract32(M, 24, 4);
 293     uint32_t length = extract32(M, 0, 17);
 294     uint32_t new_ptr = RxV + offset;
 295     uint32_t start_addr;
 296     uint32_t end_addr;
 297
 298     if (K_const == 0 && length >= 4) {
 299         start_addr = CS;
 300         end_addr = start_addr + length;
 301     } else {
 302         /*
 303          * Versions v3 and earlier used the K value to specify a power-of-2 size
 304          * 2^(K+2) that is greater than the buffer length
 305          */
 306         int32_t mask = (1 << (K_const + 2)) - 1;
 307         start_addr = RxV & (~mask);
 308         end_addr = start_addr | length;
 309     }
 310
 311     if (new_ptr >= end_addr) {
 312         new_ptr -= length;
 313     } else if (new_ptr < start_addr) {
 314         new_ptr += length;
 315     }
 316
 317     return new_ptr;
 318 }
 319
 320 uint32_t HELPER(fbrev)(uint32_t addr)
 321 {
 322     /*
 323      *  Bit reverse the low 16 bits of the address
 324      */
 325     return deposit32(addr, 0, 16, revbit16(addr));
 326 }
 327
 328 static float32 build_float32(uint8_t sign, uint32_t exp, uint32_t mant)
 329 {
 330     return make_float32(
 331         ((sign & 1) << 31) |
 332         ((exp & 0xff) << SF_MANTBITS) |
 333         (mant & ((1 << SF_MANTBITS) - 1)));
 334 }
 335
 336 /*
 337  * sfrecipa, sfinvsqrta have two 32-bit results
 338  *     r0,p0=sfrecipa(r1,r2)
 339  *     r0,p0=sfinvsqrta(r1)
 340  *
 341  * Since helpers can only return a single value, we pack the two results
 342  * into a 64-bit value.
 343  */
 344 uint64_t HELPER(sfrecipa)(CPUHexagonState *env, float32 RsV, float32 RtV)
 345 {
 346     int32_t PeV = 0;
 347     float32 RdV;
 348     int idx;
 349     int adjust;
 350     int mant;
 351     int exp;
 352
 353     arch_fpop_start(env);
 354     if (arch_sf_recip_common(&RsV, &RtV, &RdV, &adjust, &env->fp_status)) {
 355         PeV = adjust;
 356         idx = (RtV >> 16) & 0x7f;
 357         mant = (recip_lookup_table[idx] << 15) | 1;
 358         exp = SF_BIAS - (float32_getexp(RtV) - SF_BIAS) - 1;
 359         RdV = build_float32(extract32(RtV, 31, 1), exp, mant);
 360     }
 361     arch_fpop_end(env);
 362     return ((uint64_t)RdV << 32) | PeV;
 363 }
 364
 365 uint64_t HELPER(sfinvsqrta)(CPUHexagonState *env, float32 RsV)
 366 {
 367     int PeV = 0;
 368     float32 RdV;
 369     int idx;
 370     int adjust;
 371     int mant;
 372     int exp;
 373
 374     arch_fpop_start(env);
 375     if (arch_sf_invsqrt_common(&RsV, &RdV, &adjust, &env->fp_status)) {
 376         PeV = adjust;
 377         idx = (RsV >> 17) & 0x7f;
 378         mant = (invsqrt_lookup_table[idx] << 15);
 379         exp = SF_BIAS - ((float32_getexp(RsV) - SF_BIAS) >> 1) - 1;
 380         RdV = build_float32(extract32(RsV, 31, 1), exp, mant);
 381     }
 382     arch_fpop_end(env);
 383     return ((uint64_t)RdV << 32) | PeV;
 384 }
 385
 386 int64_t HELPER(vacsh_val)(CPUHexagonState *env,
 387                            int64_t RxxV, int64_t RssV, int64_t RttV)
 388 {
 389     for (int i = 0; i < 4; i++) {
 390         int xv = sextract64(RxxV, i * 16, 16);
 391         int sv = sextract64(RssV, i * 16, 16);
 392         int tv = sextract64(RttV, i * 16, 16);
 393         int max;
 394         xv = xv + tv;
 395         sv = sv - tv;
 396         max = xv > sv ? xv : sv;
 397         /* Note that fSATH can set the OVF bit in usr */
 398         RxxV = deposit64(RxxV, i * 16, 16, fSATH(max));
 399     }
 400     return RxxV;
 401 }
 402
 403 int32_t HELPER(vacsh_pred)(CPUHexagonState *env,
 404                            int64_t RxxV, int64_t RssV, int64_t RttV)
 405 {
 406     int32_t PeV = 0;
 407     for (int i = 0; i < 4; i++) {
 408         int xv = sextract64(RxxV, i * 16, 16);
 409         int sv = sextract64(RssV, i * 16, 16);
 410         int tv = sextract64(RttV, i * 16, 16);
 411         xv = xv + tv;
 412         sv = sv - tv;
 413         PeV = deposit32(PeV, i * 2, 1, (xv > sv));
 414         PeV = deposit32(PeV, i * 2 + 1, 1, (xv > sv));
 415     }
 416     return PeV;
 417 }
 418
 419 static void probe_store(CPUHexagonState *env, int slot, int mmu_idx,
 420                         bool is_predicated)
 421 {
 422     if (!is_predicated || !(env->slot_cancelled & (1 << slot))) {
 423         size1u_t width = env->mem_log_stores[slot].width;
 424         target_ulong va = env->mem_log_stores[slot].va;
 425         uintptr_t ra = GETPC();
 426         probe_write(env, va, width, mmu_idx, ra);
 427     }
 428 }
 429
 430 /*
 431  * Called from a mem_noshuf packet to make sure the load doesn't
 432  * raise an exception
 433  */
 434 void HELPER(probe_noshuf_load)(CPUHexagonState *env, target_ulong va,
 435                                int size, int mmu_idx)
 436 {
 437     uintptr_t retaddr = GETPC();
 438     probe_read(env, va, size, mmu_idx, retaddr);
 439 }
 440
 441 /* Called during packet commit when there are two scalar stores */
 442 void HELPER(probe_pkt_scalar_store_s0)(CPUHexagonState *env, int args)
 443 {
 444     int mmu_idx = FIELD_EX32(args, PROBE_PKT_SCALAR_STORE_S0, MMU_IDX);
 445     bool is_predicated =
 446         FIELD_EX32(args, PROBE_PKT_SCALAR_STORE_S0, IS_PREDICATED);
 447     probe_store(env, 0, mmu_idx, is_predicated);
 448 }
 449
 450 void HELPER(probe_hvx_stores)(CPUHexagonState *env, int mmu_idx)
 451 {
 452     uintptr_t retaddr = GETPC();
 453     int i;
 454
 455     /* Normal (possibly masked) vector store */
 456     for (i = 0; i < VSTORES_MAX; i++) {
 457         if (env->vstore_pending[i]) {
 458             target_ulong va = env->vstore[i].va;
 459             int size = env->vstore[i].size;
 460             for (int j = 0; j < size; j++) {
 461                 if (test_bit(j, env->vstore[i].mask)) {
 462                     probe_write(env, va + j, 1, mmu_idx, retaddr);
 463                 }
 464             }
 465         }
 466     }
 467
 468     /* Scatter store */
 469     if (env->vtcm_pending) {
 470         if (env->vtcm_log.op) {
 471             /* Need to perform the scatter read/modify/write at commit time */
 472             if (env->vtcm_log.op_size == 2) {
 473                 SCATTER_OP_PROBE_MEM(size2u_t, mmu_idx, retaddr);
 474             } else if (env->vtcm_log.op_size == 4) {
 475                 /* Word Scatter += */
 476                 SCATTER_OP_PROBE_MEM(size4u_t, mmu_idx, retaddr);
 477             } else {
 478                 g_assert_not_reached();
 479             }
 480         } else {
 481             for (int i = 0; i < sizeof(MMVector); i++) {
 482                 if (test_bit(i, env->vtcm_log.mask)) {
 483                     probe_write(env, env->vtcm_log.va[i], 1, mmu_idx, retaddr);
 484                 }
 485
 486             }
 487         }
 488     }
 489 }
 490
 491 void HELPER(probe_pkt_scalar_hvx_stores)(CPUHexagonState *env, int mask,
 492                                          int mmu_idx)
 493 {
 494     bool has_st0 = FIELD_EX32(mask, PROBE_PKT_SCALAR_HVX_STORES, HAS_ST0);
 495     bool has_st1 = FIELD_EX32(mask, PROBE_PKT_SCALAR_HVX_STORES, HAS_ST1);
 496     bool has_hvx_stores =
 497         FIELD_EX32(mask, PROBE_PKT_SCALAR_HVX_STORES, HAS_HVX_STORES);
 498     bool s0_is_pred = FIELD_EX32(mask, PROBE_PKT_SCALAR_HVX_STORES, S0_IS_PRED);
 499     bool s1_is_pred = FIELD_EX32(mask, PROBE_PKT_SCALAR_HVX_STORES, S1_IS_PRED);
 500
 501     if (has_st0) {
 502         probe_store(env, 0, mmu_idx, s0_is_pred);
 503     }
 504     if (has_st1) {
 505         probe_store(env, 1, mmu_idx, s1_is_pred);
 506     }
 507     if (has_hvx_stores) {
 508         HELPER(probe_hvx_stores)(env, mmu_idx);
 509     }
 510 }
 511
 512 /*
 513  * mem_noshuf
 514  * Section 5.5 of the Hexagon V67 Programmer's Reference Manual
 515  *
 516  * If the load is in slot 0 and there is a store in slot1 (that
 517  * wasn't cancelled), we have to do the store first.
 518  */
 519 static void check_noshuf(CPUHexagonState *env, uint32_t slot,
 520                          target_ulong vaddr, int size)
 521 {
 522     if (slot == 0 && env->pkt_has_store_s1 &&
 523         ((env->slot_cancelled & (1 << 1)) == 0)) {
 524         HELPER(probe_noshuf_load)(env, vaddr, size, MMU_USER_IDX);
 525         HELPER(commit_store)(env, 1);
 526     }
 527 }
 528
 529 uint8_t mem_load1(CPUHexagonState *env, uint32_t slot, target_ulong vaddr)
 530 {
 531     uintptr_t ra = GETPC();
 532     check_noshuf(env, slot, vaddr, 1);
 533     return cpu_ldub_data_ra(env, vaddr, ra);
 534 }
 535
 536 uint16_t mem_load2(CPUHexagonState *env, uint32_t slot, target_ulong vaddr)
 537 {
 538     uintptr_t ra = GETPC();
 539     check_noshuf(env, slot, vaddr, 2);
 540     return cpu_lduw_data_ra(env, vaddr, ra);
 541 }
 542
 543 uint32_t mem_load4(CPUHexagonState *env, uint32_t slot, target_ulong vaddr)
 544 {
 545     uintptr_t ra = GETPC();
 546     check_noshuf(env, slot, vaddr, 4);
 547     return cpu_ldl_data_ra(env, vaddr, ra);
 548 }
 549
 550 uint64_t mem_load8(CPUHexagonState *env, uint32_t slot, target_ulong vaddr)
 551 {
 552     uintptr_t ra = GETPC();
 553     check_noshuf(env, slot, vaddr, 8);
 554     return cpu_ldq_data_ra(env, vaddr, ra);
 555 }
 556
 557 /* Floating point */
 558 float64 HELPER(conv_sf2df)(CPUHexagonState *env, float32 RsV)
 559 {
 560     float64 out_f64;
 561     arch_fpop_start(env);
 562     out_f64 = float32_to_float64(RsV, &env->fp_status);
 563     arch_fpop_end(env);
 564     return out_f64;
 565 }
 566
 567 float32 HELPER(conv_df2sf)(CPUHexagonState *env, float64 RssV)
 568 {
 569     float32 out_f32;
 570     arch_fpop_start(env);
 571     out_f32 = float64_to_float32(RssV, &env->fp_status);
 572     arch_fpop_end(env);
 573     return out_f32;
 574 }
 575
 576 float32 HELPER(conv_uw2sf)(CPUHexagonState *env, int32_t RsV)
 577 {
 578     float32 RdV;
 579     arch_fpop_start(env);
 580     RdV = uint32_to_float32(RsV, &env->fp_status);
 581     arch_fpop_end(env);
 582     return RdV;
 583 }
 584
 585 float64 HELPER(conv_uw2df)(CPUHexagonState *env, int32_t RsV)
 586 {
 587     float64 RddV;
 588     arch_fpop_start(env);
 589     RddV = uint32_to_float64(RsV, &env->fp_status);
 590     arch_fpop_end(env);
 591     return RddV;
 592 }
 593
 594 float32 HELPER(conv_w2sf)(CPUHexagonState *env, int32_t RsV)
 595 {
 596     float32 RdV;
 597     arch_fpop_start(env);
 598     RdV = int32_to_float32(RsV, &env->fp_status);
 599     arch_fpop_end(env);
 600     return RdV;
 601 }
 602
 603 float64 HELPER(conv_w2df)(CPUHexagonState *env, int32_t RsV)
 604 {
 605     float64 RddV;
 606     arch_fpop_start(env);
 607     RddV = int32_to_float64(RsV, &env->fp_status);
 608     arch_fpop_end(env);
 609     return RddV;
 610 }
 611
 612 float32 HELPER(conv_ud2sf)(CPUHexagonState *env, int64_t RssV)
 613 {
 614     float32 RdV;
 615     arch_fpop_start(env);
 616     RdV = uint64_to_float32(RssV, &env->fp_status);
 617     arch_fpop_end(env);
 618     return RdV;
 619 }
 620
 621 float64 HELPER(conv_ud2df)(CPUHexagonState *env, int64_t RssV)
 622 {
 623     float64 RddV;
 624     arch_fpop_start(env);
 625     RddV = uint64_to_float64(RssV, &env->fp_status);
 626     arch_fpop_end(env);
 627     return RddV;
 628 }
 629
 630 float32 HELPER(conv_d2sf)(CPUHexagonState *env, int64_t RssV)
 631 {
 632     float32 RdV;
 633     arch_fpop_start(env);
 634     RdV = int64_to_float32(RssV, &env->fp_status);
 635     arch_fpop_end(env);
 636     return RdV;
 637 }
 638
 639 float64 HELPER(conv_d2df)(CPUHexagonState *env, int64_t RssV)
 640 {
 641     float64 RddV;
 642     arch_fpop_start(env);
 643     RddV = int64_to_float64(RssV, &env->fp_status);
 644     arch_fpop_end(env);
 645     return RddV;
 646 }
 647
 648 uint32_t HELPER(conv_sf2uw)(CPUHexagonState *env, float32 RsV)
 649 {
 650     uint32_t RdV;
 651     arch_fpop_start(env);
 652     /* Hexagon checks the sign before rounding */
 653     if (float32_is_neg(RsV) && !float32_is_any_nan(RsV)) {
 654         float_raise(float_flag_invalid, &env->fp_status);
 655         RdV = 0;
 656     } else {
 657         RdV = float32_to_uint32(RsV, &env->fp_status);
 658     }
 659     arch_fpop_end(env);
 660     return RdV;
 661 }
 662
 663 int32_t HELPER(conv_sf2w)(CPUHexagonState *env, float32 RsV)
 664 {
 665     int32_t RdV;
 666     arch_fpop_start(env);
 667     /* Hexagon returns -1 for NaN */
 668     if (float32_is_any_nan(RsV)) {
 669         float_raise(float_flag_invalid, &env->fp_status);
 670         RdV = -1;
 671     } else {
 672         RdV = float32_to_int32(RsV, &env->fp_status);
 673     }
 674     arch_fpop_end(env);
 675     return RdV;
 676 }
 677
 678 uint64_t HELPER(conv_sf2ud)(CPUHexagonState *env, float32 RsV)
 679 {
 680     uint64_t RddV;
 681     arch_fpop_start(env);
 682     /* Hexagon checks the sign before rounding */
 683     if (float32_is_neg(RsV) && !float32_is_any_nan(RsV)) {
 684         float_raise(float_flag_invalid, &env->fp_status);
 685         RddV = 0;
 686     } else {
 687         RddV = float32_to_uint64(RsV, &env->fp_status);
 688     }
 689     arch_fpop_end(env);
 690     return RddV;
 691 }
 692
 693 int64_t HELPER(conv_sf2d)(CPUHexagonState *env, float32 RsV)
 694 {
 695     int64_t RddV;
 696     arch_fpop_start(env);
 697     /* Hexagon returns -1 for NaN */
 698     if (float32_is_any_nan(RsV)) {
 699         float_raise(float_flag_invalid, &env->fp_status);
 700         RddV = -1;
 701     } else {
 702         RddV = float32_to_int64(RsV, &env->fp_status);
 703     }
 704     arch_fpop_end(env);
 705     return RddV;
 706 }
 707
 708 uint32_t HELPER(conv_df2uw)(CPUHexagonState *env, float64 RssV)
 709 {
 710     uint32_t RdV;
 711     arch_fpop_start(env);
 712     /* Hexagon checks the sign before rounding */
 713     if (float64_is_neg(RssV) && !float64_is_any_nan(RssV)) {
 714         float_raise(float_flag_invalid, &env->fp_status);
 715         RdV = 0;
 716     } else {
 717         RdV = float64_to_uint32(RssV, &env->fp_status);
 718     }
 719     arch_fpop_end(env);
 720     return RdV;
 721 }
 722
 723 int32_t HELPER(conv_df2w)(CPUHexagonState *env, float64 RssV)
 724 {
 725     int32_t RdV;
 726     arch_fpop_start(env);
 727     /* Hexagon returns -1 for NaN */
 728     if (float64_is_any_nan(RssV)) {
 729         float_raise(float_flag_invalid, &env->fp_status);
 730         RdV = -1;
 731     } else {
 732         RdV = float64_to_int32(RssV, &env->fp_status);
 733     }
 734     arch_fpop_end(env);
 735     return RdV;
 736 }
 737
 738 uint64_t HELPER(conv_df2ud)(CPUHexagonState *env, float64 RssV)
 739 {
 740     uint64_t RddV;
 741     arch_fpop_start(env);
 742     /* Hexagon checks the sign before rounding */
 743     if (float64_is_neg(RssV) && !float64_is_any_nan(RssV)) {
 744         float_raise(float_flag_invalid, &env->fp_status);
 745         RddV = 0;
 746     } else {
 747         RddV = float64_to_uint64(RssV, &env->fp_status);
 748     }
 749     arch_fpop_end(env);
 750     return RddV;
 751 }
 752
 753 int64_t HELPER(conv_df2d)(CPUHexagonState *env, float64 RssV)
 754 {
 755     int64_t RddV;
 756     arch_fpop_start(env);
 757     /* Hexagon returns -1 for NaN */
 758     if (float64_is_any_nan(RssV)) {
 759         float_raise(float_flag_invalid, &env->fp_status);
 760         RddV = -1;
 761     } else {
 762         RddV = float64_to_int64(RssV, &env->fp_status);
 763     }
 764     arch_fpop_end(env);
 765     return RddV;
 766 }
 767
 768 uint32_t HELPER(conv_sf2uw_chop)(CPUHexagonState *env, float32 RsV)
 769 {
 770     uint32_t RdV;
 771     arch_fpop_start(env);
 772     /* Hexagon checks the sign before rounding */
 773     if (float32_is_neg(RsV) && !float32_is_any_nan(RsV)) {
 774         float_raise(float_flag_invalid, &env->fp_status);
 775         RdV = 0;
 776     } else {
 777         RdV = float32_to_uint32_round_to_zero(RsV, &env->fp_status);
 778     }
 779     arch_fpop_end(env);
 780     return RdV;
 781 }
 782
 783 int32_t HELPER(conv_sf2w_chop)(CPUHexagonState *env, float32 RsV)
 784 {
 785     int32_t RdV;
 786     arch_fpop_start(env);
 787     /* Hexagon returns -1 for NaN */
 788     if (float32_is_any_nan(RsV)) {
 789         float_raise(float_flag_invalid, &env->fp_status);
 790         RdV = -1;
 791     } else {
 792         RdV = float32_to_int32_round_to_zero(RsV, &env->fp_status);
 793     }
 794     arch_fpop_end(env);
 795     return RdV;
 796 }
 797
 798 uint64_t HELPER(conv_sf2ud_chop)(CPUHexagonState *env, float32 RsV)
 799 {
 800     uint64_t RddV;
 801     arch_fpop_start(env);
 802     /* Hexagon checks the sign before rounding */
 803     if (float32_is_neg(RsV) && !float32_is_any_nan(RsV)) {
 804         float_raise(float_flag_invalid, &env->fp_status);
 805         RddV = 0;
 806     } else {
 807         RddV = float32_to_uint64_round_to_zero(RsV, &env->fp_status);
 808     }
 809     arch_fpop_end(env);
 810     return RddV;
 811 }
 812
 813 int64_t HELPER(conv_sf2d_chop)(CPUHexagonState *env, float32 RsV)
 814 {
 815     int64_t RddV;
 816     arch_fpop_start(env);
 817     /* Hexagon returns -1 for NaN */
 818     if (float32_is_any_nan(RsV)) {
 819         float_raise(float_flag_invalid, &env->fp_status);
 820         RddV = -1;
 821     } else {
 822         RddV = float32_to_int64_round_to_zero(RsV, &env->fp_status);
 823     }
 824     arch_fpop_end(env);
 825     return RddV;
 826 }
 827
 828 uint32_t HELPER(conv_df2uw_chop)(CPUHexagonState *env, float64 RssV)
 829 {
 830     uint32_t RdV;
 831     arch_fpop_start(env);
 832     /* Hexagon checks the sign before rounding */
 833     if (float64_is_neg(RssV) && !float64_is_any_nan(RssV)) {
 834         float_raise(float_flag_invalid, &env->fp_status);
 835         RdV = 0;
 836     } else {
 837         RdV = float64_to_uint32_round_to_zero(RssV, &env->fp_status);
 838     }
 839     arch_fpop_end(env);
 840     return RdV;
 841 }
 842
 843 int32_t HELPER(conv_df2w_chop)(CPUHexagonState *env, float64 RssV)
 844 {
 845     int32_t RdV;
 846     arch_fpop_start(env);
 847     /* Hexagon returns -1 for NaN */
 848     if (float64_is_any_nan(RssV)) {
 849         float_raise(float_flag_invalid, &env->fp_status);
 850         RdV = -1;
 851     } else {
 852         RdV = float64_to_int32_round_to_zero(RssV, &env->fp_status);
 853     }
 854     arch_fpop_end(env);
 855     return RdV;
 856 }
 857
 858 uint64_t HELPER(conv_df2ud_chop)(CPUHexagonState *env, float64 RssV)
 859 {
 860     uint64_t RddV;
 861     arch_fpop_start(env);
 862     /* Hexagon checks the sign before rounding */
 863     if (float64_is_neg(RssV) && !float64_is_any_nan(RssV)) {
 864         float_raise(float_flag_invalid, &env->fp_status);
 865         RddV = 0;
 866     } else {
 867         RddV = float64_to_uint64_round_to_zero(RssV, &env->fp_status);
 868     }
 869     arch_fpop_end(env);
 870     return RddV;
 871 }
 872
 873 int64_t HELPER(conv_df2d_chop)(CPUHexagonState *env, float64 RssV)
 874 {
 875     int64_t RddV;
 876     arch_fpop_start(env);
 877     /* Hexagon returns -1 for NaN */
 878     if (float64_is_any_nan(RssV)) {
 879         float_raise(float_flag_invalid, &env->fp_status);
 880         RddV = -1;
 881     } else {
 882         RddV = float64_to_int64_round_to_zero(RssV, &env->fp_status);
 883     }
 884     arch_fpop_end(env);
 885     return RddV;
 886 }
 887
 888 float32 HELPER(sfadd)(CPUHexagonState *env, float32 RsV, float32 RtV)
 889 {
 890     float32 RdV;
 891     arch_fpop_start(env);
 892     RdV = float32_add(RsV, RtV, &env->fp_status);
 893     arch_fpop_end(env);
 894     return RdV;
 895 }
 896
 897 float32 HELPER(sfsub)(CPUHexagonState *env, float32 RsV, float32 RtV)
 898 {
 899     float32 RdV;
 900     arch_fpop_start(env);
 901     RdV = float32_sub(RsV, RtV, &env->fp_status);
 902     arch_fpop_end(env);
 903     return RdV;
 904 }
 905
 906 int32_t HELPER(sfcmpeq)(CPUHexagonState *env, float32 RsV, float32 RtV)
 907 {
 908     int32_t PdV;
 909     arch_fpop_start(env);
 910     PdV = f8BITSOF(float32_eq_quiet(RsV, RtV, &env->fp_status));
 911     arch_fpop_end(env);
 912     return PdV;
 913 }
 914
 915 int32_t HELPER(sfcmpgt)(CPUHexagonState *env, float32 RsV, float32 RtV)
 916 {
 917     int cmp;
 918     int32_t PdV;
 919     arch_fpop_start(env);
 920     cmp = float32_compare_quiet(RsV, RtV, &env->fp_status);
 921     PdV = f8BITSOF(cmp == float_relation_greater);
 922     arch_fpop_end(env);
 923     return PdV;
 924 }
 925
 926 int32_t HELPER(sfcmpge)(CPUHexagonState *env, float32 RsV, float32 RtV)
 927 {
 928     int cmp;
 929     int32_t PdV;
 930     arch_fpop_start(env);
 931     cmp = float32_compare_quiet(RsV, RtV, &env->fp_status);
 932     PdV = f8BITSOF(cmp == float_relation_greater ||
 933                    cmp == float_relation_equal);
 934     arch_fpop_end(env);
 935     return PdV;
 936 }
 937
 938 int32_t HELPER(sfcmpuo)(CPUHexagonState *env, float32 RsV, float32 RtV)
 939 {
 940     int32_t PdV;
 941     arch_fpop_start(env);
 942     PdV = f8BITSOF(float32_unordered_quiet(RsV, RtV, &env->fp_status));
 943     arch_fpop_end(env);
 944     return PdV;
 945 }
 946
 947 float32 HELPER(sfmax)(CPUHexagonState *env, float32 RsV, float32 RtV)
 948 {
 949     float32 RdV;
 950     arch_fpop_start(env);
 951     RdV = float32_maximum_number(RsV, RtV, &env->fp_status);
 952     arch_fpop_end(env);
 953     return RdV;
 954 }
 955
 956 float32 HELPER(sfmin)(CPUHexagonState *env, float32 RsV, float32 RtV)
 957 {
 958     float32 RdV;
 959     arch_fpop_start(env);
 960     RdV = float32_minimum_number(RsV, RtV, &env->fp_status);
 961     arch_fpop_end(env);
 962     return RdV;
 963 }
 964
 965 int32_t HELPER(sfclass)(CPUHexagonState *env, float32 RsV, int32_t uiV)
 966 {
 967     int32_t PdV = 0;
 968     arch_fpop_start(env);
 969     if (fGETBIT(0, uiV) && float32_is_zero(RsV)) {
 970         PdV = 0xff;
 971     }
 972     if (fGETBIT(1, uiV) && float32_is_normal(RsV)) {
 973         PdV = 0xff;
 974     }
 975     if (fGETBIT(2, uiV) && float32_is_denormal(RsV)) {
 976         PdV = 0xff;
 977     }
 978     if (fGETBIT(3, uiV) && float32_is_infinity(RsV)) {
 979         PdV = 0xff;
 980     }
 981     if (fGETBIT(4, uiV) && float32_is_any_nan(RsV)) {
 982         PdV = 0xff;
 983     }
 984     set_float_exception_flags(0, &env->fp_status);
 985     arch_fpop_end(env);
 986     return PdV;
 987 }
 988
 989 float32 HELPER(sffixupn)(CPUHexagonState *env, float32 RsV, float32 RtV)
 990 {
 991     float32 RdV = 0;
 992     int adjust;
 993     arch_fpop_start(env);
 994     arch_sf_recip_common(&RsV, &RtV, &RdV, &adjust, &env->fp_status);
 995     RdV = RsV;
 996     arch_fpop_end(env);
 997     return RdV;
 998 }
 999
1000 float32 HELPER(sffixupd)(CPUHexagonState *env, float32 RsV, float32 RtV)
1001 {
1002     float32 RdV = 0;
1003     int adjust;
1004     arch_fpop_start(env);
1005     arch_sf_recip_common(&RsV, &RtV, &RdV, &adjust, &env->fp_status);
1006     RdV = RtV;
1007     arch_fpop_end(env);
1008     return RdV;
1009 }
1010
1011 float32 HELPER(sffixupr)(CPUHexagonState *env, float32 RsV)
1012 {
1013     float32 RdV = 0;
1014     int adjust;
1015     arch_fpop_start(env);
1016     arch_sf_invsqrt_common(&RsV, &RdV, &adjust, &env->fp_status);
1017     RdV = RsV;
1018     arch_fpop_end(env);
1019     return RdV;
1020 }
1021
1022 float64 HELPER(dfadd)(CPUHexagonState *env, float64 RssV, float64 RttV)
1023 {
1024     float64 RddV;
1025     arch_fpop_start(env);
1026     RddV = float64_add(RssV, RttV, &env->fp_status);
1027     arch_fpop_end(env);
1028     return RddV;
1029 }
1030
1031 float64 HELPER(dfsub)(CPUHexagonState *env, float64 RssV, float64 RttV)
1032 {
1033     float64 RddV;
1034     arch_fpop_start(env);
1035     RddV = float64_sub(RssV, RttV, &env->fp_status);
1036     arch_fpop_end(env);
1037     return RddV;
1038 }
1039
1040 float64 HELPER(dfmax)(CPUHexagonState *env, float64 RssV, float64 RttV)
1041 {
1042     float64 RddV;
1043     arch_fpop_start(env);
1044     RddV = float64_maximum_number(RssV, RttV, &env->fp_status);
1045     arch_fpop_end(env);
1046     return RddV;
1047 }
1048
1049 float64 HELPER(dfmin)(CPUHexagonState *env, float64 RssV, float64 RttV)
1050 {
1051     float64 RddV;
1052     arch_fpop_start(env);
1053     RddV = float64_minimum_number(RssV, RttV, &env->fp_status);
1054     arch_fpop_end(env);
1055     return RddV;
1056 }
1057
1058 int32_t HELPER(dfcmpeq)(CPUHexagonState *env, float64 RssV, float64 RttV)
1059 {
1060     int32_t PdV;
1061     arch_fpop_start(env);
1062     PdV = f8BITSOF(float64_eq_quiet(RssV, RttV, &env->fp_status));
1063     arch_fpop_end(env);
1064     return PdV;
1065 }
1066
1067 int32_t HELPER(dfcmpgt)(CPUHexagonState *env, float64 RssV, float64 RttV)
1068 {
1069     int cmp;
1070     int32_t PdV;
1071     arch_fpop_start(env);
1072     cmp = float64_compare_quiet(RssV, RttV, &env->fp_status);
1073     PdV = f8BITSOF(cmp == float_relation_greater);
1074     arch_fpop_end(env);
1075     return PdV;
1076 }
1077
1078 int32_t HELPER(dfcmpge)(CPUHexagonState *env, float64 RssV, float64 RttV)
1079 {
1080     int cmp;
1081     int32_t PdV;
1082     arch_fpop_start(env);
1083     cmp = float64_compare_quiet(RssV, RttV, &env->fp_status);
1084     PdV = f8BITSOF(cmp == float_relation_greater ||
1085                    cmp == float_relation_equal);
1086     arch_fpop_end(env);
1087     return PdV;
1088 }
1089
1090 int32_t HELPER(dfcmpuo)(CPUHexagonState *env, float64 RssV, float64 RttV)
1091 {
1092     int32_t PdV;
1093     arch_fpop_start(env);
1094     PdV = f8BITSOF(float64_unordered_quiet(RssV, RttV, &env->fp_status));
1095     arch_fpop_end(env);
1096     return PdV;
1097 }
1098
1099 int32_t HELPER(dfclass)(CPUHexagonState *env, float64 RssV, int32_t uiV)
1100 {
1101     int32_t PdV = 0;
1102     arch_fpop_start(env);
1103     if (fGETBIT(0, uiV) && float64_is_zero(RssV)) {
1104         PdV = 0xff;
1105     }
1106     if (fGETBIT(1, uiV) && float64_is_normal(RssV)) {
1107         PdV = 0xff;
1108     }
1109     if (fGETBIT(2, uiV) && float64_is_denormal(RssV)) {
1110         PdV = 0xff;
1111     }
1112     if (fGETBIT(3, uiV) && float64_is_infinity(RssV)) {
1113         PdV = 0xff;
1114     }
1115     if (fGETBIT(4, uiV) && float64_is_any_nan(RssV)) {
1116         PdV = 0xff;
1117     }
1118     set_float_exception_flags(0, &env->fp_status);
1119     arch_fpop_end(env);
1120     return PdV;
1121 }
1122
1123 float32 HELPER(sfmpy)(CPUHexagonState *env, float32 RsV, float32 RtV)
1124 {
1125     float32 RdV;
1126     arch_fpop_start(env);
1127     RdV = internal_mpyf(RsV, RtV, &env->fp_status);
1128     arch_fpop_end(env);
1129     return RdV;
1130 }
1131
1132 float32 HELPER(sffma)(CPUHexagonState *env, float32 RxV,
1133                       float32 RsV, float32 RtV)
1134 {
1135     arch_fpop_start(env);
1136     RxV = internal_fmafx(RsV, RtV, RxV, 0, &env->fp_status);
1137     arch_fpop_end(env);
1138     return RxV;
1139 }
1140
1141 static bool is_zero_prod(float32 a, float32 b)
1142 {
1143     return ((float32_is_zero(a) && is_finite(b)) ||
1144             (float32_is_zero(b) && is_finite(a)));
1145 }
1146
1147 static float32 check_nan(float32 dst, float32 x, float_status *fp_status)
1148 {
1149     float32 ret = dst;
1150     if (float32_is_any_nan(x)) {
1151         if (extract32(x, 22, 1) == 0) {
1152             float_raise(float_flag_invalid, fp_status);
1153         }
1154         ret = make_float32(0xffffffff);    /* nan */
1155     }
1156     return ret;
1157 }
1158
1159 float32 HELPER(sffma_sc)(CPUHexagonState *env, float32 RxV,
1160                          float32 RsV, float32 RtV, float32 PuV)
1161 {
1162     size4s_t tmp;
1163     arch_fpop_start(env);
1164     RxV = check_nan(RxV, RxV, &env->fp_status);
1165     RxV = check_nan(RxV, RsV, &env->fp_status);
1166     RxV = check_nan(RxV, RtV, &env->fp_status);
1167     tmp = internal_fmafx(RsV, RtV, RxV, fSXTN(8, 64, PuV), &env->fp_status);
1168     if (!(float32_is_zero(RxV) && is_zero_prod(RsV, RtV))) {
1169         RxV = tmp;
1170     }
1171     arch_fpop_end(env);
1172     return RxV;
1173 }
1174
1175 float32 HELPER(sffms)(CPUHexagonState *env, float32 RxV,
1176                       float32 RsV, float32 RtV)
1177 {
1178     float32 neg_RsV;
1179     arch_fpop_start(env);
1180     neg_RsV = float32_set_sign(RsV, float32_is_neg(RsV) ? 0 : 1);
1181     RxV = internal_fmafx(neg_RsV, RtV, RxV, 0, &env->fp_status);
1182     arch_fpop_end(env);
1183     return RxV;
1184 }
1185
1186 static bool is_inf_prod(int32_t a, int32_t b)
1187 {
1188     return (float32_is_infinity(a) && float32_is_infinity(b)) ||
1189            (float32_is_infinity(a) && is_finite(b) && !float32_is_zero(b)) ||
1190            (float32_is_infinity(b) && is_finite(a) && !float32_is_zero(a));
1191 }
1192
1193 float32 HELPER(sffma_lib)(CPUHexagonState *env, float32 RxV,
1194                           float32 RsV, float32 RtV)
1195 {
1196     bool infinp;
1197     bool infminusinf;
1198     float32 tmp;
1199
1200     arch_fpop_start(env);
1201     set_float_rounding_mode(float_round_nearest_even, &env->fp_status);
1202     infminusinf = float32_is_infinity(RxV) &&
1203                   is_inf_prod(RsV, RtV) &&
1204                   (fGETBIT(31, RsV ^ RxV ^ RtV) != 0);
1205     infinp = float32_is_infinity(RxV) ||
1206              float32_is_infinity(RtV) ||
1207              float32_is_infinity(RsV);
1208     RxV = check_nan(RxV, RxV, &env->fp_status);
1209     RxV = check_nan(RxV, RsV, &env->fp_status);
1210     RxV = check_nan(RxV, RtV, &env->fp_status);
1211     tmp = internal_fmafx(RsV, RtV, RxV, 0, &env->fp_status);
1212     if (!(float32_is_zero(RxV) && is_zero_prod(RsV, RtV))) {
1213         RxV = tmp;
1214     }
1215     set_float_exception_flags(0, &env->fp_status);
1216     if (float32_is_infinity(RxV) && !infinp) {
1217         RxV = RxV - 1;
1218     }
1219     if (infminusinf) {
1220         RxV = 0;
1221     }
1222     arch_fpop_end(env);
1223     return RxV;
1224 }
1225
1226 float32 HELPER(sffms_lib)(CPUHexagonState *env, float32 RxV,
1227                           float32 RsV, float32 RtV)
1228 {
1229     bool infinp;
1230     bool infminusinf;
1231     float32 tmp;
1232
1233     arch_fpop_start(env);
1234     set_float_rounding_mode(float_round_nearest_even, &env->fp_status);
1235     infminusinf = float32_is_infinity(RxV) &&
1236                   is_inf_prod(RsV, RtV) &&
1237                   (fGETBIT(31, RsV ^ RxV ^ RtV) == 0);
1238     infinp = float32_is_infinity(RxV) ||
1239              float32_is_infinity(RtV) ||
1240              float32_is_infinity(RsV);
1241     RxV = check_nan(RxV, RxV, &env->fp_status);
1242     RxV = check_nan(RxV, RsV, &env->fp_status);
1243     RxV = check_nan(RxV, RtV, &env->fp_status);
1244     float32 minus_RsV = float32_sub(float32_zero, RsV, &env->fp_status);
1245     tmp = internal_fmafx(minus_RsV, RtV, RxV, 0, &env->fp_status);
1246     if (!(float32_is_zero(RxV) && is_zero_prod(RsV, RtV))) {
1247         RxV = tmp;
1248     }
1249     set_float_exception_flags(0, &env->fp_status);
1250     if (float32_is_infinity(RxV) && !infinp) {
1251         RxV = RxV - 1;
1252     }
1253     if (infminusinf) {
1254         RxV = 0;
1255     }
1256     arch_fpop_end(env);
1257     return RxV;
1258 }
1259
1260 float64 HELPER(dfmpyfix)(CPUHexagonState *env, float64 RssV, float64 RttV)
1261 {
1262     int64_t RddV;
1263     arch_fpop_start(env);
1264     if (float64_is_denormal(RssV) &&
1265         (float64_getexp(RttV) >= 512) &&
1266         float64_is_normal(RttV)) {
1267         RddV = float64_mul(RssV, make_float64(0x4330000000000000),
1268                            &env->fp_status);
1269     } else if (float64_is_denormal(RttV) &&
1270                (float64_getexp(RssV) >= 512) &&
1271                float64_is_normal(RssV)) {
1272         RddV = float64_mul(RssV, make_float64(0x3cb0000000000000),
1273                            &env->fp_status);
1274     } else {
1275         RddV = RssV;
1276     }
1277     arch_fpop_end(env);
1278     return RddV;
1279 }
1280
1281 float64 HELPER(dfmpyhh)(CPUHexagonState *env, float64 RxxV,
1282                         float64 RssV, float64 RttV)
1283 {
1284     arch_fpop_start(env);
1285     RxxV = internal_mpyhh(RssV, RttV, RxxV, &env->fp_status);
1286     arch_fpop_end(env);
1287     return RxxV;
1288 }
1289
1290 /* Histogram instructions */
1291
1292 void HELPER(vhist)(CPUHexagonState *env)
1293 {
1294     MMVector *input = &env->tmp_VRegs[0];
1295
1296     for (int lane = 0; lane < 8; lane++) {
1297         for (int i = 0; i < sizeof(MMVector) / 8; ++i) {
1298             unsigned char value = input->ub[(sizeof(MMVector) / 8) * lane + i];
1299             unsigned char regno = value >> 3;
1300             unsigned char element = value & 7;
1301
1302             env->VRegs[regno].uh[(sizeof(MMVector) / 16) * lane + element]++;
1303         }
1304     }
1305 }
1306
1307 void HELPER(vhistq)(CPUHexagonState *env)
1308 {
1309     MMVector *input = &env->tmp_VRegs[0];
1310
1311     for (int lane = 0; lane < 8; lane++) {
1312         for (int i = 0; i < sizeof(MMVector) / 8; ++i) {
1313             unsigned char value = input->ub[(sizeof(MMVector) / 8) * lane + i];
1314             unsigned char regno = value >> 3;
1315             unsigned char element = value & 7;
1316
1317             if (fGETQBIT(env->qtmp, sizeof(MMVector) / 8 * lane + i)) {
1318                 env->VRegs[regno].uh[
1319                     (sizeof(MMVector) / 16) * lane + element]++;
1320             }
1321         }
1322     }
1323 }
1324
1325 void HELPER(vwhist256)(CPUHexagonState *env)
1326 {
1327     MMVector *input = &env->tmp_VRegs[0];
1328
1329     for (int i = 0; i < (sizeof(MMVector) / 2); i++) {
1330         unsigned int bucket = fGETUBYTE(0, input->h[i]);
1331         unsigned int weight = fGETUBYTE(1, input->h[i]);
1332         unsigned int vindex = (bucket >> 3) & 0x1F;
1333         unsigned int elindex = ((i >> 0) & (~7)) | ((bucket >> 0) & 7);
1334
1335         env->VRegs[vindex].uh[elindex] =
1336             env->VRegs[vindex].uh[elindex] + weight;
1337     }
1338 }
1339
1340 void HELPER(vwhist256q)(CPUHexagonState *env)
1341 {
1342     MMVector *input = &env->tmp_VRegs[0];
1343
1344     for (int i = 0; i < (sizeof(MMVector) / 2); i++) {
1345         unsigned int bucket = fGETUBYTE(0, input->h[i]);
1346         unsigned int weight = fGETUBYTE(1, input->h[i]);
1347         unsigned int vindex = (bucket >> 3) & 0x1F;
1348         unsigned int elindex = ((i >> 0) & (~7)) | ((bucket >> 0) & 7);
1349
1350         if (fGETQBIT(env->qtmp, 2 * i)) {
1351             env->VRegs[vindex].uh[elindex] =
1352                 env->VRegs[vindex].uh[elindex] + weight;
1353         }
1354     }
1355 }
1356
1357 void HELPER(vwhist256_sat)(CPUHexagonState *env)
1358 {
1359     MMVector *input = &env->tmp_VRegs[0];
1360
1361     for (int i = 0; i < (sizeof(MMVector) / 2); i++) {
1362         unsigned int bucket = fGETUBYTE(0, input->h[i]);
1363         unsigned int weight = fGETUBYTE(1, input->h[i]);
1364         unsigned int vindex = (bucket >> 3) & 0x1F;
1365         unsigned int elindex = ((i >> 0) & (~7)) | ((bucket >> 0) & 7);
1366
1367         env->VRegs[vindex].uh[elindex] =
1368             fVSATUH(env->VRegs[vindex].uh[elindex] + weight);
1369     }
1370 }
1371
1372 void HELPER(vwhist256q_sat)(CPUHexagonState *env)
1373 {
1374     MMVector *input = &env->tmp_VRegs[0];
1375
1376     for (int i = 0; i < (sizeof(MMVector) / 2); i++) {
1377         unsigned int bucket = fGETUBYTE(0, input->h[i]);
1378         unsigned int weight = fGETUBYTE(1, input->h[i]);
1379         unsigned int vindex = (bucket >> 3) & 0x1F;
1380         unsigned int elindex = ((i >> 0) & (~7)) | ((bucket >> 0) & 7);
1381
1382         if (fGETQBIT(env->qtmp, 2 * i)) {
1383             env->VRegs[vindex].uh[elindex] =
1384                 fVSATUH(env->VRegs[vindex].uh[elindex] + weight);
1385         }
1386     }
1387 }
1388
1389 void HELPER(vwhist128)(CPUHexagonState *env)
1390 {
1391     MMVector *input = &env->tmp_VRegs[0];
1392
1393     for (int i = 0; i < (sizeof(MMVector) / 2); i++) {
1394         unsigned int bucket = fGETUBYTE(0, input->h[i]);
1395         unsigned int weight = fGETUBYTE(1, input->h[i]);
1396         unsigned int vindex = (bucket >> 3) & 0x1F;
1397         unsigned int elindex = ((i >> 1) & (~3)) | ((bucket >> 1) & 3);
1398
1399         env->VRegs[vindex].uw[elindex] =
1400             env->VRegs[vindex].uw[elindex] + weight;
1401     }
1402 }
1403
1404 void HELPER(vwhist128q)(CPUHexagonState *env)
1405 {
1406     MMVector *input = &env->tmp_VRegs[0];
1407
1408     for (int i = 0; i < (sizeof(MMVector) / 2); i++) {
1409         unsigned int bucket = fGETUBYTE(0, input->h[i]);
1410         unsigned int weight = fGETUBYTE(1, input->h[i]);
1411         unsigned int vindex = (bucket >> 3) & 0x1F;
1412         unsigned int elindex = ((i >> 1) & (~3)) | ((bucket >> 1) & 3);
1413
1414         if (fGETQBIT(env->qtmp, 2 * i)) {
1415             env->VRegs[vindex].uw[elindex] =
1416                 env->VRegs[vindex].uw[elindex] + weight;
1417         }
1418     }
1419 }
1420
1421 void HELPER(vwhist128m)(CPUHexagonState *env, int32_t uiV)
1422 {
1423     MMVector *input = &env->tmp_VRegs[0];
1424
1425     for (int i = 0; i < (sizeof(MMVector) / 2); i++) {
1426         unsigned int bucket = fGETUBYTE(0, input->h[i]);
1427         unsigned int weight = fGETUBYTE(1, input->h[i]);
1428         unsigned int vindex = (bucket >> 3) & 0x1F;
1429         unsigned int elindex = ((i >> 1) & (~3)) | ((bucket >> 1) & 3);
1430
1431         if ((bucket & 1) == uiV) {
1432             env->VRegs[vindex].uw[elindex] =
1433                 env->VRegs[vindex].uw[elindex] + weight;
1434         }
1435     }
1436 }
1437
1438 void HELPER(vwhist128qm)(CPUHexagonState *env, int32_t uiV)
1439 {
1440     MMVector *input = &env->tmp_VRegs[0];
1441
1442     for (int i = 0; i < (sizeof(MMVector) / 2); i++) {
1443         unsigned int bucket = fGETUBYTE(0, input->h[i]);
1444         unsigned int weight = fGETUBYTE(1, input->h[i]);
1445         unsigned int vindex = (bucket >> 3) & 0x1F;
1446         unsigned int elindex = ((i >> 1) & (~3)) | ((bucket >> 1) & 3);
1447
1448         if (((bucket & 1) == uiV) && fGETQBIT(env->qtmp, 2 * i)) {
1449             env->VRegs[vindex].uw[elindex] =
1450                 env->VRegs[vindex].uw[elindex] + weight;
1451         }
1452     }
1453 }
1454
1455 /* These macros can be referenced in the generated helper functions */
1456 #define warn(...) /* Nothing */
1457 #define fatal(...) g_assert_not_reached();
1458
1459 #define BOGUS_HELPER(tag) \
1460     printf("ERROR: bogus helper: " #tag "\n")
1461
1462 #include "helper_funcs_generated.c.inc"