From f0b582df6c277d8025595ceeddca2373bd6eb251 Mon Sep 17 00:00:00 2001 From: Eyck-Alexander Jentzsch Date: Thu, 6 Mar 2025 12:55:22 +0100 Subject: [PATCH] corrects ambiguity in frsqrt --- src/vm/fp_functions.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/vm/fp_functions.cpp b/src/vm/fp_functions.cpp index ee5dd6c..20a6610 100644 --- a/src/vm/fp_functions.cpp +++ b/src/vm/fp_functions.cpp @@ -556,7 +556,7 @@ uint64_t constexpr frsqrt7_general(const unsigned s, const unsigned e, const uin int64_t normalized_exp = exp; uint64_t normalized_sig = sig; if(subnormal) { - unsigned nr_leadingzeros = __builtin_clzll(sig) - (64 - s); + signed nr_leadingzeros = __builtin_clzll(sig) - (64 - s); normalized_exp = -nr_leadingzeros; normalized_sig = (sig << (1 + nr_leadingzeros)) & ((1ULL << s) - 1); } @@ -566,8 +566,8 @@ uint64_t constexpr frsqrt7_general(const unsigned s, const unsigned e, const uin // result significand is zero. uint64_t out_sig = rsqrt_table[exp_idx][sig_idx] << (s - 7); // The output exponent equals floor((3*B - 1 - the normalized input exponent) / 2), where B is the exponent bias. - unsigned bias = 1UL << (e - 1); - uint64_t out_exp = (3 * (bias - 1) - 1 - normalized_exp) / 2; + unsigned bias = (1UL << (e - 1)) - 1; + uint64_t out_exp = (3 * bias - 1 - normalized_exp) / 2; // The output sign equals the input sign. return (sign << (s + e)) | (out_exp << s) | out_sig; } @@ -668,8 +668,8 @@ bool frec_general(uint64_t& res, const unsigned s, const unsigned e, const uint6 int64_t normalized_exp = subnormal ? -nr_leadingzeros : exp; uint64_t normalized_sig = subnormal ? ((sig << (1 + nr_leadingzeros)) & ((1ULL << s) - 1)) : sig; unsigned idx = (normalized_sig >> (s - 7)) & 0x7f; - unsigned bias = 1UL << (e - 1); - uint64_t mid_exp = 2 * (bias - 1) - 1 - normalized_exp; + unsigned bias = (1UL << (e - 1)) - 1; + uint64_t mid_exp = 2 * (bias)-1 - normalized_exp; uint64_t mid_sig = rec_table[idx] << (s - 7); uint64_t out_exp = mid_exp;