| with some routines to assist in this task (those are marked
| static and cannot be called from user programs)
*/
-#define MAXABS (INT_MAX - 2)
+
+// Integers above 2^24 do not have unique representations in
+// 32-bit floats ie with 24 bits of precision. We use maxAbsoluteInt
+// to check that float values can be transformed into an in-range
+// 32-bit integer. There is no need to ensure we are within the range
+// of ints with exact floating-point representations. However, we should
+// reject all floats above that which converts to an in-range 32-bit integer.
+const float maxAbsoluteInt = nextafterf(float(INT_MAX), 0.f); // NOLINT(cert-err58-cpp)
#ifndef SQR
# define SQR(x) ((x) * (x))
{
lf = *lfp * *precision - 0.5;
}
- if (std::fabs(lf) > MAXABS)
+ if (std::fabs(lf) > maxAbsoluteInt)
{
/* scaling would cause overflow */
errval = 0;
{
lf = *lfp * *precision - 0.5;
}
- if (std::fabs(lf) > MAXABS)
+ if (std::fabs(lf) > maxAbsoluteInt)
{
/* scaling would cause overflow */
errval = 0;
{
lf = *lfp * *precision - 0.5;
}
- if (std::abs(lf) > MAXABS)
+ if (std::abs(lf) > maxAbsoluteInt)
{
/* scaling would cause overflow */
errval = 0;
return 0;
}
- if (static_cast<float>(maxint[0]) - static_cast<float>(minint[0]) >= MAXABS
- || static_cast<float>(maxint[1]) - static_cast<float>(minint[1]) >= MAXABS
- || static_cast<float>(maxint[2]) - static_cast<float>(minint[2]) >= MAXABS)
+ if (static_cast<float>(maxint[0]) - static_cast<float>(minint[0]) >= maxAbsoluteInt
+ || static_cast<float>(maxint[1]) - static_cast<float>(minint[1]) >= maxAbsoluteInt
+ || static_cast<float>(maxint[2]) - static_cast<float>(minint[2]) >= maxAbsoluteInt)
{
/* turning value in unsigned by subtracting minint
* would cause overflow