[FFmpeg-devel] [PATCH] avutil/libm: correct isnan, isinf compat hacks

[Ganesh Ajjanagadde]

isnan and isinf are actually macros as per the standard. In particular,
the existing implementation has incorrect signature. Furthermore, this
results in undefined behavior for e.g double values outside float range
as per the standard.

This patch corrects the undefined behavior for all usage within FFmpeg.
There are some issues with long double, but they are theoretical at the
moment. For instance, the usual culprit for lacking isinf and that uses
this fallback is Microsoft, and on Microsoft, long double = double
anyway. Furthermore, no client of isinf, isnan in the codebase actually
uses long double arguments.

The above issue is harder because long double may be an IEEE 128 bit quad
(very rare) or a 80 bit extended precision value (on GCC/Clang).

Signed-off-by: Ganesh Ajjanagadde <gajjanagadde at gmail.com>
---
 libavutil/libm.h | 50 ++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 48 insertions(+), 2 deletions(-)

diff --git a/libavutil/libm.h b/libavutil/libm.h
index ab5df1b..04d9411 100644
--- a/libavutil/libm.h
+++ b/libavutil/libm.h
@@ -91,23 +91,69 @@ static av_always_inline av_const double hypot(double x, double y)
 #endif /* HAVE_HYPOT */
 
 #if !HAVE_ISINF
-static av_always_inline av_const int isinf(float x)
+#undef isinf
+/* Note: these do not follow the BSD/Apple/GNU convention of returning -1 for
+-Inf, +1 for Inf, 0 otherwise, but merely follow the POSIX/ISO mandated spec of
+returning a non-zero value for +/-Inf, 0 otherwise. */
+static av_always_inline av_const int avpriv_isinff(float x)
 {
     uint32_t v = av_float2int(x);
     if ((v & 0x7f800000) != 0x7f800000)
         return 0;
     return !(v & 0x007fffff);
 }
+
+static av_always_inline av_const int avpriv_isinf(double x)
+{
+    uint64_t v = av_double2int(x);
+    if ((v & 0x7ff0000000000000) != 0x7ff0000000000000)
+        return 0;
+    return !(v & 0x000fffffffffffff);
+}
+
+static av_always_inline av_const int avpriv_isinfl(long double x)
+{
+    // FIXME: just a stub, hard as long double width can vary between platforms
+    // Also currently irrelevant
+    return avpriv_isinf(x);
+}
+
+#define isinf(x)                             \
+    (sizeof(x) == sizeof(float)              \
+        ? avpriv_isinff(x)                   \
+        : sizeof(x) == sizeof(double)        \
+        ? avpriv_isinf(x) : avpriv_isinfl(x))
 #endif /* HAVE_ISINF */
 
 #if !HAVE_ISNAN
-static av_always_inline av_const int isnan(float x)
+static av_always_inline av_const int avpriv_isnanf(float x)
 {
     uint32_t v = av_float2int(x);
     if ((v & 0x7f800000) != 0x7f800000)
         return 0;
     return v & 0x007fffff;
 }
+
+static av_always_inline av_const int avpriv_isnan(double x)
+{
+    uint64_t v = av_double2int(x);
+    if ((v & 0x7ff0000000000000) != 0x7ff0000000000000)
+        return 0;
+    return v & 0x000fffffffffffff;
+}
+
+static av_always_inline av_const int avpriv_isnanl(long double x)
+{
+    // FIXME: just a stub, hard as long double width can vary between platforms
+    // Also currently irrelevant
+    return avpriv_isnan(x);
+}
+
+#define isnan(x)                             \
+    (sizeof(x) == sizeof(float)              \
+        ? avpriv_isnanf(x)                   \
+        : sizeof(x) == sizeof(double)        \
+        ? avpriv_isnan(x) : avpriv_isnanl(x))
 #endif /* HAVE_ISNAN */
 
 #if !HAVE_LDEXPF
-- 
2.6.2