Ensure floats are returned losslessly by the Rust ABI on 32-bit x86

Author: beetrees

compiler/rustc_ty_utils/src/abi.rs

@@ -741,6 +741,40 @@ fn fn_abi_adjust_for_abi<'tcx>(
                 return;
             }
 
+            // Avoid returning floats in x87 registers on x86 as loading and storing from x87
+            // registers will quiet signalling NaNs.
+            if cx.tcx.sess.target.arch == "x86"
+                && arg_idx.is_none()
+                // Intrinsics themselves are not actual "real" functions, so theres no need to
+                // change their ABIs.
+                && abi != SpecAbi::RustIntrinsic
+            {
+                match arg.layout.abi {
+                    // Handle similar to the way arguments with an `Abi::Aggregate` abi are handled
+                    // below, by returning arguments up to the size of a pointer (32 bits on x86)
+                    // cast to an appropriately sized integer.
+                    Abi::Scalar(s) if s.primitive() == F32 => {
+                        // Same size as a pointer, return in a register.
+                        arg.cast_to(Reg::i32());
+                        return;
+                    }
+                    Abi::Scalar(s) if s.primitive() == F64 => {
+                        // Larger than a pointer, return indirectly.
+                        arg.make_indirect();
+                        return;
+                    }
+                    Abi::ScalarPair(s1, s2)
+                        if matches!(s1.primitive(), F32 | F64)
+                            || matches!(s2.primitive(), F32 | F64) =>
+                    {
+                        // Larger than a pointer, return indirectly.
+                        arg.make_indirect();
+                        return;
+                    }
+                    _ => {}
+                };
+            }
+
             match arg.layout.abi {
                 Abi::Aggregate { .. } => {}
 

tests/ui/abi/numbers-arithmetic/return-float.rs

@@ -0,0 +1,57 @@
+//@ run-pass
+//@ compile-flags: -Copt-level=0
+
+// Test that floats (in particular signalling NaNs) are losslessly returned from functions.
+
+fn main() {
+    let bits_f32 = std::hint::black_box([
+        4.2_f32.to_bits(),
+        f32::INFINITY.to_bits(),
+        f32::NEG_INFINITY.to_bits(),
+        f32::NAN.to_bits(),
+        // These two masks cover all the mantissa bits. One of them is a signalling NaN, the other
+        // is quiet.
+        // Similar to the masks in `test_float_bits_conv` in library/std/src/f32/tests.rs
+        f32::NAN.to_bits() ^ 0x002A_AAAA,
+        f32::NAN.to_bits() ^ 0x0055_5555,
+        // Same as above but with the sign bit flipped.
+        f32::NAN.to_bits() ^ 0x802A_AAAA,
+        f32::NAN.to_bits() ^ 0x8055_5555,
+    ]);
+    for bits in bits_f32 {
+        assert_eq!(identity(f32::from_bits(bits)).to_bits(), bits);
+        // Test types that are returned as scalar pairs.
+        assert_eq!(identity((f32::from_bits(bits), 42)).0.to_bits(), bits);
+        assert_eq!(identity((42, f32::from_bits(bits))).1.to_bits(), bits);
+        let (a, b) = identity((f32::from_bits(bits), f32::from_bits(bits)));
+        assert_eq!((a.to_bits(), b.to_bits()), (bits, bits));
+    }
+
+    let bits_f64 = std::hint::black_box([
+        4.2_f64.to_bits(),
+        f64::INFINITY.to_bits(),
+        f64::NEG_INFINITY.to_bits(),
+        f64::NAN.to_bits(),
+        // These two masks cover all the mantissa bits. One of them is a signalling NaN, the other
+        // is quiet.
+        // Similar to the masks in `test_float_bits_conv` in library/std/src/f64/tests.rs
+        f64::NAN.to_bits() ^ 0x000A_AAAA_AAAA_AAAA,
+        f64::NAN.to_bits() ^ 0x0005_5555_5555_5555,
+        // Same as above but with the sign bit flipped.
+        f64::NAN.to_bits() ^ 0x800A_AAAA_AAAA_AAAA,
+        f64::NAN.to_bits() ^ 0x8005_5555_5555_5555,
+    ]);
+    for bits in bits_f64 {
+        assert_eq!(identity(f64::from_bits(bits)).to_bits(), bits);
+        // Test types that are returned as scalar pairs.
+        assert_eq!(identity((f64::from_bits(bits), 42)).0.to_bits(), bits);
+        assert_eq!(identity((42, f64::from_bits(bits))).1.to_bits(), bits);
+        let (a, b) = identity((f64::from_bits(bits), f64::from_bits(bits)));
+        assert_eq!((a.to_bits(), b.to_bits()), (bits, bits));
+    }
+}
+
+#[inline(never)]
+fn identity<T>(x: T) -> T {
+    x
+}