native freeze

build.rs

@@ -6,10 +6,23 @@ fn main() {
         builder.compiler("clang");
     }
 
+    builder.flag("-O3").flag("-flto=thin");
+
+    build_ll(builder.clone());
+    build_c(builder);
+}
+
+fn build_ll(mut builder: cc::Build) {
+    // the ll files are written bare, let the compiler override module annotations and don't warn
+    // about it
+    builder.flag("-Wno-override-module");
+
+    builder.file("src/poison/freeze.ll").compile("freeze");
+}
+
+fn build_c(mut builder: cc::Build) {
     builder
         .file("src/math/math.c")
-        .flag("-O3")
-        .flag("-flto=thin")
         .flag("-ffinite-math-only")
         .flag("-fassociative-math")
         .flag("-freciprocal-math")

src/lib.rs

@@ -176,15 +176,13 @@ macro_rules! impl_fast_ops {
                 fn $op_fn(self, other: $fast_ty) -> Self::Output {
                     // Safety:
                     //
-                    // - dereferencing the pointers is safe because every bit pattern is valid in float
-                    // primitives
                     // - encountering poison operands is safe because LLVM's fast ops documents not producing
                     // UB on any inputs; it may produce poison on inf/nan (or if the sum is inf/nan), but these
                     // are then wrapped in the MaybePoison to control propagation
                     <$fast_ty>::new(unsafe {
                         $op_impl(
-                            *self.0.maybe_poison().as_ptr(),
+                            self.0.maybe_poison(),
-                            *other.0.maybe_poison().as_ptr(),
+                            other.0.maybe_poison(),
                         )
                     })
                 }
@@ -340,11 +338,7 @@ macro_rules! impls {
 
             #[inline(always)]
             fn freeze_raw(self) -> $base_ty {
-                let inner = self.0.freeze();
+                self.0.freeze()
-
-                // Safety:
-                // every bit pattern is valid in float
-                unsafe { inner.assume_init() }
             }
 
             // TODO migrate these to native implementations to freeze less and fast-math more
@@ -479,12 +473,10 @@ macro_rules! impls {
             fn neg(self) -> Self::Output {
                 // Safety:
                 //
-                // - dereferencing the pointers is safe because every bit pattern is valid in float
-                // primitives
                 // - encountering poison is safe because LLVM's negate instruction documents
                 // not producing UB on any inputs. The value is also immediately wrapped, so
                 // poison propagation is controlled
-                let val = unsafe { *self.0.maybe_poison().as_ptr() };
+                let val = unsafe { self.0.maybe_poison() };
                 $fast_ty::new(-val)
             }
         }

src/math/mod.rs

@@ -20,7 +20,7 @@ macro_rules! impl_generic_math {
                 //
                 // - `to_bits` should be valid for any input bits
                 // - poison propagation is controlled with MaybePoison
-                MaybePoison::new(unsafe { <$base_ty>::to_bits(*self.0.maybe_poison().as_ptr()) })
+                MaybePoison::new(unsafe { <$base_ty>::to_bits(self.0.maybe_poison()) })
             }
 
             #[inline]
@@ -30,7 +30,7 @@ macro_rules! impl_generic_math {
                 // - `from_bits` should be valid for any input bits
                 // - poison propagation is controlled with MaybePoison
                 Self(MaybePoison::new(unsafe {
-                    <$base_ty>::from_bits(*bits.maybe_poison().as_ptr())
+                    <$base_ty>::from_bits(bits.maybe_poison())
                 }))
             }
 
@@ -38,7 +38,7 @@ macro_rules! impl_generic_math {
             pub fn abs(self) -> Self {
                 let bits = self.to_bits();
                 <$fast_ty>::from_bits(MaybePoison::new(unsafe {
-                    *bits.maybe_poison().as_ptr() & Self::UNSIGNED_MASK
+                    bits.maybe_poison() & Self::UNSIGNED_MASK
                 }))
             }
 
@@ -52,8 +52,8 @@ macro_rules! impl_generic_math {
                 // - & of poison is safe because & does not produce UB for any input values
                 // - poison propagation is handled by wrapping in maybe poison
                 <$fast_ty>::from_bits(MaybePoison::new(unsafe {
-                    (*this.maybe_poison().as_ptr() & Self::UNSIGNED_MASK)
+                    (this.maybe_poison() & Self::UNSIGNED_MASK)
-                        | (*that.maybe_poison().as_ptr() & Self::SIGN_BIT)
+                        | (that.maybe_poison() & Self::SIGN_BIT)
                 }))
             }
 

src/poison.rs

@@ -1,74 +0,0 @@
-use core::mem::MaybeUninit;
-
-/// A wrapper used to model LLVM's [poison
-/// values](https://llvm.org/docs/LangRef.html#poisonvalues)
-#[derive(Copy)]
-#[repr(transparent)]
-pub(crate) struct MaybePoison<T>(MaybeUninit<T>);
-
-impl<T: Copy> Clone for MaybePoison<T> {
-    #[inline(always)]
-    fn clone(&self) -> Self {
-        *self
-    }
-}
-
-impl<T> MaybePoison<T> {
-    #[inline(always)]
-    pub(crate) const fn new(t: T) -> Self {
-        MaybePoison(MaybeUninit::new(t))
-    }
-
-    /// Get the (possibly poison) value from this instance.
-    ///
-    /// The compiler may relax poison values to undefined values. That means, among other
-    /// consequences, that calls to this function from copies of the same value could manifest
-    /// different return values. Poison values are also transitive: an instruction that depends on
-    /// a poison value, produces a poison value itself.
-    ///
-    /// Propogation of poison values can be stopped using [`freeze`](MaybePoison::freeze)
-    ///
-    /// # Safety
-    ///
-    /// It is UB to use a poison value as an operand to an instruction where _any_ of the operand's
-    /// values trigger UB. This includes, for example, use as the divisor in integer division, or
-    /// as the condition of a branch.
-    ///
-    /// See more examples and explanations in the [LLVM
-    /// documentation](https://llvm.org/docs/LangRef.html#poisonvalues)
-    #[inline(always)]
-    pub(crate) unsafe fn maybe_poison(self) -> MaybeUninit<T> {
-        self.0
-    }
-
-    /// Freeze the poisoned value into a concrete (but arbitrary) value.
-    ///
-    /// Note that the value may not be a valid representation of T, so the return type is still
-    /// unsafe to dereference unless T is valid with any representation.
-    #[inline(always)]
-    pub(crate) fn freeze(self) -> MaybeUninit<T> {
-        // As of this writing, rust does not have any intrinsic to call LLVM's freeze instruction.
-        // Instead, we do the next best thing by tricking the compiler into de-optimizing poison
-        // values by introducing inline assembly. This is the same technique used by
-        // `core::hint::black_box` and (the unmerged) https://github.com/rust-lang/rust/pull/58363.
-        // We cannot use black_box directly, however, as it is documented as only a best-effort
-        // hint, and could in theory be changed in the future.
-
-        // Safety:
-        //
-        // - The poison value will no longer be poisoned, its safety restrictions no longer apply
-        // - The asm macro emits no actual assembly, there's nothing to be unsafe
-        unsafe {
-            let inner = self.maybe_poison();
-            // There is no actual assembly, it's just a trick to restrict the compiler from
-            // optimizing around poison values. However the asm macro requires the format
-            // string to capture all inputs, so put the captured pointer in an assembly comment.
-            // The possibly poison value is labelled as input to the assembly block by providing a
-            // pointer to the value; the compiler then must assume that anything could be done with
-            // that pointer (e.g. reading and writing the value) so the compiler must materialize
-            // a concrete (though arbitrary) value before the assembly
-            asm!("/* {0} */", in(reg) inner.as_ptr(), options(nostack, preserves_flags));
-            inner
-        }
-    }
-}

src/poison/freeze.ll

@@ -0,0 +1,26 @@
+define float @freeze_f32(float %a) unnamed_addr #0 {
+  %b = freeze float %a
+  ret float %b
+}
+
+define double @freeze_f64(double %a) unnamed_addr #0 {
+  %b = freeze double %a
+  ret double %b
+}
+
+define i1 @freeze_i1(i1 %a) unnamed_addr #0 {
+  %b = freeze i1 %a
+  ret i1 %b
+}
+
+define i32 @freeze_i32(i32 %a) unnamed_addr #0 {
+  %b = freeze i32 %a
+  ret i32 %b
+}
+
+define i64 @freeze_i64(i64 %a) unnamed_addr #0 {
+  %b = freeze i64 %a
+  ret i64 %b
+}
+
+attributes #0 = { alwaysinline nofree norecurse willreturn nosync nounwind readnone }

src/poison/mod.rs

@@ -0,0 +1,78 @@
+use core::mem::MaybeUninit;
+
+/// A wrapper used to model LLVM's [poison
+/// values](https://llvm.org/docs/LangRef.html#poisonvalues)
+#[derive(Copy)]
+#[repr(transparent)]
+pub(crate) struct MaybePoison<T>(MaybeUninit<T>);
+
+impl<T: Copy> Clone for MaybePoison<T> {
+    #[inline(always)]
+    fn clone(&self) -> Self {
+        *self
+    }
+}
+
+impl<T> MaybePoison<T> {
+    #[inline(always)]
+    pub(crate) const fn new(t: T) -> Self {
+        MaybePoison(MaybeUninit::new(t))
+    }
+}
+
+/// A macro to implement poison handling *only* on types which are valid for every bit pattern
+macro_rules! impl_maybe_poison {
+    ($($raw_ty:ty),*) => {
+        $(
+            impl MaybePoison<$raw_ty> {
+                /// Get the (possibly poison) value from this instance.
+                ///
+                /// The compiler may relax poison values to undefined values. That means, among other
+                /// consequences, that calls to this function from copies of the same value could manifest
+                /// different return values. Poison values are also transitive: an instruction that depends on
+                /// a poison value, produces a poison value itself.
+                ///
+                /// Propogation of poison values can be stopped using [`freeze`](MaybePoison::freeze)
+                ///
+                /// # Safety
+                ///
+                /// It is UB to use a poison value as an operand to an instruction where _any_ of the operand's
+                /// values trigger UB. This includes, for example, use as the divisor in integer division, or
+                /// as the condition of a branch.
+                ///
+                /// See more examples and explanations in the [LLVM
+                /// documentation](https://llvm.org/docs/LangRef.html#poisonvalues)
+                #[inline(always)]
+                pub(crate) unsafe fn maybe_poison(self) -> $raw_ty {
+                    *self.0.as_ptr()
+                }
+            }
+        )*
+    }
+}
+
+macro_rules! impl_freeze {
+    ($($raw_ty:ty, $fn_name:ident;)*) => {
+        $(
+            extern "C" {
+                fn $fn_name(val: MaybePoison<$raw_ty>) -> $raw_ty;
+            }
+
+            impl MaybePoison<$raw_ty> {
+                #[inline(always)]
+                pub(crate) fn freeze(self) -> $raw_ty {
+                    unsafe { $fn_name(self) }
+                }
+            }
+        )*
+    }
+}
+
+impl_maybe_poison! { f32, f64, u32, u64 }
+impl_freeze! {
+    f32, freeze_f32;
+    f64, freeze_f64;
+    //u32, freeze_i32;
+    //u64, freeze_i64;
+    //bool, freeze_i1;
+}