dymk/fast_fp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

native freeze

Browse Source
This commit is contained in:
Renar Narubin
2021-11-15 18:40:39 -08:00
parent 5bb809d657
commit 43dc1419a8
6 changed files with 128 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
build.rs
View File

@@ -6,10 +6,23 @@ fn main() {
builder.compiler("clang");
}
builder.flag("-O3").flag("-flto=thin");
build_ll(builder.clone());
build_c(builder);
}
fn build_ll(mut builder: cc::Build) {
// the ll files are written bare, let the compiler override module annotations and don't warn
// about it
builder.flag("-Wno-override-module");
builder.file("src/poison/freeze.ll").compile("freeze");
}
fn build_c(mut builder: cc::Build) {
builder
.file("src/math/math.c")
.flag("-O3")
.flag("-flto=thin")
.flag("-ffinite-math-only")
.flag("-fassociative-math")
.flag("-freciprocal-math")

16
src/lib.rs
View File

@@ -176,15 +176,13 @@ macro_rules! impl_fast_ops {
fn $op_fn(self, other: $fast_ty) -> Self::Output {
// Safety:
//
// - dereferencing the pointers is safe because every bit pattern is valid in float
// primitives
// - encountering poison operands is safe because LLVM's fast ops documents not producing
// UB on any inputs; it may produce poison on inf/nan (or if the sum is inf/nan), but these
// are then wrapped in the MaybePoison to control propagation
<$fast_ty>::new(unsafe {
$op_impl(
*self.0.maybe_poison().as_ptr(),
*other.0.maybe_poison().as_ptr(),
self.0.maybe_poison(),
other.0.maybe_poison(),
)
})
}
@@ -340,11 +338,7 @@ macro_rules! impls {
#[inline(always)]
fn freeze_raw(self) -> $base_ty {
let inner = self.0.freeze();
// Safety:
// every bit pattern is valid in float
unsafe { inner.assume_init() }
self.0.freeze()
}
// TODO migrate these to native implementations to freeze less and fast-math more
@@ -479,12 +473,10 @@ macro_rules! impls {
fn neg(self) -> Self::Output {
// Safety:
//
// - dereferencing the pointers is safe because every bit pattern is valid in float
// primitives
// - encountering poison is safe because LLVM's negate instruction documents
// not producing UB on any inputs. The value is also immediately wrapped, so
// poison propagation is controlled
let val = unsafe { *self.0.maybe_poison().as_ptr() };
let val = unsafe { self.0.maybe_poison() };
$fast_ty::new(-val)
}
}

10
src/math/mod.rs
View File

@@ -20,7 +20,7 @@ macro_rules! impl_generic_math {
//
// - `to_bits` should be valid for any input bits
// - poison propagation is controlled with MaybePoison
MaybePoison::new(unsafe { <$base_ty>::to_bits(*self.0.maybe_poison().as_ptr()) })
MaybePoison::new(unsafe { <$base_ty>::to_bits(self.0.maybe_poison()) })
}
#[inline]
@@ -30,7 +30,7 @@ macro_rules! impl_generic_math {
// - `from_bits` should be valid for any input bits
// - poison propagation is controlled with MaybePoison
Self(MaybePoison::new(unsafe {
<$base_ty>::from_bits(*bits.maybe_poison().as_ptr())
<$base_ty>::from_bits(bits.maybe_poison())
}))
}
@@ -38,7 +38,7 @@ macro_rules! impl_generic_math {
pub fn abs(self) -> Self {
let bits = self.to_bits();
<$fast_ty>::from_bits(MaybePoison::new(unsafe {
*bits.maybe_poison().as_ptr() & Self::UNSIGNED_MASK
bits.maybe_poison() & Self::UNSIGNED_MASK
}))
}
@@ -52,8 +52,8 @@ macro_rules! impl_generic_math {
// - & of poison is safe because & does not produce UB for any input values
// - poison propagation is handled by wrapping in maybe poison
<$fast_ty>::from_bits(MaybePoison::new(unsafe {
(*this.maybe_poison().as_ptr() & Self::UNSIGNED_MASK)
| (*that.maybe_poison().as_ptr() & Self::SIGN_BIT)
(this.maybe_poison() & Self::UNSIGNED_MASK)
| (that.maybe_poison() & Self::SIGN_BIT)
}))
}

74
src/poison.rs
View File

@@ -1,74 +0,0 @@
use core::mem::MaybeUninit;
/// A wrapper used to model LLVM's [poison
/// values](https://llvm.org/docs/LangRef.html#poisonvalues)
#[derive(Copy)]
#[repr(transparent)]
pub(crate) struct MaybePoison<T>(MaybeUninit<T>);
impl<T: Copy> Clone for MaybePoison<T> {
#[inline(always)]
fn clone(&self) -> Self {
*self
}
}
impl<T> MaybePoison<T> {
#[inline(always)]
pub(crate) const fn new(t: T) -> Self {
MaybePoison(MaybeUninit::new(t))
}
/// Get the (possibly poison) value from this instance.
///
/// The compiler may relax poison values to undefined values. That means, among other
/// consequences, that calls to this function from copies of the same value could manifest
/// different return values. Poison values are also transitive: an instruction that depends on
/// a poison value, produces a poison value itself.
///
/// Propogation of poison values can be stopped using [`freeze`](MaybePoison::freeze)
///
/// # Safety
///
/// It is UB to use a poison value as an operand to an instruction where _any_ of the operand's
/// values trigger UB. This includes, for example, use as the divisor in integer division, or
/// as the condition of a branch.
///
/// See more examples and explanations in the [LLVM
/// documentation](https://llvm.org/docs/LangRef.html#poisonvalues)
#[inline(always)]
pub(crate) unsafe fn maybe_poison(self) -> MaybeUninit<T> {
self.0
}
/// Freeze the poisoned value into a concrete (but arbitrary) value.
///
/// Note that the value may not be a valid representation of T, so the return type is still
/// unsafe to dereference unless T is valid with any representation.
#[inline(always)]
pub(crate) fn freeze(self) -> MaybeUninit<T> {
// As of this writing, rust does not have any intrinsic to call LLVM's freeze instruction.
// Instead, we do the next best thing by tricking the compiler into de-optimizing poison
// values by introducing inline assembly. This is the same technique used by
// `core::hint::black_box` and (the unmerged) https://github.com/rust-lang/rust/pull/58363.
// We cannot use black_box directly, however, as it is documented as only a best-effort
// hint, and could in theory be changed in the future.
// Safety:
//
// - The poison value will no longer be poisoned, its safety restrictions no longer apply
// - The asm macro emits no actual assembly, there's nothing to be unsafe
unsafe {
let inner = self.maybe_poison();
// There is no actual assembly, it's just a trick to restrict the compiler from
// optimizing around poison values. However the asm macro requires the format
// string to capture all inputs, so put the captured pointer in an assembly comment.
// The possibly poison value is labelled as input to the assembly block by providing a
// pointer to the value; the compiler then must assume that anything could be done with
// that pointer (e.g. reading and writing the value) so the compiler must materialize
// a concrete (though arbitrary) value before the assembly
asm!("/* {0} */", in(reg) inner.as_ptr(), options(nostack, preserves_flags));
inner
}
}
}

26
src/poison/freeze.ll Normal file
View File

@@ -0,0 +1,26 @@
define float @freeze_f32(float %a) unnamed_addr #0 {
%b = freeze float %a
ret float %b
}
define double @freeze_f64(double %a) unnamed_addr #0 {
%b = freeze double %a
ret double %b
}
define i1 @freeze_i1(i1 %a) unnamed_addr #0 {
%b = freeze i1 %a
ret i1 %b
}
define i32 @freeze_i32(i32 %a) unnamed_addr #0 {
%b = freeze i32 %a
ret i32 %b
}
define i64 @freeze_i64(i64 %a) unnamed_addr #0 {
%b = freeze i64 %a
ret i64 %b
}
attributes #0 = { alwaysinline nofree norecurse willreturn nosync nounwind readnone }

78
src/poison/mod.rs Normal file
View File

@@ -0,0 +1,78 @@
use core::mem::MaybeUninit;
/// A wrapper used to model LLVM's [poison
/// values](https://llvm.org/docs/LangRef.html#poisonvalues)
#[derive(Copy)]
#[repr(transparent)]
pub(crate) struct MaybePoison<T>(MaybeUninit<T>);
impl<T: Copy> Clone for MaybePoison<T> {
#[inline(always)]
fn clone(&self) -> Self {
*self
}
}
impl<T> MaybePoison<T> {
#[inline(always)]
pub(crate) const fn new(t: T) -> Self {
MaybePoison(MaybeUninit::new(t))
}
}
/// A macro to implement poison handling *only* on types which are valid for every bit pattern
macro_rules! impl_maybe_poison {
($($raw_ty:ty),*) => {
$(
impl MaybePoison<$raw_ty> {
/// Get the (possibly poison) value from this instance.
///
/// The compiler may relax poison values to undefined values. That means, among other
/// consequences, that calls to this function from copies of the same value could manifest
/// different return values. Poison values are also transitive: an instruction that depends on
/// a poison value, produces a poison value itself.
///
/// Propogation of poison values can be stopped using [`freeze`](MaybePoison::freeze)
///
/// # Safety
///
/// It is UB to use a poison value as an operand to an instruction where _any_ of the operand's
/// values trigger UB. This includes, for example, use as the divisor in integer division, or
/// as the condition of a branch.
///
/// See more examples and explanations in the [LLVM
/// documentation](https://llvm.org/docs/LangRef.html#poisonvalues)
#[inline(always)]
pub(crate) unsafe fn maybe_poison(self) -> $raw_ty {
*self.0.as_ptr()
}
}
)*
}
}
macro_rules! impl_freeze {
($($raw_ty:ty, $fn_name:ident;)*) => {
$(
extern "C" {
fn $fn_name(val: MaybePoison<$raw_ty>) -> $raw_ty;
}
impl MaybePoison<$raw_ty> {
#[inline(always)]
pub(crate) fn freeze(self) -> $raw_ty {
unsafe { $fn_name(self) }
}
}
)*
}
}
impl_maybe_poison! { f32, f64, u32, u64 }
impl_freeze! {
f32, freeze_f32;
f64, freeze_f64;
//u32, freeze_i32;
//u64, freeze_i64;
//bool, freeze_i1;
}