Skip to content

Commit 6342116

Browse files
saethlinsaethlin
committed
Add "algebraic" versions of the fast-math intrinsics
1 parent 11f32b7 commit 6342116

File tree

10 files changed

+178
-9
lines changed

10 files changed

+178
-9
lines changed

‎compiler/rustc_codegen_gcc/src/builder.rs

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -705,6 +705,31 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
705705
self.frem(lhs, rhs)
706706
}
707707

708+
fn fadd_algebraic(&mut self, lhs: RValue<'gcc>, rhs: RValue<'gcc>) -> RValue<'gcc> {
709+
// NOTE: it seems like we cannot enable fast-mode for a single operation in GCC.
710+
lhs + rhs
711+
}
712+
713+
fn fsub_algebraic(&mut self, lhs: RValue<'gcc>, rhs: RValue<'gcc>) -> RValue<'gcc> {
714+
// NOTE: it seems like we cannot enable fast-mode for a single operation in GCC.
715+
lhs - rhs
716+
}
717+
718+
fn fmul_algebraic(&mut self, lhs: RValue<'gcc>, rhs: RValue<'gcc>) -> RValue<'gcc> {
719+
// NOTE: it seems like we cannot enable fast-mode for a single operation in GCC.
720+
lhs * rhs
721+
}
722+
723+
fn fdiv_algebraic(&mut self, lhs: RValue<'gcc>, rhs: RValue<'gcc>) -> RValue<'gcc> {
724+
// NOTE: it seems like we cannot enable fast-mode for a single operation in GCC.
725+
lhs / rhs
726+
}
727+
728+
fn frem_algebraic(&mut self, lhs: RValue<'gcc>, rhs: RValue<'gcc>) -> RValue<'gcc> {
729+
// NOTE: it seems like we cannot enable fast-mode for a single operation in GCC.
730+
self.frem(lhs, rhs)
731+
}
732+
708733
fn checked_binop(&mut self, oop: OverflowOp, typ: Ty<'_>, lhs: Self::Value, rhs: Self::Value) -> (Self::Value, Self::Value) {
709734
self.gcc_checked_binop(oop, typ, lhs, rhs)
710735
}

‎compiler/rustc_codegen_llvm/src/builder.rs

Lines changed: 44 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,46 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
340340
}
341341
}
342342

343+
fn fadd_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
344+
unsafe {
345+
let instr = llvm::LLVMBuildFAdd(self.llbuilder, lhs, rhs, UNNAMED);
346+
llvm::LLVMRustSetAlgebraicMath(instr);
347+
instr
348+
}
349+
}
350+
351+
fn fsub_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
352+
unsafe {
353+
let instr = llvm::LLVMBuildFSub(self.llbuilder, lhs, rhs, UNNAMED);
354+
llvm::LLVMRustSetAlgebraicMath(instr);
355+
instr
356+
}
357+
}
358+
359+
fn fmul_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
360+
unsafe {
361+
let instr = llvm::LLVMBuildFMul(self.llbuilder, lhs, rhs, UNNAMED);
362+
llvm::LLVMRustSetAlgebraicMath(instr);
363+
instr
364+
}
365+
}
366+
367+
fn fdiv_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
368+
unsafe {
369+
let instr = llvm::LLVMBuildFDiv(self.llbuilder, lhs, rhs, UNNAMED);
370+
llvm::LLVMRustSetAlgebraicMath(instr);
371+
instr
372+
}
373+
}
374+
375+
fn frem_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
376+
unsafe {
377+
let instr = llvm::LLVMBuildFRem(self.llbuilder, lhs, rhs, UNNAMED);
378+
llvm::LLVMRustSetAlgebraicMath(instr);
379+
instr
380+
}
381+
}
382+
343383
fn checked_binop(
344384
&mut self,
345385
oop: OverflowOp,
@@ -1327,17 +1367,17 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
13271367
pub fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
13281368
unsafe { llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src) }
13291369
}
1330-
pub fn vector_reduce_fadd_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1370+
pub fn vector_reduce_fadd_algebraic(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
13311371
unsafe {
13321372
let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src);
1333-
llvm::LLVMRustSetFastMath(instr);
1373+
llvm::LLVMRustSetAlgebraicMath(instr);
13341374
instr
13351375
}
13361376
}
1337-
pub fn vector_reduce_fmul_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1377+
pub fn vector_reduce_fmul_algebraic(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
13381378
unsafe {
13391379
let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src);
1340-
llvm::LLVMRustSetFastMath(instr);
1380+
llvm::LLVMRustSetAlgebraicMath(instr);
13411381
instr
13421382
}
13431383
}

‎compiler/rustc_codegen_llvm/src/intrinsic.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1867,14 +1867,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
18671867
arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0);
18681868
arith_red!(
18691869
simd_reduce_add_unordered: vector_reduce_add,
1870-
vector_reduce_fadd_fast,
1870+
vector_reduce_fadd_algebraic,
18711871
false,
18721872
add,
18731873
0.0
18741874
);
18751875
arith_red!(
18761876
simd_reduce_mul_unordered: vector_reduce_mul,
1877-
vector_reduce_fmul_fast,
1877+
vector_reduce_fmul_algebraic,
18781878
false,
18791879
mul,
18801880
1.0

‎compiler/rustc_codegen_llvm/src/llvm/ffi.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1619,6 +1619,7 @@ extern "C" {
16191619
) -> &'a Value;
16201620

16211621
pub fn LLVMRustSetFastMath(Instr: &Value);
1622+
pub fn LLVMRustSetAlgebraicMath(Instr: &Value);
16221623

16231624
// Miscellaneous instructions
16241625
pub fn LLVMRustGetInstrProfIncrementIntrinsic(M: &Module) -> &Value;

‎compiler/rustc_codegen_ssa/src/mir/intrinsic.rs

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,38 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
249249
}
250250
}
251251
}
252+
sym::fadd_algebraic
253+
| sym::fsub_algebraic
254+
| sym::fmul_algebraic
255+
| sym::fdiv_algebraic
256+
| sym::frem_algebraic => match float_type_width(arg_tys[0]) {
257+
Some(_width) => match name {
258+
sym::fadd_algebraic => {
259+
bx.fadd_algebraic(args[0].immediate(), args[1].immediate())
260+
}
261+
sym::fsub_algebraic => {
262+
bx.fsub_algebraic(args[0].immediate(), args[1].immediate())
263+
}
264+
sym::fmul_algebraic => {
265+
bx.fmul_algebraic(args[0].immediate(), args[1].immediate())
266+
}
267+
sym::fdiv_algebraic => {
268+
bx.fdiv_algebraic(args[0].immediate(), args[1].immediate())
269+
}
270+
sym::frem_algebraic => {
271+
bx.frem_algebraic(args[0].immediate(), args[1].immediate())
272+
}
273+
_ => bug!(),
274+
},
275+
None => {
276+
bx.tcx().dcx().emit_err(InvalidMonomorphization::BasicFloatType {
277+
span,
278+
name,
279+
ty: arg_tys[0],
280+
});
281+
return;
282+
}
283+
},
252284

253285
sym::float_to_int_unchecked => {
254286
if float_type_width(arg_tys[0]).is_none() {

‎compiler/rustc_codegen_ssa/src/traits/builder.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,22 +86,27 @@ pub trait BuilderMethods<'a, 'tcx>:
8686
fn add(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
8787
fn fadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
8888
fn fadd_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
89+
fn fadd_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
8990
fn sub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
9091
fn fsub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
9192
fn fsub_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
93+
fn fsub_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
9294
fn mul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
9395
fn fmul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
9496
fn fmul_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
97+
fn fmul_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
9598
fn udiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
9699
fn exactudiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
97100
fn sdiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
98101
fn exactsdiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
99102
fn fdiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
100103
fn fdiv_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
104+
fn fdiv_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
101105
fn urem(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
102106
fn srem(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
103107
fn frem(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
104108
fn frem_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
109+
fn frem_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
105110
fn shl(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
106111
fn lshr(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
107112
fn ashr(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;

‎compiler/rustc_hir_analysis/src/check/intrinsic.rs

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,12 @@ pub fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: DefId) -> hir
112112
| sym::forget
113113
| sym::black_box
114114
| sym::variant_count
115-
| sym::ptr_mask => hir::Unsafety::Normal,
115+
| sym::ptr_mask
116+
| sym::fadd_algebraic
117+
| sym::fsub_algebraic
118+
| sym::fmul_algebraic
119+
| sym::fdiv_algebraic
120+
| sym::frem_algebraic => hir::Unsafety::Normal,
116121
_ => hir::Unsafety::Unsafe,
117122
};
118123

@@ -370,6 +375,11 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
370375
sym::fadd_fast | sym::fsub_fast | sym::fmul_fast | sym::fdiv_fast | sym::frem_fast => {
371376
(1, vec![param(0), param(0)], param(0))
372377
}
378+
sym::fadd_algebraic
379+
| sym::fsub_algebraic
380+
| sym::fmul_algebraic
381+
| sym::fdiv_algebraic
382+
| sym::frem_algebraic => (1, vec![param(0), param(0)], param(0)),
373383
sym::float_to_int_unchecked => (2, vec![param(0)], param(1)),
374384

375385
sym::assume => (0, vec![tcx.types.bool], Ty::new_unit(tcx)),

‎compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -419,8 +419,7 @@ extern "C" LLVMAttributeRef LLVMRustCreateMemoryEffectsAttr(LLVMContextRef C,
419419
report_fatal_error("bad MemoryEffects.");
420420
}
421421
}
422-
423-
// Enable a fast-math flag
422+
// Enable all fast-math flags
424423
//
425424
// https://llvm.org/docs/LangRef.html#fast-math-flags
426425
extern "C" void LLVMRustSetFastMath(LLVMValueRef V) {
@@ -429,6 +428,18 @@ extern "C" void LLVMRustSetFastMath(LLVMValueRef V) {
429428
}
430429
}
431430

431+
// Enable fast-math flags which do not let operations return poison.
432+
//
433+
// https://llvm.org/docs/LangRef.html#fast-math-flags
434+
extern "C" void LLVMRustSetAlgebraicMath(LLVMValueRef V) {
435+
if (auto I = dyn_cast<Instruction>(unwrap<Value>(V))) {
436+
I->setHasAllowReassoc(true);
437+
I->setHasAllowContract(true);
438+
I->setHasAllowReciprocal(true);
439+
I->setHasNoSignedZeros(true);
440+
}
441+
}
442+
432443
extern "C" LLVMValueRef
433444
LLVMRustBuildAtomicLoad(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Source,
434445
const char *Name, LLVMAtomicOrdering Order) {

‎compiler/rustc_span/src/symbol.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -755,8 +755,10 @@ symbols! {
755755
f64_nan,
756756
fabsf32,
757757
fabsf64,
758+
fadd_algebraic,
758759
fadd_fast,
759760
fake_variadic,
761+
fdiv_algebraic,
760762
fdiv_fast,
761763
feature,
762764
fence,
@@ -776,6 +778,7 @@ symbols! {
776778
fmaf32,
777779
fmaf64,
778780
fmt,
781+
fmul_algebraic,
779782
fmul_fast,
780783
fn_align,
781784
fn_delegation,
@@ -801,6 +804,7 @@ symbols! {
801804
format_unsafe_arg,
802805
freeze,
803806
freg,
807+
frem_algebraic,
804808
frem_fast,
805809
from,
806810
from_desugaring,
@@ -814,6 +818,7 @@ symbols! {
814818
from_usize,
815819
from_yeet,
816820
fs_create_dir,
821+
fsub_algebraic,
817822
fsub_fast,
818823
fundamental,
819824
future,

‎library/core/src/intrinsics.rs

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1882,6 +1882,46 @@ extern "rust-intrinsic" {
18821882
#[rustc_nounwind]
18831883
pub fn frem_fast<T: Copy>(a: T, b: T) -> T;
18841884

1885+
/// Float addition that allows optimizations based on algebraic rules.
1886+
///
1887+
/// This intrinsic does not have a stable counterpart.
1888+
#[rustc_nounwind]
1889+
#[rustc_safe_intrinsic]
1890+
#[cfg(not(bootstrap))]
1891+
pub fn fadd_algebraic<T: Copy>(a: T, b: T) -> T;
1892+
1893+
/// Float subtraction that allows optimizations based on algebraic rules.
1894+
///
1895+
/// This intrinsic does not have a stable counterpart.
1896+
#[rustc_nounwind]
1897+
#[rustc_safe_intrinsic]
1898+
#[cfg(not(bootstrap))]
1899+
pub fn fsub_algebraic<T: Copy>(a: T, b: T) -> T;
1900+
1901+
/// Float multiplication that allows optimizations based on algebraic rules.
1902+
///
1903+
/// This intrinsic does not have a stable counterpart.
1904+
#[rustc_nounwind]
1905+
#[rustc_safe_intrinsic]
1906+
#[cfg(not(bootstrap))]
1907+
pub fn fmul_algebraic<T: Copy>(a: T, b: T) -> T;
1908+
1909+
/// Float division that allows optimizations based on algebraic rules.
1910+
///
1911+
/// This intrinsic does not have a stable counterpart.
1912+
#[rustc_nounwind]
1913+
#[rustc_safe_intrinsic]
1914+
#[cfg(not(bootstrap))]
1915+
pub fn fdiv_algebraic<T: Copy>(a: T, b: T) -> T;
1916+
1917+
/// Float remainder that allows optimizations based on algebraic rules.
1918+
///
1919+
/// This intrinsic does not have a stable counterpart.
1920+
#[rustc_nounwind]
1921+
#[rustc_safe_intrinsic]
1922+
#[cfg(not(bootstrap))]
1923+
pub fn frem_algebraic<T: Copy>(a: T, b: T) -> T;
1924+
18851925
/// Convert with LLVM’s fptoui/fptosi, which may return undef for values out of range
18861926
/// (<https://github.com/rust-lang/rust/issues/10184>)
18871927
///

0 commit comments

Comments
 (0)