diff --git a/src/libcore/intrinsics.rs b/src/libcore/intrinsics.rs index 0136273ebc941..f1e51e995c238 100644 --- a/src/libcore/intrinsics.rs +++ b/src/libcore/intrinsics.rs @@ -1387,4 +1387,9 @@ extern "rust-intrinsic" { /// # } } /// ``` pub fn align_offset(ptr: *const (), align: usize) -> usize; + + /// Emits a `!nontemporal` store according to LLVM (see their docs). + /// Probably will never become stable. + #[cfg(not(stage0))] + pub fn nontemporal_store(ptr: *mut T, val: T); } diff --git a/src/librustc_llvm/ffi.rs b/src/librustc_llvm/ffi.rs index aab6139349d83..dd64d76bc0c51 100644 --- a/src/librustc_llvm/ffi.rs +++ b/src/librustc_llvm/ffi.rs @@ -587,6 +587,7 @@ extern "C" { // Operations on other types pub fn LLVMVoidTypeInContext(C: ContextRef) -> TypeRef; + pub fn LLVMX86MMXTypeInContext(C: ContextRef) -> TypeRef; pub fn LLVMRustMetadataTypeInContext(C: ContextRef) -> TypeRef; // Operations on all values diff --git a/src/librustc_trans/builder.rs b/src/librustc_trans/builder.rs index 50e673bdbfdd7..e40311af595cd 100644 --- a/src/librustc_trans/builder.rs +++ b/src/librustc_trans/builder.rs @@ -612,6 +612,29 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { } } + pub fn nontemporal_store(&self, val: ValueRef, ptr: ValueRef) -> ValueRef { + debug!("Store {:?} -> {:?}", Value(val), Value(ptr)); + assert!(!self.llbuilder.is_null()); + self.count_insn("store.nontemporal"); + let ptr = self.check_store(val, ptr); + unsafe { + let insn = llvm::LLVMBuildStore(self.llbuilder, val, ptr); + + // According to LLVM [1] building a nontemporal store must *always* + // point to a metadata value of the integer 1. Who knew? + // + // [1]: http://llvm.org/docs/LangRef.html#store-instruction + let one = C_i32(self.ccx, 1); + let node = llvm::LLVMMDNodeInContext(self.ccx.llcx(), + &one, + 1); + llvm::LLVMSetMetadata(insn, + llvm::MD_nontemporal as c_uint, + node); + insn + } + } + pub fn gep(&self, ptr: ValueRef, indices: &[ValueRef]) -> ValueRef { self.count_insn("gep"); unsafe { diff --git a/src/librustc_trans/diagnostics.rs b/src/librustc_trans/diagnostics.rs index 8485867689129..8f5d836f56f3f 100644 --- a/src/librustc_trans/diagnostics.rs +++ b/src/librustc_trans/diagnostics.rs @@ -37,13 +37,13 @@ The generic type has to be a SIMD type. Example: #[repr(simd)] #[derive(Copy, Clone)] -struct i32x1(i32); +struct i32x2(i32, i32); extern "platform-intrinsic" { fn simd_add(a: T, b: T) -> T; } -unsafe { simd_add(i32x1(0), i32x1(1)); } // ok! +unsafe { simd_add(i32x2(0, 0), i32x2(1, 2)); } // ok! ``` "##, diff --git a/src/librustc_trans/intrinsic.rs b/src/librustc_trans/intrinsic.rs index adbb45f893b08..997dd5573538a 100644 --- a/src/librustc_trans/intrinsic.rs +++ b/src/librustc_trans/intrinsic.rs @@ -540,6 +540,22 @@ pub fn trans_intrinsic_call<'a, 'tcx>(bcx: &Builder<'a, 'tcx>, } } + "nontemporal_store" => { + let tp_ty = substs.type_at(0); + let dst = args[0].deref(bcx.ccx); + let val = if let OperandValue::Ref(ptr, align) = args[1].val { + bcx.load(ptr, align.non_abi()) + } else { + from_immediate(bcx, args[1].immediate()) + }; + let ptr = bcx.pointercast(dst.llval, val_ty(val).ptr_to()); + let store = bcx.nontemporal_store(val, ptr); + unsafe { + llvm::LLVMSetAlignment(store, ccx.align_of(tp_ty).abi() as u32); + } + return + } + _ => { let intr = match Intrinsic::find(&name) { Some(intr) => intr, diff --git a/src/librustc_trans/type_.rs b/src/librustc_trans/type_.rs index 02224858b4692..1775e53284963 100644 --- a/src/librustc_trans/type_.rs +++ b/src/librustc_trans/type_.rs @@ -286,4 +286,8 @@ impl Type { Type::i8(ccx) } } + + pub fn x86_mmx(ccx: &CrateContext) -> Type { + ty!(llvm::LLVMX86MMXTypeInContext(ccx.llcx())) + } } diff --git a/src/librustc_trans/type_of.rs b/src/librustc_trans/type_of.rs index 9b32c825117ee..42e45b3a36fe4 100644 --- a/src/librustc_trans/type_of.rs +++ b/src/librustc_trans/type_of.rs @@ -26,8 +26,23 @@ fn uncached_llvm_type<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>, match layout.abi { layout::Abi::Scalar(_) => bug!("handled elsewhere"), layout::Abi::Vector => { - return Type::vector(&layout.field(ccx, 0).llvm_type(ccx), - layout.fields.count() as u64); + // LLVM has a separate type for 64-bit SIMD vectors on X86 called + // `x86_mmx` which is needed for some SIMD operations. As a bit of a + // hack (all SIMD definitions are super unstable anyway) we + // recognize any one-element SIMD vector as "this should be an + // x86_mmx" type. In general there shouldn't be a need for other + // one-element SIMD vectors, so it's assumed this won't clash with + // much else. + let use_x86_mmx = layout.fields.count() == 1 && + layout.size.bits() == 64 && + (ccx.sess().target.target.arch == "x86" || + ccx.sess().target.target.arch == "x86_64"); + if use_x86_mmx { + return Type::x86_mmx(ccx) + } else { + return Type::vector(&layout.field(ccx, 0).llvm_type(ccx), + layout.fields.count() as u64); + } } layout::Abi::ScalarPair(..) => { return Type::struct_(ccx, &[ diff --git a/src/librustc_typeck/check/intrinsic.rs b/src/librustc_typeck/check/intrinsic.rs index 6ef0c37a31583..c1adb0e65a4da 100644 --- a/src/librustc_typeck/check/intrinsic.rs +++ b/src/librustc_typeck/check/intrinsic.rs @@ -318,6 +318,10 @@ pub fn check_intrinsic_type<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, (0, vec![ptr_ty, tcx.types.usize], tcx.types.usize) }, + "nontemporal_store" => { + (1, vec![ tcx.mk_mut_ptr(param(0)), param(0) ], tcx.mk_nil()) + } + ref other => { struct_span_err!(tcx.sess, it.span, E0093, "unrecognized intrinsic function: `{}`", diff --git a/src/test/codegen/nontemporal.rs b/src/test/codegen/nontemporal.rs new file mode 100644 index 0000000000000..28ec534b97a68 --- /dev/null +++ b/src/test/codegen/nontemporal.rs @@ -0,0 +1,23 @@ +// Copyright 2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// compile-flags: -O + +#![feature(core_intrinsics)] +#![crate_type = "lib"] + +#[no_mangle] +pub fn a(a: &mut u32, b: u32) { + // CHECK-LABEL: define void @a + // CHECK: store i32 %b, i32* %a, align 4, !nontemporal + unsafe { + std::intrinsics::nontemporal_store(a, b); + } +} diff --git a/src/test/codegen/x86_mmx.rs b/src/test/codegen/x86_mmx.rs new file mode 100644 index 0000000000000..bedda63bbff31 --- /dev/null +++ b/src/test/codegen/x86_mmx.rs @@ -0,0 +1,30 @@ +// Copyright 2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// ignore-arm +// ignore-aarch64 +// ignore-emscripten +// compile-flags: -O + +#![feature(repr_simd)] +#![crate_type="lib"] + +#[repr(simd)] +#[derive(Clone, Copy)] +pub struct i8x8(u64); + +#[no_mangle] +pub fn a(a: &mut i8x8, b: i8x8) -> i8x8 { + // CHECK-LABEL: define x86_mmx @a(x86_mmx*{{.*}}, x86_mmx{{.*}}) + // CHECK: store x86_mmx %b, x86_mmx* %a + // CHECK: ret x86_mmx %b + *a = b; + return b +}