Adding support for the llvm prefetch intrinsic

Related to #37251

Author: hirschenberger

src/libcore/intrinsics.rs

@@ -564,8 +564,55 @@ extern "rust-intrinsic" {
     pub fn atomic_umax_rel<T>(dst: *mut T, src: T) -> T;
     pub fn atomic_umax_acqrel<T>(dst: *mut T, src: T) -> T;
     pub fn atomic_umax_relaxed<T>(dst: *mut T, src: T) -> T;
+
+    /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction
+    /// if supported; otherwise, it is a noop.
+    /// Prefetches have no effect on the behavior of the program but can change its performance
+    /// characteristics.
+    ///
+    /// The `locality` argument must be a constant integer and is a temporal locality specifier
+    /// ranging from (0) - no locality, to (3) - extremely local keep in cache
+    #[cfg(not(stage0))]
+    pub fn prefetch_read_data<T>(data: *const T, locality: i32);
+    /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction
+    /// if supported; otherwise, it is a noop.
+    /// Prefetches have no effect on the behavior of the program but can change its performance
+    /// characteristics.
+    ///
+    /// The `locality` argument must be a constant integer and is a temporal locality specifier
+    /// ranging from (0) - no locality, to (3) - extremely local keep in cache
+    #[cfg(not(stage0))]
+    pub fn prefetch_write_data<T>(data: *const T, locality: i32);
+    /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction
+    /// if supported; otherwise, it is a noop.
+    /// Prefetches have no effect on the behavior of the program but can change its performance
+    /// characteristics.
+    ///
+    /// The `locality` argument must be a constant integer and is a temporal locality specifier
+    /// ranging from (0) - no locality, to (3) - extremely local keep in cache
+    #[cfg(not(stage0))]
+    pub fn prefetch_read_instruction<T>(data: *const T, locality: i32);
+    /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction
+    /// if supported; otherwise, it is a noop.
+    /// Prefetches have no effect on the behavior of the program but can change its performance
+    /// characteristics.
+    ///
+    /// The `locality` argument must be a constant integer and is a temporal locality specifier
+    /// ranging from (0) - no locality, to (3) - extremely local keep in cache
+    #[cfg(not(stage0))]
+    pub fn prefetch_write_instruction<T>(data: *const T, locality: i32);
 }
 
+// Empty bootstrap implementations for stage0 compilation
+#[cfg(stage0)]
+pub fn prefetch_read_data<T>(_data: *const T, _locality: i32) { /* EMPTY */ }
+#[cfg(stage0)]
+pub fn prefetch_write_data<T>(_data: *const T, _locality: i32) { /* EMPTY */ }
+#[cfg(stage0)]
+pub fn prefetch_read_instruction<T>(_data: *const T, _locality: i32) { /* EMPTY */ }
+#[cfg(stage0)]
+pub fn prefetch_write_instruction<T>(_data: *const T, _locality: i32) { /* EMPTY */ }
+
 extern "rust-intrinsic" {
 
     pub fn atomic_fence();

src/librustc_trans/context.rs

@@ -942,6 +942,7 @@ fn declare_intrinsic(ccx: &CrateContext, key: &str) -> Option<ValueRef> {
     ifn!("llvm.x86.seh.recoverfp", fn(i8p, i8p) -> i8p);
 
     ifn!("llvm.assume", fn(i1) -> void);
+    ifn!("llvm.prefetch", fn(i8p, t_i32, t_i32, t_i32) -> void);
 
     if ccx.sess().opts.debuginfo != NoDebugInfo {
         ifn!("llvm.dbg.declare", fn(Type::metadata(ccx), Type::metadata(ccx)) -> void);

src/librustc_trans/intrinsic.rs

@@ -255,7 +255,18 @@ pub fn trans_intrinsic_call<'a, 'tcx>(bcx: &Builder<'a, 'tcx>,
             }
             C_nil(ccx)
         },
-
+        "prefetch_read_data" | "prefetch_write_data" |
+        "prefetch_read_instruction" | "prefetch_write_instruction" => {
+            let expect = ccx.get_intrinsic(&("llvm.prefetch"));
+            let (rw, cache_type) = match name {
+                "prefetch_read_data" => (0, 1),
+                "prefetch_write_data" => (1, 1),
+                "prefetch_read_instruction" => (0, 0),
+                "prefetch_write_instruction" => (1, 0),
+                _ => bug!()
+            };
+            bcx.call(expect, &[llargs[0], C_i32(ccx, rw), llargs[1], C_i32(ccx, cache_type)], None)
+        },
         "ctlz" | "cttz" | "ctpop" | "bswap" |
         "add_with_overflow" | "sub_with_overflow" | "mul_with_overflow" |
         "overflowing_add" | "overflowing_sub" | "overflowing_mul" |

src/librustc_typeck/check/intrinsic.rs

@@ -133,6 +133,14 @@ pub fn check_intrinsic_type<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
                   ],
                tcx.mk_nil())
             }
+            "prefetch_read_data" | "prefetch_write_data" |
+            "prefetch_read_instruction" | "prefetch_write_instruction" => {
+                (1, vec![tcx.mk_ptr(ty::TypeAndMut {
+                          ty: param(0),
+                          mutbl: hir::MutImmutable
+                         }), tcx.types.i32],
+                    tcx.mk_nil())
+            }
             "drop_in_place" => {
                 (1, vec![tcx.mk_mut_ptr(param(0))], tcx.mk_nil())
             }

src/test/codegen/prefetch.rs

@@ -0,0 +1,75 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// compile-flags: -C no-prepopulate-passes
+
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::{prefetch_read_data, prefetch_write_data,
+                      prefetch_read_instruction, prefetch_write_instruction};
+
+#[no_mangle]
+pub fn check_prefetch_read_data(data: &[i8]) {
+    unsafe {
+        // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 0, i32 0, i32 1)
+        prefetch_read_data(data.as_ptr(), 0);
+        // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 0, i32 1, i32 1)
+        prefetch_read_data(data.as_ptr(), 1);
+        // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 0, i32 2, i32 1)
+        prefetch_read_data(data.as_ptr(), 2);
+        // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 0, i32 3, i32 1)
+        prefetch_read_data(data.as_ptr(), 3);
+    }
+}
+
+#[no_mangle]
+pub fn check_prefetch_write_data(data: &[i8]) {
+    unsafe {
+        // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 1, i32 0, i32 1)
+        prefetch_write_data(data.as_ptr(), 0);
+        // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 1, i32 1, i32 1)
+        prefetch_write_data(data.as_ptr(), 1);
+        // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 1, i32 2, i32 1)
+        prefetch_write_data(data.as_ptr(), 2);
+        // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 1, i32 3, i32 1)
+        prefetch_write_data(data.as_ptr(), 3);
+    }
+}
+
+#[no_mangle]
+pub fn check_prefetch_read_instruction(data: &[i8]) {
+    unsafe {
+        // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 0, i32 0, i32 0)
+        prefetch_read_instruction(data.as_ptr(), 0);
+        // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 0, i32 1, i32 0)
+        prefetch_read_instruction(data.as_ptr(), 1);
+        // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 0, i32 2, i32 0)
+        prefetch_read_instruction(data.as_ptr(), 2);
+        // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 0, i32 3, i32 0)
+        prefetch_read_instruction(data.as_ptr(), 3);
+    }
+}
+
+#[no_mangle]
+pub fn check_prefetch_write_instruction(data: &[i8]) {
+    unsafe {
+        // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 1, i32 0, i32 0)
+        prefetch_write_instruction(data.as_ptr(), 0);
+        // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 1, i32 1, i32 0)
+        prefetch_write_instruction(data.as_ptr(), 1);
+        // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 1, i32 2, i32 0)
+        prefetch_write_instruction(data.as_ptr(), 2);
+        // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 1, i32 3, i32 0)
+        prefetch_write_instruction(data.as_ptr(), 3);
+    }
+}
+
+