rustc: Implement stack probes for x86

This commit implements stack probes on x86/x86_64 using the freshly landed
support upstream in LLVM. The purpose of stack probes here are to guarantee a
segfault on stack overflow rather than having a chance of running over the guard
page already present on all threads by accident.

At this time there's no support for any other architecture because LLVM itself
does not have support for other architectures.

Author: alexcrichton

src/libcompiler_builtins

@@ -26,6 +26,7 @@ pub fn target() -> TargetResult {
         linker_flavor: LinkerFlavor::Gcc,
         options: TargetOptions {
             max_atomic_width: Some(64),
+            stack_probes: true,
             .. base
         }
     })

src/librustc_back/target/i386_apple_ios.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "yonah".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.insert(LinkerFlavor::Gcc, vec!["-m32".to_string()]);
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "i686-apple-darwin".to_string(),

src/librustc_back/target/i686_apple_darwin.rs

@@ -22,6 +22,7 @@ pub fn target() -> TargetResult {
     // http://developer.android.com/ndk/guides/abis.html#x86
     base.cpu = "pentiumpro".to_string();
     base.features = "+mmx,+sse,+sse2,+sse3,+ssse3".to_string();
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "i686-linux-android".to_string(),

src/librustc_back/target/i686_linux_android.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "pentium4".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m32".to_string());
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "i686-unknown-dragonfly".to_string(),

src/librustc_back/target/i686_unknown_dragonfly.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "pentium4".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m32".to_string());
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "i686-unknown-freebsd".to_string(),

src/librustc_back/target/i686_unknown_freebsd.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "pentium4".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.insert(LinkerFlavor::Gcc, vec!["-m32".to_string()]);
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "i686-unknown-haiku".to_string(),

src/librustc_back/target/i686_unknown_haiku.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "pentium4".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m32".to_string());
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "i686-unknown-linux-gnu".to_string(),

src/librustc_back/target/i686_unknown_linux_gnu.rs

@@ -17,6 +17,7 @@ pub fn target() -> TargetResult {
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m32".to_string());
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-Wl,-melf_i386".to_string());
+    base.stack_probes = true;
 
     // The unwinder used by i686-unknown-linux-musl, the LLVM libunwind
     // implementation, apparently relies on frame pointers existing... somehow.

src/librustc_back/target/i686_unknown_linux_musl.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "pentium4".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m32".to_string());
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "i686-unknown-netbsdelf".to_string(),

src/librustc_back/target/i686_unknown_netbsd.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "pentium4".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m32".to_string());
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "i686-unknown-openbsd".to_string(),

src/librustc_back/target/i686_unknown_openbsd.rs

@@ -409,6 +409,9 @@ pub struct TargetOptions {
 
     /// Whether or not the CRT is statically linked by default.
     pub crt_static_default: bool,
+
+    /// Whether or not stack probes (__rust_probestack) are enabled
+    pub stack_probes: bool,
 }
 
 impl Default for TargetOptions {
@@ -466,6 +469,7 @@ impl Default for TargetOptions {
             panic_strategy: PanicStrategy::Unwind,
             abi_blacklist: vec![],
             crt_static_default: false,
+            stack_probes: false,
         }
     }
 }
@@ -688,6 +692,7 @@ impl Target {
         key!(min_atomic_width, Option<u64>);
         try!(key!(panic_strategy, PanicStrategy));
         key!(crt_static_default, bool);
+        key!(stack_probes, bool);
 
         if let Some(array) = obj.find("abi-blacklist").and_then(Json::as_array) {
             for name in array.iter().filter_map(|abi| abi.as_string()) {
@@ -874,6 +879,7 @@ impl ToJson for Target {
         target_option_val!(max_atomic_width);
         target_option_val!(panic_strategy);
         target_option_val!(crt_static_default);
+        target_option_val!(stack_probes);
 
         if default.abi_blacklist != self.options.abi_blacklist {
             d.insert("abi-blacklist".to_string(), self.options.abi_blacklist.iter()

src/librustc_back/target/mod.rs

@@ -17,6 +17,7 @@ pub fn target() -> TargetResult {
     base.max_atomic_width = Some(128); // core2 support cmpxchg16b
     base.eliminate_frame_pointer = false;
     base.pre_link_args.insert(LinkerFlavor::Gcc, vec!["-m64".to_string()]);
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "x86_64-apple-darwin".to_string(),

src/librustc_back/target/x86_64_apple_darwin.rs

@@ -26,6 +26,7 @@ pub fn target() -> TargetResult {
         linker_flavor: LinkerFlavor::Gcc,
         options: TargetOptions {
             max_atomic_width: Some(64),
+            stack_probes: true,
             .. base
         }
     })

src/librustc_back/target/x86_64_apple_ios.rs

@@ -18,6 +18,7 @@ pub fn target() -> TargetResult {
     base.features = "+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "x86_64-linux-android".to_string(),

src/librustc_back/target/x86_64_linux_android.rs

@@ -25,6 +25,7 @@ pub fn target() -> TargetResult {
     base.disable_redzone = true;
     base.no_default_libraries = false;
     base.exe_allocation_crate = None;
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "x86_64-rumprun-netbsd".to_string(),

src/librustc_back/target/x86_64_rumprun_netbsd.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.pre_link_args.insert(LinkerFlavor::Gcc, vec!["-m64".to_string()]);
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "x86_64-pc-solaris".to_string(),

src/librustc_back/target/x86_64_sun_solaris.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.insert(LinkerFlavor::Gcc, vec!["-m64".to_string()]);
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "x86_64-unknown-bitrig".to_string(),

src/librustc_back/target/x86_64_unknown_bitrig.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "x86_64-unknown-dragonfly".to_string(),

src/librustc_back/target/x86_64_unknown_dragonfly.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "x86_64-unknown-freebsd".to_string(),

src/librustc_back/target/x86_64_unknown_freebsd.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "x86_64-unknown-fuchsia".to_string(),

src/librustc_back/target/x86_64_unknown_fuchsia.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.insert(LinkerFlavor::Gcc, vec!["-m64".to_string()]);
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "x86_64-unknown-haiku".to_string(),

src/librustc_back/target/x86_64_unknown_haiku.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "x86_64-unknown-linux-gnu".to_string(),

src/librustc_back/target/x86_64_unknown_linux_gnu.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "x86_64-unknown-linux-musl".to_string(),

src/librustc_back/target/x86_64_unknown_linux_musl.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "x86_64-unknown-netbsd".to_string(),

src/librustc_back/target/x86_64_unknown_netbsd.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "x86_64-unknown-openbsd".to_string(),

src/librustc_back/target/x86_64_unknown_openbsd.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "x86_64-unknown-redox".to_string(),

src/librustc_back/target/x86_64_unknown_redox.rs

@@ -11,13 +11,14 @@
 
 use std::ffi::{CStr, CString};
 
+use rustc::session::config::Sanitizer;
+
 use llvm::{self, Attribute, ValueRef};
 use llvm::AttributePlace::Function;
 pub use syntax::attr::{self, InlineAttr};
 use syntax::ast;
 use context::CrateContext;
 
-
 /// Mark LLVM function to use provided inline heuristic.
 #[inline]
 pub fn inline(val: ValueRef, inline: InlineAttr) {
@@ -69,13 +70,36 @@ pub fn set_frame_pointer_elimination(ccx: &CrateContext, llfn: ValueRef) {
     }
 }
 
+pub fn set_probestack(ccx: &CrateContext, llfn: ValueRef) {
+    // Only use stack probes if the target specification indicates that we
+    // should be using stack probes
+    if !ccx.sess().target.target.options.stack_probes {
+        return
+    }
+
+    // Currently stack probes seem somewhat incompatible with the address
+    // sanitizer. With asan we're already protected from stack overflow anyway
+    // so we don't really need stack probes regardless.
+    match ccx.sess().opts.debugging_opts.sanitizer {
+        Some(Sanitizer::Address) => return,
+        _ => {}
+    }
+
+    // Flag our internal `__rust_probestack` function as the stack probe symbol.
+    // This is defined in the `compiler-builtins` crate for each architecture.
+    llvm::AddFunctionAttrStringValue(
+        llfn, llvm::AttributePlace::Function,
+        cstr("probe-stack\0"), cstr("__rust_probestack\0"));
+}
+
 /// Composite function which sets LLVM attributes for function depending on its AST (#[attribute])
 /// attributes.
 pub fn from_fn_attrs(ccx: &CrateContext, attrs: &[ast::Attribute], llfn: ValueRef) {
     use syntax::attr::*;
     inline(llfn, find_inline_attr(Some(ccx.sess().diagnostic()), attrs));
 
     set_frame_pointer_elimination(ccx, llfn);
+    set_probestack(ccx, llfn);
     let mut target_features = vec![];
     for attr in attrs {
         if attr.check_name("target_feature") {

src/librustc_trans/attributes.rs

@@ -0,0 +1,24 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// ignore-arm
+// ignore-wasm
+// ignore-emscripten
+// ignore-windows
+// no-system-llvm
+// compile-flags: -C no-prepopulate-passes
+
+#![crate_type = "lib"]
+
+#[no_mangle]
+pub fn foo() {
+// CHECK: @foo() unnamed_addr #0
+// CHECK: attributes #0 = { {{.*}}"probe-stack"="__rust_probestack"{{.*}} }
+}

src/llvm

@@ -0,0 +1,20 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// ignore-arm
+// ignore-wasm
+// ignore-emscripten
+// ignore-musl FIXME #31506
+// ignore-pretty
+// no-system-llvm
+// compile-flags: -C lto
+// no-prefer-dynamic
+
+include!("stack-probes.rs");