diff --git a/src/libcollections/vec.rs b/src/libcollections/vec.rs index f5d61a644a3c3..f378bb67ff1b7 100644 --- a/src/libcollections/vec.rs +++ b/src/libcollections/vec.rs @@ -1687,7 +1687,7 @@ impl MutableSeq for Vec { unsafe { mem::forget(value); } return } - if self.len == self.cap { + if unlikely!(self.len == self.cap) { let old_size = self.cap * mem::size_of::(); let size = max(old_size, 2 * mem::size_of::()) * 2; if old_size > size { fail!("capacity overflow") } diff --git a/src/libcore/intrinsics.rs b/src/libcore/intrinsics.rs index 7d86b65168f3c..befdaee2f3007 100644 --- a/src/libcore/intrinsics.rs +++ b/src/libcore/intrinsics.rs @@ -616,8 +616,37 @@ extern "rust-intrinsic" { pub fn u32_mul_with_overflow(x: u32, y: u32) -> (u32, bool); /// Performs checked `u64` multiplication. pub fn u64_mul_with_overflow(x: u64, y: u64) -> (u64, bool); + + /// Inform the optimizer that `expected_val` is the expected (most probable) value of `val`. + #[cfg(not(stage0))] + pub fn expect8(val: u8, expected_val: u8) -> u8; + /// Inform the optimizer that `expected_val` is the expected (most probable) value of `val`. + #[cfg(not(stage0))] + pub fn expect16(val: u16, expected_val: u16) -> u16; + /// Inform the optimizer that `expected_val` is the expected (most probable) value of `val`. + #[cfg(not(stage0))] + pub fn expect32(val: u32, expected_val: u32) -> u32; + /// Inform the optimizer that `expected_val` is the expected (most probable) value of `val`. + #[cfg(not(stage0))] + pub fn expect64(val: u64, expected_val: u64) -> u64; } +/// Inform the optimizer that `expected_val` is the expected (most probable) value of `val`. +#[cfg(stage0)] +#[inline(always)] +pub unsafe fn expect8(val: u8, _expected_val: u8) -> u8 { val } +/// Inform the optimizer that `expected_val` is the expected (most probable) value of `val`. +#[cfg(stage0)] +#[inline(always)] +pub unsafe fn expect16(val: u16, _expected_val: u16) -> u16 { val } +/// Inform the optimizer that `expected_val` is the expected (most probable) value of `val`. +#[cfg(stage0)] +#[inline(always)] +pub unsafe fn expect32(val: u32, _expected_val: u32) -> u32 { val } +/// Inform the optimizer that `expected_val` is the expected (most probable) value of `val`. +#[cfg(stage0)] +#[inline(always)] +pub unsafe fn expect64(val: u64, _expected_val: u64) -> u64 { val } /// `TypeId` represents a globally unique identifier for a type #[lang="type_id"] // This needs to be kept in lockstep with the code in trans/intrinsic.rs and diff --git a/src/libcore/macros.rs b/src/libcore/macros.rs index 17fcf0254575d..416a5ae9137c0 100644 --- a/src/libcore/macros.rs +++ b/src/libcore/macros.rs @@ -131,3 +131,27 @@ macro_rules! write( #[macro_export] macro_rules! unreachable( () => (fail!("unreachable code")) ) + +// FIXME: #17316: should be marked experimental +/// Inform the optimizer that a boolean condition is likely. +#[macro_export] +macro_rules! likely( + ($val:expr) => { + { + let x: bool = $val; + unsafe { ::core::intrinsics::expect8(x as u8, 1) != 0 } + } + } +) + +// FIXME: #17316: should be marked experimental +/// Inform the optimizer that a boolean condition is unlikely. +#[macro_export] +macro_rules! unlikely( + ($val:expr) => { + { + let x: bool = $val; + unsafe { ::core::intrinsics::expect8(x as u8, 0) != 0 } + } + } +) diff --git a/src/librustc/middle/trans/context.rs b/src/librustc/middle/trans/context.rs index 5c8c6a24b4fa5..a9b6063c60cf7 100644 --- a/src/librustc/middle/trans/context.rs +++ b/src/librustc/middle/trans/context.rs @@ -860,6 +860,10 @@ fn declare_intrinsic(ccx: &CrateContext, key: & &'static str) -> Option void); ifn!("llvm.expect.i1" fn(i1, i1) -> i1); + ifn!("llvm.expect.i8" fn(t_i8, t_i8) -> t_i8); + ifn!("llvm.expect.i16" fn(t_i16, t_i16) -> t_i16); + ifn!("llvm.expect.i32" fn(t_i32, t_i32) -> t_i32); + ifn!("llvm.expect.i64" fn(t_i64, t_i64) -> t_i64); // Some intrinsics were introduced in later versions of LLVM, but they have // fallbacks in libc or libm and such. Currently, all of these intrinsics diff --git a/src/librustc/middle/trans/intrinsic.rs b/src/librustc/middle/trans/intrinsic.rs index f9d55143c8400..961dbc4aedd8f 100644 --- a/src/librustc/middle/trans/intrinsic.rs +++ b/src/librustc/middle/trans/intrinsic.rs @@ -81,6 +81,10 @@ pub fn get_simple_intrinsic(ccx: &CrateContext, item: &ast::ForeignItem) -> Opti "bswap16" => "llvm.bswap.i16", "bswap32" => "llvm.bswap.i32", "bswap64" => "llvm.bswap.i64", + "expect8" => "llvm.expect.i8", + "expect16" => "llvm.expect.i16", + "expect32" => "llvm.expect.i32", + "expect64" => "llvm.expect.i64", _ => return None }; Some(ccx.get_intrinsic(&name)) diff --git a/src/librustc/middle/typeck/check/mod.rs b/src/librustc/middle/typeck/check/mod.rs index bb490273e9f0d..71a7afd509b1f 100644 --- a/src/librustc/middle/typeck/check/mod.rs +++ b/src/librustc/middle/typeck/check/mod.rs @@ -5811,6 +5811,11 @@ pub fn check_intrinsic_type(ccx: &CrateCtxt, it: &ast::ForeignItem) { "return_address" => (0, vec![], ty::mk_imm_ptr(tcx, ty::mk_u8())), + "expect8" => (0, vec![ty::mk_u8(), ty::mk_u8()], ty::mk_u8()), + "expect16" => (0, vec![ty::mk_u16(), ty::mk_u16()], ty::mk_u16()), + "expect32" => (0, vec![ty::mk_u32(), ty::mk_u32()], ty::mk_u32()), + "expect64" => (0, vec![ty::mk_u64(), ty::mk_u64()], ty::mk_u64()), + ref other => { span_err!(tcx.sess, it.span, E0093, "unrecognized intrinsic function: `{}`", *other); diff --git a/src/libstd/macros.rs b/src/libstd/macros.rs index fa356432a6731..3fc7ad2b55ded 100644 --- a/src/libstd/macros.rs +++ b/src/libstd/macros.rs @@ -214,6 +214,30 @@ macro_rules! unreachable( () => (fail!("internal error: entered unreachable code")) ) +// FIXME: #17316: should be marked experimental +/// Inform the optimizer that a boolean condition is likely. +#[macro_export] +macro_rules! likely( + ($val:expr) => { + { + let x: bool = $val; + unsafe { ::std::intrinsics::expect8(x as u8, 1) != 0 } + } + } +) + +// FIXME: #17316: should be marked experimental +/// Inform the optimizer that a boolean condition is unlikely. +#[macro_export] +macro_rules! unlikely( + ($val:expr) => { + { + let x: bool = $val; + unsafe { ::std::intrinsics::expect8(x as u8, 0) != 0 } + } + } +) + /// A standardised placeholder for marking unfinished code. It fails with the /// message `"not yet implemented"` when executed. #[macro_export]