From d39255616004ea43dfabcf33b20ed2a80cd31dff Mon Sep 17 00:00:00 2001 From: Brian Anderson Date: Fri, 9 Aug 2013 01:15:31 -0700 Subject: [PATCH] std: Fix perf of local allocations in newsched Mostly optimizing TLS accesses to bring local heap allocation performance closer to that of oldsched. It's not completely at parity but removing the branches involved in supporting oldsched and optimizing pthread_get/setspecific to instead use our dedicated TCB slot will probably make up for it. --- src/libstd/os.rs | 4 +-- src/libstd/rt/comm.rs | 4 +-- src/libstd/rt/local_heap.rs | 22 +++++++++++----- src/libstd/rt/mod.rs | 51 +++++++++++++++++++++++-------------- src/libstd/sys.rs | 4 +-- src/libstd/task/mod.rs | 15 ++++++----- src/libstd/task/spawn.rs | 17 ++++++------- src/libstd/unstable/lang.rs | 17 +++++-------- src/libstd/unstable/sync.rs | 26 +++++++++++-------- 9 files changed, 92 insertions(+), 68 deletions(-) diff --git a/src/libstd/os.rs b/src/libstd/os.rs index f246a61a4d580..f7bd2aa240d7d 100644 --- a/src/libstd/os.rs +++ b/src/libstd/os.rs @@ -1142,9 +1142,9 @@ pub fn real_args() -> ~[~str] { #[cfg(target_os = "freebsd")] pub fn real_args() -> ~[~str] { use rt; - use rt::TaskContext; + use rt::NewRtContext; - if rt::context() == TaskContext { + if rt::context() == NewRtContext { match rt::args::clone() { Some(args) => args, None => fail!("process arguments not initialized") diff --git a/src/libstd/rt/comm.rs b/src/libstd/rt/comm.rs index 936a6526508a9..793e244bec7b9 100644 --- a/src/libstd/rt/comm.rs +++ b/src/libstd/rt/comm.rs @@ -15,6 +15,7 @@ use cast; use ops::Drop; use rt::kill::BlockedTask; use kinds::Send; +use rt; use rt::sched::Scheduler; use rt::local::Local; use rt::select::{Select, SelectPort}; @@ -24,7 +25,6 @@ use util::Void; use comm::{GenericChan, GenericSmartChan, GenericPort, Peekable}; use cell::Cell; use clone::Clone; -use rt::{context, SchedulerContext}; use tuple::ImmutableTuple; /// A combined refcount / BlockedTask-as-uint pointer. @@ -113,7 +113,7 @@ impl ChanOne { // 'do_resched' configures whether the scheduler immediately switches to // the receiving task, or leaves the sending task still running. fn try_send_inner(self, val: T, do_resched: bool) -> bool { - rtassert!(context() != SchedulerContext); + rtassert!(!rt::in_sched_context()); let mut this = self; let mut recvr_active = true; diff --git a/src/libstd/rt/local_heap.rs b/src/libstd/rt/local_heap.rs index e1e7ceacc3834..8832597f40c45 100644 --- a/src/libstd/rt/local_heap.rs +++ b/src/libstd/rt/local_heap.rs @@ -13,6 +13,7 @@ use libc; use libc::{c_void, uintptr_t, size_t}; use ops::Drop; +use option::{Some, None}; use rt; use rt::OldTaskContext; use rt::local::Local; @@ -86,8 +87,12 @@ impl Drop for LocalHeap { // A little compatibility function pub unsafe fn local_free(ptr: *libc::c_char) { - match rt::context() { - OldTaskContext => { + // XXX: Unsafe borrow for speed. Lame. + match Local::try_unsafe_borrow::() { + Some(task) => { + (*task).heap.free(ptr as *libc::c_void); + } + None => { rust_upcall_free_noswitch(ptr); extern { @@ -95,11 +100,6 @@ pub unsafe fn local_free(ptr: *libc::c_char) { fn rust_upcall_free_noswitch(ptr: *libc::c_char); } } - _ => { - do Local::borrow:: |task| { - task.heap.free(ptr as *libc::c_void); - } - } } } @@ -119,20 +119,28 @@ pub fn live_allocs() -> *raw::Box<()> { } extern { + #[fast_ffi] fn rust_new_memory_region(synchronized: uintptr_t, detailed_leaks: uintptr_t, poison_on_free: uintptr_t) -> *MemoryRegion; + #[fast_ffi] fn rust_delete_memory_region(region: *MemoryRegion); + #[fast_ffi] fn rust_new_boxed_region(region: *MemoryRegion, poison_on_free: uintptr_t) -> *BoxedRegion; + #[fast_ffi] fn rust_delete_boxed_region(region: *BoxedRegion); + #[fast_ffi] fn rust_boxed_region_malloc(region: *BoxedRegion, td: *TypeDesc, size: size_t) -> *OpaqueBox; + #[fast_ffi] fn rust_boxed_region_realloc(region: *BoxedRegion, ptr: *OpaqueBox, size: size_t) -> *OpaqueBox; + #[fast_ffi] fn rust_boxed_region_free(region: *BoxedRegion, box: *OpaqueBox); + #[fast_ffi] fn rust_current_boxed_region() -> *BoxedRegion; } diff --git a/src/libstd/rt/mod.rs b/src/libstd/rt/mod.rs index 01a52892f633b..be71bc651df59 100644 --- a/src/libstd/rt/mod.rs +++ b/src/libstd/rt/mod.rs @@ -407,14 +407,10 @@ fn run_(main: ~fn(), use_main_sched: bool) -> int { /// or the old scheduler. #[deriving(Eq)] pub enum RuntimeContext { - // Only the exchange heap is available - GlobalContext, - // The scheduler may be accessed - SchedulerContext, - // Full task services, e.g. local heap, unwinding - TaskContext, // Running in an old-style task - OldTaskContext + OldTaskContext, + // Not old task context + NewRtContext } /// Determine the current RuntimeContext @@ -424,19 +420,8 @@ pub fn context() -> RuntimeContext { if unsafe { rust_try_get_task().is_not_null() } { return OldTaskContext; - } else if Local::exists::() { - // In this case we know it is a new runtime context, but we - // need to check which one. Going to try borrowing task to - // check. Task should always be in TLS, so hopefully this - // doesn't conflict with other ops that borrow. - return do Local::borrow:: |task| { - match task.task_type { - SchedTask => SchedulerContext, - GreenTask(_) => TaskContext - } - }; } else { - return GlobalContext; + return NewRtContext; } extern { @@ -444,3 +429,31 @@ pub fn context() -> RuntimeContext { pub fn rust_try_get_task() -> *rust_task; } } + +pub fn in_sched_context() -> bool { + unsafe { + match Local::try_unsafe_borrow::() { + Some(task) => { + match (*task).task_type { + SchedTask => true, + _ => false + } + } + None => false + } + } +} + +pub fn in_green_task_context() -> bool { + unsafe { + match Local::try_unsafe_borrow::() { + Some(task) => { + match (*task).task_type { + GreenTask(_) => true, + _ => false + } + } + None => false + } + } +} \ No newline at end of file diff --git a/src/libstd/sys.rs b/src/libstd/sys.rs index 9d853087123e8..15c096ad04fd5 100644 --- a/src/libstd/sys.rs +++ b/src/libstd/sys.rs @@ -136,7 +136,7 @@ impl FailWithCause for &'static str { pub fn begin_unwind_(msg: *c_char, file: *c_char, line: size_t) -> ! { use either::Left; use option::{Some, None}; - use rt::{context, OldTaskContext, TaskContext}; + use rt::{context, OldTaskContext, in_green_task_context}; use rt::task::Task; use rt::local::Local; use rt::logging::Logger; @@ -158,7 +158,7 @@ pub fn begin_unwind_(msg: *c_char, file: *c_char, line: size_t) -> ! { // XXX: Logging doesn't work correctly in non-task context because it // invokes the local heap - if context == TaskContext { + if in_green_task_context() { // XXX: Logging doesn't work here - the check to call the log // function never passes - so calling the log function directly. do Local::borrow:: |task| { diff --git a/src/libstd/task/mod.rs b/src/libstd/task/mod.rs index 2e0c9c1d1ad1e..269c828a9845f 100644 --- a/src/libstd/task/mod.rs +++ b/src/libstd/task/mod.rs @@ -42,7 +42,7 @@ use cmp::Eq; use comm::{stream, Chan, GenericChan, GenericPort, Port}; use result::Result; use result; -use rt::{context, OldTaskContext, TaskContext}; +use rt::{context, OldTaskContext, in_green_task_context}; use rt::local::Local; use unstable::finally::Finally; use util; @@ -527,14 +527,15 @@ pub fn try(f: ~fn() -> T) -> Result { pub fn with_task_name(blk: &fn(Option<&str>) -> U) -> U { use rt::task::Task; - match context() { - TaskContext => do Local::borrow:: |task| { + if in_green_task_context() { + do Local::borrow:: |task| { match task.name { Some(ref name) => blk(Some(name.as_slice())), None => blk(None) } - }, - _ => fail!("no task name exists in %?", context()), + } + } else { + fail!("no task name exists in %?", context()) } } @@ -614,7 +615,7 @@ pub fn unkillable(f: &fn() -> U) -> U { rt::rust_task_allow_kill(t); } } - TaskContext => { + _ if in_green_task_context() => { // The inhibits/allows might fail and need to borrow the task. let t = Local::unsafe_borrow::(); do (|| { @@ -645,7 +646,7 @@ pub unsafe fn rekillable(f: &fn() -> U) -> U { rt::rust_task_inhibit_kill(t); } } - TaskContext => { + _ if in_green_task_context() => { let t = Local::unsafe_borrow::(); do (|| { (*t).death.allow_kill((*t).unwinder.unwinding); diff --git a/src/libstd/task/spawn.rs b/src/libstd/task/spawn.rs index 05a17f8539c21..314377b8dc9c1 100644 --- a/src/libstd/task/spawn.rs +++ b/src/libstd/task/spawn.rs @@ -91,7 +91,7 @@ use to_bytes::IterBytes; use uint; use util; use unstable::sync::Exclusive; -use rt::{OldTaskContext, TaskContext, SchedulerContext, GlobalContext, context}; +use rt::{OldTaskContext, NewRtContext, context, in_green_task_context}; use rt::local::Local; use rt::task::{Task, Sched}; use rt::kill::KillHandle; @@ -526,7 +526,7 @@ impl RuntimeGlue { let me = rt::rust_get_task(); blk(OldTask(me), rt::rust_task_is_unwinding(me)) }, - TaskContext => unsafe { + NewRtContext if in_green_task_context() => unsafe { // Can't use safe borrow, because the taskgroup destructor needs to // access the scheduler again to send kill signals to other tasks. let me = Local::unsafe_borrow::(); @@ -535,7 +535,7 @@ impl RuntimeGlue { blk(NewTask((*me).death.kill_handle.get_ref().clone()), (*me).unwinder.unwinding) }, - SchedulerContext | GlobalContext => rtabort!("task dying in bad context"), + NewRtContext => rtabort!("task dying in bad context"), } } @@ -563,7 +563,7 @@ impl RuntimeGlue { } } }, - TaskContext => unsafe { + NewRtContext if in_green_task_context() => unsafe { // Can't use safe borrow, because creating new hashmaps for the // tasksets requires an rng, which needs to borrow the sched. let me = Local::unsafe_borrow::(); @@ -588,7 +588,7 @@ impl RuntimeGlue { Some(ref group) => group, }) }, - SchedulerContext | GlobalContext => rtabort!("spawning in bad context"), + NewRtContext => rtabort!("spawning in bad context"), } } } @@ -666,10 +666,9 @@ fn enlist_many(child: TaskHandle, child_arc: &TaskGroupArc, pub fn spawn_raw(opts: TaskOpts, f: ~fn()) { match context() { - OldTaskContext => spawn_raw_oldsched(opts, f), - TaskContext => spawn_raw_newsched(opts, f), - SchedulerContext => fail!("can't spawn from scheduler context"), - GlobalContext => fail!("can't spawn from global context"), + OldTaskContext => spawn_raw_oldsched(opts, f), + _ if in_green_task_context() => spawn_raw_newsched(opts, f), + _ => fail!("can't spawn from this context") } } diff --git a/src/libstd/unstable/lang.rs b/src/libstd/unstable/lang.rs index 98c0fe254b697..c5112529aed9f 100644 --- a/src/libstd/unstable/lang.rs +++ b/src/libstd/unstable/lang.rs @@ -12,9 +12,9 @@ use cast::transmute; use libc::{c_char, c_uchar, c_void, size_t, uintptr_t, c_int}; +use option::{Some, None}; use str; use sys; -use rt::{context, OldTaskContext}; use rt::task::Task; use rt::local::Local; use rt::borrowck; @@ -56,16 +56,13 @@ pub fn fail_bounds_check(file: *c_char, line: size_t, #[lang="malloc"] pub unsafe fn local_malloc(td: *c_char, size: uintptr_t) -> *c_char { - match context() { - OldTaskContext => { - return rustrt::rust_upcall_malloc_noswitch(td, size); + // XXX: Unsafe borrow for speed. Lame. + match Local::try_unsafe_borrow::() { + Some(task) => { + (*task).heap.alloc(td as *c_void, size as uint) as *c_char } - _ => { - let mut alloc = ::ptr::null(); - do Local::borrow:: |task| { - alloc = task.heap.alloc(td as *c_void, size as uint) as *c_char; - } - return alloc; + None => { + rustrt::rust_upcall_malloc_noswitch(td, size) } } } diff --git a/src/libstd/unstable/sync.rs b/src/libstd/unstable/sync.rs index 225ac5c92adcf..a8d942a46b315 100644 --- a/src/libstd/unstable/sync.rs +++ b/src/libstd/unstable/sync.rs @@ -282,7 +282,7 @@ pub unsafe fn atomically(f: &fn() -> U) -> U { use rt::task::Task; use task::rt; use rt::local::Local; - use rt::{context, OldTaskContext, TaskContext}; + use rt::{context, OldTaskContext}; match context() { OldTaskContext => { @@ -296,17 +296,23 @@ pub unsafe fn atomically(f: &fn() -> U) -> U { rt::rust_task_allow_kill(t); } } - TaskContext => { - let t = Local::unsafe_borrow::(); - do (|| { - (*t).death.inhibit_yield(); - f() - }).finally { - (*t).death.allow_yield(); + _ => { + let t = Local::try_unsafe_borrow::(); + match t { + Some(t) => { + do (|| { + (*t).death.inhibit_yield(); + f() + }).finally { + (*t).death.allow_yield(); + } + } + None => { + // FIXME(#3095): As in unkillable(). + f() + } } } - // FIXME(#3095): As in unkillable(). - _ => f() } }