Skip to content

Commit f40a4ec

Browse files
faernfaern
committed
Base Once on parking_lot::Once
1 parent 22bfd51 commit f40a4ec

File tree

1 file changed

+29
-239
lines changed

1 file changed

+29
-239
lines changed

‎src/libstd/sync/once.rs

+29-239
Original file line numberDiff line numberDiff line change
@@ -10,53 +10,13 @@
1010
//
1111
// As a result, we end up implementing it ourselves in the standard library.
1212
// This also gives us the opportunity to optimize the implementation a bit which
13-
// should help the fast path on call sites. Consequently, let's explain how this
14-
// primitive works now!
13+
// should help the fast path on call sites.
1514
//
16-
// So to recap, the guarantees of a Once are that it will call the
17-
// initialization closure at most once, and it will never return until the one
18-
// that's running has finished running. This means that we need some form of
19-
// blocking here while the custom callback is running at the very least.
20-
// Additionally, we add on the restriction of **poisoning**. Whenever an
21-
// initialization closure panics, the Once enters a "poisoned" state which means
22-
// that all future calls will immediately panic as well.
23-
//
24-
// So to implement this, one might first reach for a `Mutex`, but those cannot
25-
// be put into a `static`. It also gets a lot harder with poisoning to figure
26-
// out when the mutex needs to be deallocated because it's not after the closure
27-
// finishes, but after the first successful closure finishes.
28-
//
29-
// All in all, this is instead implemented with atomics and lock-free
30-
// operations! Whee! Each `Once` has one word of atomic state, and this state is
31-
// CAS'd on to determine what to do. There are four possible state of a `Once`:
32-
//
33-
// * Incomplete - no initialization has run yet, and no thread is currently
34-
// using the Once.
35-
// * Poisoned - some thread has previously attempted to initialize the Once, but
36-
// it panicked, so the Once is now poisoned. There are no other
37-
// threads currently accessing this Once.
38-
// * Running - some thread is currently attempting to run initialization. It may
39-
// succeed, so all future threads need to wait for it to finish.
40-
// Note that this state is accompanied with a payload, described
41-
// below.
42-
// * Complete - initialization has completed and all future calls should finish
43-
// immediately.
44-
//
45-
// With 4 states we need 2 bits to encode this, and we use the remaining bits
46-
// in the word we have allocated as a queue of threads waiting for the thread
47-
// responsible for entering the RUNNING state. This queue is just a linked list
48-
// of Waiter nodes which is monotonically increasing in size. Each node is
49-
// allocated on the stack, and whenever the running closure finishes it will
50-
// consume the entire queue and notify all waiters they should try again.
51-
//
52-
// You'll find a few more details in the implementation, but that's the gist of
53-
// it!
15+
// This primitive is now just a wrapper around `parking_lot::Once`. Which is faster,
16+
// uses less memory and has fewer constraints than the implementation that was
17+
// here before.
5418

55-
use crate::fmt;
56-
use crate::marker;
57-
use crate::ptr;
58-
use crate::sync::atomic::{AtomicUsize, AtomicBool, Ordering};
59-
use crate::thread::{self, Thread};
19+
use crate::{fmt, parking_lot};
6020

6121
/// A synchronization primitive which can be used to run a one-time global
6222
/// initialization. Useful for one-time initialization for FFI or related
@@ -79,19 +39,9 @@ use crate::thread::{self, Thread};
7939
/// ```
8040
#[stable(feature = "rust1", since = "1.0.0")]
8141
pub struct Once {
82-
// This `state` word is actually an encoded version of just a pointer to a
83-
// `Waiter`, so we add the `PhantomData` appropriately.
84-
state: AtomicUsize,
85-
_marker: marker::PhantomData<*mut Waiter>,
42+
inner: parking_lot::Once,
8643
}
8744

88-
// The `PhantomData` of a raw pointer removes these two auto traits, but we
89-
// enforce both below in the implementation so this should be safe to add.
90-
#[stable(feature = "rust1", since = "1.0.0")]
91-
unsafe impl Sync for Once {}
92-
#[stable(feature = "rust1", since = "1.0.0")]
93-
unsafe impl Send for Once {}
94-
9545
/// State yielded to [`call_once_force`]’s closure parameter. The state can be
9646
/// used to query the poison status of the [`Once`].
9747
///
@@ -117,38 +67,12 @@ pub struct OnceState {
11767
#[stable(feature = "rust1", since = "1.0.0")]
11868
pub const ONCE_INIT: Once = Once::new();
11969

120-
// Four states that a Once can be in, encoded into the lower bits of `state` in
121-
// the Once structure.
122-
const INCOMPLETE: usize = 0x0;
123-
const POISONED: usize = 0x1;
124-
const RUNNING: usize = 0x2;
125-
const COMPLETE: usize = 0x3;
126-
127-
// Mask to learn about the state. All other bits are the queue of waiters if
128-
// this is in the RUNNING state.
129-
const STATE_MASK: usize = 0x3;
130-
131-
// Representation of a node in the linked list of waiters in the RUNNING state.
132-
struct Waiter {
133-
thread: Option<Thread>,
134-
signaled: AtomicBool,
135-
next: *mut Waiter,
136-
}
137-
138-
// Helper struct used to clean up after a closure call with a `Drop`
139-
// implementation to also run on panic.
140-
struct Finish<'a> {
141-
panicked: bool,
142-
me: &'a Once,
143-
}
144-
14570
impl Once {
14671
/// Creates a new `Once` value.
14772
#[stable(feature = "once_new", since = "1.2.0")]
14873
pub const fn new() -> Once {
14974
Once {
150-
state: AtomicUsize::new(INCOMPLETE),
151-
_marker: marker::PhantomData,
75+
inner: parking_lot::Once::new(),
15276
}
15377
}
15478

@@ -167,8 +91,7 @@ impl Once {
16791
/// return).
16892
///
16993
/// If the given closure recursively invokes `call_once` on the same `Once`
170-
/// instance the exact behavior is not specified, allowed outcomes are
171-
/// a panic or a deadlock.
94+
/// instance, it will deadlock.
17295
///
17396
/// # Examples
17497
///
@@ -210,14 +133,11 @@ impl Once {
210133
///
211134
/// [poison]: struct.Mutex.html#poisoning
212135
#[stable(feature = "rust1", since = "1.0.0")]
213-
pub fn call_once<F>(&self, f: F) where F: FnOnce() {
214-
// Fast path check
215-
if self.is_completed() {
216-
return;
217-
}
218-
219-
let mut f = Some(f);
220-
self.call_inner(false, &mut |_| f.take().unwrap()());
136+
pub fn call_once<F>(&self, f: F)
137+
where
138+
F: FnOnce(),
139+
{
140+
self.inner.call_once(f);
221141
}
222142

223143
/// Performs the same function as [`call_once`] except ignores poisoning.
@@ -267,16 +187,16 @@ impl Once {
267187
/// INIT.call_once(|| {});
268188
/// ```
269189
#[unstable(feature = "once_poison", issue = "33577")]
270-
pub fn call_once_force<F>(&self, f: F) where F: FnOnce(&OnceState) {
271-
// Fast path check
272-
if self.is_completed() {
273-
return;
274-
}
275-
276-
let mut f = Some(f);
277-
self.call_inner(true, &mut |p| {
278-
f.take().unwrap()(&OnceState { poisoned: p })
279-
});
190+
pub fn call_once_force<F>(&self, f: F)
191+
where
192+
F: FnOnce(&OnceState),
193+
{
194+
self.inner
195+
.call_once_force(move |state: parking_lot::OnceState| {
196+
f(&OnceState {
197+
poisoned: state.poisoned(),
198+
});
199+
});
280200
}
281201

282202
/// Returns `true` if some `call_once` call has completed
@@ -322,110 +242,7 @@ impl Once {
322242
#[unstable(feature = "once_is_completed", issue = "54890")]
323243
#[inline]
324244
pub fn is_completed(&self) -> bool {
325-
// An `Acquire` load is enough because that makes all the initialization
326-
// operations visible to us, and, this being a fast path, weaker
327-
// ordering helps with performance. This `Acquire` synchronizes with
328-
// `SeqCst` operations on the slow path.
329-
self.state.load(Ordering::Acquire) == COMPLETE
330-
}
331-
332-
// This is a non-generic function to reduce the monomorphization cost of
333-
// using `call_once` (this isn't exactly a trivial or small implementation).
334-
//
335-
// Additionally, this is tagged with `#[cold]` as it should indeed be cold
336-
// and it helps let LLVM know that calls to this function should be off the
337-
// fast path. Essentially, this should help generate more straight line code
338-
// in LLVM.
339-
//
340-
// Finally, this takes an `FnMut` instead of a `FnOnce` because there's
341-
// currently no way to take an `FnOnce` and call it via virtual dispatch
342-
// without some allocation overhead.
343-
#[cold]
344-
fn call_inner(&self,
345-
ignore_poisoning: bool,
346-
init: &mut dyn FnMut(bool)) {
347-
348-
// This cold path uses SeqCst consistently because the
349-
// performance difference really does not matter there, and
350-
// SeqCst minimizes the chances of something going wrong.
351-
let mut state = self.state.load(Ordering::SeqCst);
352-
353-
'outer: loop {
354-
match state {
355-
// If we're complete, then there's nothing to do, we just
356-
// jettison out as we shouldn't run the closure.
357-
COMPLETE => return,
358-
359-
// If we're poisoned and we're not in a mode to ignore
360-
// poisoning, then we panic here to propagate the poison.
361-
POISONED if !ignore_poisoning => {
362-
panic!("Once instance has previously been poisoned");
363-
}
364-
365-
// Otherwise if we see a poisoned or otherwise incomplete state
366-
// we will attempt to move ourselves into the RUNNING state. If
367-
// we succeed, then the queue of waiters starts at null (all 0
368-
// bits).
369-
POISONED |
370-
INCOMPLETE => {
371-
let old = self.state.compare_and_swap(state, RUNNING,
372-
Ordering::SeqCst);
373-
if old != state {
374-
state = old;
375-
continue
376-
}
377-
378-
// Run the initialization routine, letting it know if we're
379-
// poisoned or not. The `Finish` struct is then dropped, and
380-
// the `Drop` implementation here is responsible for waking
381-
// up other waiters both in the normal return and panicking
382-
// case.
383-
let mut complete = Finish {
384-
panicked: true,
385-
me: self,
386-
};
387-
init(state == POISONED);
388-
complete.panicked = false;
389-
return
390-
}
391-
392-
// All other values we find should correspond to the RUNNING
393-
// state with an encoded waiter list in the more significant
394-
// bits. We attempt to enqueue ourselves by moving us to the
395-
// head of the list and bail out if we ever see a state that's
396-
// not RUNNING.
397-
_ => {
398-
assert!(state & STATE_MASK == RUNNING);
399-
let mut node = Waiter {
400-
thread: Some(thread::current()),
401-
signaled: AtomicBool::new(false),
402-
next: ptr::null_mut(),
403-
};
404-
let me = &mut node as *mut Waiter as usize;
405-
assert!(me & STATE_MASK == 0);
406-
407-
while state & STATE_MASK == RUNNING {
408-
node.next = (state & !STATE_MASK) as *mut Waiter;
409-
let old = self.state.compare_and_swap(state,
410-
me | RUNNING,
411-
Ordering::SeqCst);
412-
if old != state {
413-
state = old;
414-
continue
415-
}
416-
417-
// Once we've enqueued ourselves, wait in a loop.
418-
// Afterwards reload the state and continue with what we
419-
// were doing from before.
420-
while !node.signaled.load(Ordering::SeqCst) {
421-
thread::park();
422-
}
423-
state = self.state.load(Ordering::SeqCst);
424-
continue 'outer
425-
}
426-
}
427-
}
428-
}
245+
self.inner.state().done()
429246
}
430247
}
431248

@@ -436,34 +253,6 @@ impl fmt::Debug for Once {
436253
}
437254
}
438255

439-
impl Drop for Finish<'_> {
440-
fn drop(&mut self) {
441-
// Swap out our state with however we finished. We should only ever see
442-
// an old state which was RUNNING.
443-
let queue = if self.panicked {
444-
self.me.state.swap(POISONED, Ordering::SeqCst)
445-
} else {
446-
self.me.state.swap(COMPLETE, Ordering::SeqCst)
447-
};
448-
assert_eq!(queue & STATE_MASK, RUNNING);
449-
450-
// Decode the RUNNING to a list of waiters, then walk that entire list
451-
// and wake them up. Note that it is crucial that after we store `true`
452-
// in the node it can be free'd! As a result we load the `thread` to
453-
// signal ahead of time and then unpark it after the store.
454-
unsafe {
455-
let mut queue = (queue & !STATE_MASK) as *mut Waiter;
456-
while !queue.is_null() {
457-
let next = (*queue).next;
458-
let thread = (*queue).thread.take().unwrap();
459-
(*queue).signaled.store(true, Ordering::SeqCst);
460-
thread.unpark();
461-
queue = next;
462-
}
463-
}
464-
}
465-
}
466-
467256
impl OnceState {
468257
/// Returns `true` if the associated [`Once`] was poisoned prior to the
469258
/// invocation of the closure passed to [`call_once_force`].
@@ -514,10 +303,10 @@ impl OnceState {
514303

515304
#[cfg(all(test, not(target_os = "emscripten")))]
516305
mod tests {
306+
use super::Once;
517307
use crate::panic;
518308
use crate::sync::mpsc::channel;
519309
use crate::thread;
520-
use super::Once;
521310

522311
#[test]
523312
fn smoke_once() {
@@ -537,8 +326,10 @@ mod tests {
537326
let (tx, rx) = channel();
538327
for _ in 0..10 {
539328
let tx = tx.clone();
540-
thread::spawn(move|| {
541-
for _ in 0..4 { thread::yield_now() }
329+
thread::spawn(move || {
330+
for _ in 0..4 {
331+
thread::yield_now()
332+
}
542333
unsafe {
543334
O.call_once(|| {
544335
assert!(!RUN);
@@ -627,6 +418,5 @@ mod tests {
627418

628419
assert!(t1.join().is_ok());
629420
assert!(t2.join().is_ok());
630-
631421
}
632422
}

0 commit comments

Comments
 (0)