From 4e80909b6b592853028ddc803e73ab83604d2a7c Mon Sep 17 00:00:00 2001 From: Noah Kennedy Date: Wed, 21 Sep 2022 12:15:12 -0500 Subject: [PATCH] chore: vendor once_cell Fixes #3212. With the release of `v1.15`, the once_cell crate is breaking our current MSRV. Vendoring it will allow us to keep using certain APIs from that crate without compromising our MSRV. --- tokio/Cargo.toml | 8 +- tokio/src/process/unix/mod.rs | 2 +- tokio/src/signal/registry.rs | 2 +- tokio/src/util/mod.rs | 3 + tokio/src/util/once_cell/imp_pl.rs | 173 +++++++ tokio/src/util/once_cell/imp_std.rs | 423 +++++++++++++++++ tokio/src/util/once_cell/mod.rs | 695 ++++++++++++++++++++++++++++ 7 files changed, 1300 insertions(+), 6 deletions(-) create mode 100644 tokio/src/util/once_cell/imp_pl.rs create mode 100644 tokio/src/util/once_cell/imp_std.rs create mode 100644 tokio/src/util/once_cell/mod.rs diff --git a/tokio/Cargo.toml b/tokio/Cargo.toml index 2e172ee8512..046c68583f0 100644 --- a/tokio/Cargo.toml +++ b/tokio/Cargo.toml @@ -63,7 +63,7 @@ net = [ ] process = [ "bytes", - "once_cell", + "parking_lot_core", "libc", "mio/os-poll", "mio/os-ext", @@ -77,13 +77,13 @@ process = [ "winapi/winnt", ] # Includes basic task execution capabilities -rt = ["once_cell"] +rt = ["parking_lot_core"] rt-multi-thread = [ "num_cpus", "rt", ] signal = [ - "once_cell", + "parking_lot_core", "libc", "mio/os-poll", "mio/net", @@ -112,11 +112,11 @@ pin-project-lite = "0.2.0" # Everything else is optional... bytes = { version = "1.0.0", optional = true } -once_cell = { version = "1.5.2", optional = true } memchr = { version = "2.2", optional = true } mio = { version = "0.8.4", optional = true } num_cpus = { version = "1.8.0", optional = true } parking_lot = { version = "0.12.0", optional = true } +parking_lot_core = { version = "0.9.3", optional = true } [target.'cfg(not(any(target_arch = "wasm32", target_arch = "wasm64")))'.dependencies] socket2 = { version = "0.4.4", optional = true, features = [ "all" ] } diff --git a/tokio/src/process/unix/mod.rs b/tokio/src/process/unix/mod.rs index ba34c852b58..7e082bbc79f 100644 --- a/tokio/src/process/unix/mod.rs +++ b/tokio/src/process/unix/mod.rs @@ -35,9 +35,9 @@ use crate::process::SpawnedChild; use crate::signal::unix::driver::Handle as SignalHandle; use crate::signal::unix::{signal, Signal, SignalKind}; +use crate::util::once_cell::sync::Lazy; use mio::event::Source; use mio::unix::SourceFd; -use once_cell::sync::Lazy; use std::fmt; use std::fs::File; use std::future::Future; diff --git a/tokio/src/signal/registry.rs b/tokio/src/signal/registry.rs index 7795ca8dfa8..c42284d48e9 100644 --- a/tokio/src/signal/registry.rs +++ b/tokio/src/signal/registry.rs @@ -4,7 +4,7 @@ use crate::signal::os::{OsExtraData, OsStorage}; use crate::sync::watch; -use once_cell::sync::Lazy; +use crate::util::once_cell::sync::Lazy; use std::ops; use std::pin::Pin; use std::sync::atomic::{AtomicBool, Ordering}; diff --git a/tokio/src/util/mod.rs b/tokio/src/util/mod.rs index 65907231b61..ef707c83546 100644 --- a/tokio/src/util/mod.rs +++ b/tokio/src/util/mod.rs @@ -6,6 +6,9 @@ cfg_io_driver! { #[cfg(feature = "rt")] pub(crate) mod atomic_cell; +#[cfg(any(feature = "process", feature = "rt", feature = "signal"))] +pub(crate) mod once_cell; + #[cfg(any( // io driver uses `WakeList` directly feature = "net", diff --git a/tokio/src/util/once_cell/imp_pl.rs b/tokio/src/util/once_cell/imp_pl.rs new file mode 100644 index 00000000000..d2beb158dad --- /dev/null +++ b/tokio/src/util/once_cell/imp_pl.rs @@ -0,0 +1,173 @@ +use std::{ + cell::UnsafeCell, + hint, + panic::{RefUnwindSafe, UnwindSafe}, + sync::atomic::{AtomicU8, Ordering}, +}; + +pub(crate) struct OnceCell { + state: AtomicU8, + value: UnsafeCell>, +} + +const INCOMPLETE: u8 = 0x0; +const RUNNING: u8 = 0x1; +const COMPLETE: u8 = 0x2; + +// Why do we need `T: Send`? +// Thread A creates a `OnceCell` and shares it with +// scoped thread B, which fills the cell, which is +// then destroyed by A. That is, destructor observes +// a sent value. +unsafe impl Sync for OnceCell {} +unsafe impl Send for OnceCell {} + +impl RefUnwindSafe for OnceCell {} +impl UnwindSafe for OnceCell {} + +impl OnceCell { + pub(crate) const fn new() -> OnceCell { + OnceCell { + state: AtomicU8::new(INCOMPLETE), + value: UnsafeCell::new(None), + } + } + + pub(crate) const fn with_value(value: T) -> OnceCell { + OnceCell { + state: AtomicU8::new(COMPLETE), + value: UnsafeCell::new(Some(value)), + } + } + + /// Safety: synchronizes with store to value via Release/Acquire. + #[inline] + pub(crate) fn is_initialized(&self) -> bool { + self.state.load(Ordering::Acquire) == COMPLETE + } + + /// Safety: synchronizes with store to value via `is_initialized` or mutex + /// lock/unlock, writes value only once because of the mutex. + #[cold] + pub(crate) fn initialize(&self, f: F) -> Result<(), E> + where + F: FnOnce() -> Result, + { + let mut f = Some(f); + let mut res: Result<(), E> = Ok(()); + let slot: *mut Option = self.value.get(); + initialize_inner(&self.state, &mut || { + // We are calling user-supplied function and need to be careful. + // - if it returns Err, we unlock mutex and return without touching anything + // - if it panics, we unlock mutex and propagate panic without touching anything + // - if it calls `set` or `get_or_try_init` re-entrantly, we get a deadlock on + // mutex, which is important for safety. We *could* detect this and panic, + // but that is more complicated + // - finally, if it returns Ok, we store the value and store the flag with + // `Release`, which synchronizes with `Acquire`s. + let f = unsafe { super::take_unchecked(&mut f) }; + match f() { + Ok(value) => unsafe { + // Safe b/c we have a unique access and no panic may happen + // until the cell is marked as initialized. + debug_assert!((*slot).is_none()); + *slot = Some(value); + true + }, + Err(err) => { + res = Err(err); + false + } + } + }); + res + } + + /// Get the reference to the underlying value, without checking if the cell + /// is initialized. + /// + /// # Safety + /// + /// Caller must ensure that the cell is in initialized state, and that + /// the contents are acquired by (synchronized to) this thread. + pub(crate) unsafe fn get_unchecked(&self) -> &T { + debug_assert!(self.is_initialized()); + let slot: &Option = &*self.value.get(); + match slot { + Some(value) => value, + // This unsafe does improve performance, see `examples/bench`. + None => { + debug_assert!(false); + hint::unreachable_unchecked() + } + } + } + + /// Gets the mutable reference to the underlying value. + /// Returns `None` if the cell is empty. + pub(crate) fn get_mut(&mut self) -> Option<&mut T> { + // Safe b/c we have an exclusive access + let slot: &mut Option = unsafe { &mut *self.value.get() }; + slot.as_mut() + } +} + +struct Guard<'a> { + state: &'a AtomicU8, + new_state: u8, +} + +impl<'a> Drop for Guard<'a> { + fn drop(&mut self) { + self.state.store(self.new_state, Ordering::Release); + unsafe { + let key = self.state as *const AtomicU8 as usize; + parking_lot_core::unpark_all(key, parking_lot_core::DEFAULT_UNPARK_TOKEN); + } + } +} + +// Note: this is intentionally monomorphic +#[inline(never)] +fn initialize_inner(state: &AtomicU8, init: &mut dyn FnMut() -> bool) { + loop { + let exchange = + state.compare_exchange_weak(INCOMPLETE, RUNNING, Ordering::Acquire, Ordering::Acquire); + match exchange { + Ok(_) => { + let mut guard = Guard { + state, + new_state: INCOMPLETE, + }; + if init() { + guard.new_state = COMPLETE; + } + return; + } + Err(COMPLETE) => return, + Err(RUNNING) => unsafe { + let key = state as *const AtomicU8 as usize; + parking_lot_core::park( + key, + || state.load(Ordering::Relaxed) == RUNNING, + || (), + |_, _| (), + parking_lot_core::DEFAULT_PARK_TOKEN, + None, + ); + }, + Err(INCOMPLETE) => (), + Err(_) => debug_assert!(false), + } + } +} + +#[test] +fn test_size() { + use std::mem::size_of; + + assert_eq!( + size_of::>(), + 1 * size_of::() + size_of::() + ); +} diff --git a/tokio/src/util/once_cell/imp_std.rs b/tokio/src/util/once_cell/imp_std.rs new file mode 100644 index 00000000000..304e3e05436 --- /dev/null +++ b/tokio/src/util/once_cell/imp_std.rs @@ -0,0 +1,423 @@ +// There's a lot of scary concurrent code in this module, but it is copied from +// `std::sync::Once` with two changes: +// * no poisoning +// * init function can fail + +use std::{ + cell::{Cell, UnsafeCell}, + hint::unreachable_unchecked, + marker::PhantomData, + panic::{RefUnwindSafe, UnwindSafe}, + sync::atomic::{AtomicBool, AtomicPtr, Ordering}, + thread::{self, Thread}, +}; + +use super::take_unchecked; + +#[derive(Debug)] +pub(crate) struct OnceCell { + // This `queue` field is the core of the implementation. It encodes two + // pieces of information: + // + // * The current state of the cell (`INCOMPLETE`, `RUNNING`, `COMPLETE`) + // * Linked list of threads waiting for the current cell. + // + // State is encoded in two low bits. Only `INCOMPLETE` and `RUNNING` states + // allow waiters. + queue: AtomicPtr, + _marker: PhantomData<*mut Waiter>, + value: UnsafeCell>, +} + +// Why do we need `T: Send`? +// Thread A creates a `OnceCell` and shares it with +// scoped thread B, which fills the cell, which is +// then destroyed by A. That is, destructor observes +// a sent value. +unsafe impl Sync for OnceCell {} +unsafe impl Send for OnceCell {} + +impl RefUnwindSafe for OnceCell {} +impl UnwindSafe for OnceCell {} + +impl OnceCell { + pub(crate) const fn new() -> OnceCell { + OnceCell { + queue: AtomicPtr::new(INCOMPLETE_PTR), + _marker: PhantomData, + value: UnsafeCell::new(None), + } + } + + pub(crate) const fn with_value(value: T) -> OnceCell { + OnceCell { + queue: AtomicPtr::new(COMPLETE_PTR), + _marker: PhantomData, + value: UnsafeCell::new(Some(value)), + } + } + + /// Safety: synchronizes with store to value via Release/(Acquire|SeqCst). + #[inline] + pub(crate) fn is_initialized(&self) -> bool { + // An `Acquire` load is enough because that makes all the initialization + // operations visible to us, and, this being a fast path, weaker + // ordering helps with performance. This `Acquire` synchronizes with + // `SeqCst` operations on the slow path. + self.queue.load(Ordering::Acquire) == COMPLETE_PTR + } + + /// Safety: synchronizes with store to value via SeqCst read from state, + /// writes value only once because we never get to INCOMPLETE state after a + /// successful write. + #[cold] + pub(crate) fn initialize(&self, f: F) -> Result<(), E> + where + F: FnOnce() -> Result, + { + let mut f = Some(f); + let mut res: Result<(), E> = Ok(()); + let slot: *mut Option = self.value.get(); + initialize_or_wait( + &self.queue, + Some(&mut || { + let f = unsafe { take_unchecked(&mut f) }; + match f() { + Ok(value) => { + unsafe { *slot = Some(value) }; + true + } + Err(err) => { + res = Err(err); + false + } + } + }), + ); + res + } + + /// Get the reference to the underlying value, without checking if the cell + /// is initialized. + /// + /// # Safety + /// + /// Caller must ensure that the cell is in initialized state, and that + /// the contents are acquired by (synchronized to) this thread. + pub(crate) unsafe fn get_unchecked(&self) -> &T { + debug_assert!(self.is_initialized()); + let slot: &Option = &*self.value.get(); + match slot { + Some(value) => value, + // This unsafe does improve performance, see `examples/bench`. + None => { + debug_assert!(false); + unreachable_unchecked() + } + } + } + + /// Gets the mutable reference to the underlying value. + /// Returns `None` if the cell is empty. + pub(crate) fn get_mut(&mut self) -> Option<&mut T> { + // Safe b/c we have a unique access. + unsafe { &mut *self.value.get() }.as_mut() + } +} + +// Three states that a OnceCell can be in, encoded into the lower bits of `queue` in +// the OnceCell structure. +const INCOMPLETE: usize = 0x0; +const RUNNING: usize = 0x1; +const COMPLETE: usize = 0x2; +const INCOMPLETE_PTR: *mut Waiter = INCOMPLETE as *mut Waiter; +const COMPLETE_PTR: *mut Waiter = COMPLETE as *mut Waiter; + +// Mask to learn about the state. All other bits are the queue of waiters if +// this is in the RUNNING state. +const STATE_MASK: usize = 0x3; + +/// Representation of a node in the linked list of waiters in the RUNNING state. +/// A waiters is stored on the stack of the waiting threads. +#[repr(align(4))] // Ensure the two lower bits are free to use as state bits. +struct Waiter { + thread: Cell>, + signaled: AtomicBool, + next: *mut Waiter, +} + +/// Drains and notifies the queue of waiters on drop. +struct Guard<'a> { + queue: &'a AtomicPtr, + new_queue: *mut Waiter, +} + +impl Drop for Guard<'_> { + fn drop(&mut self) { + let queue = self.queue.swap(self.new_queue, Ordering::AcqRel); + + let state = strict::addr(queue) & STATE_MASK; + assert_eq!(state, RUNNING); + + unsafe { + let mut waiter = strict::map_addr(queue, |q| q & !STATE_MASK); + while !waiter.is_null() { + let next = (*waiter).next; + let thread = (*waiter).thread.take().unwrap(); + (*waiter).signaled.store(true, Ordering::Release); + waiter = next; + thread.unpark(); + } + } + } +} + +// Corresponds to `std::sync::Once::call_inner`. +// +// Originally copied from std, but since modified to remove poisoning and to +// support wait. +// +// Note: this is intentionally monomorphic +#[inline(never)] +fn initialize_or_wait(queue: &AtomicPtr, mut init: Option<&mut dyn FnMut() -> bool>) { + let mut curr_queue = queue.load(Ordering::Acquire); + + loop { + let curr_state = strict::addr(curr_queue) & STATE_MASK; + match (curr_state, &mut init) { + (COMPLETE, _) => return, + (INCOMPLETE, Some(init)) => { + let exchange = queue.compare_exchange( + curr_queue, + strict::map_addr(curr_queue, |q| (q & !STATE_MASK) | RUNNING), + Ordering::Acquire, + Ordering::Acquire, + ); + if let Err(new_queue) = exchange { + curr_queue = new_queue; + continue; + } + let mut guard = Guard { + queue, + new_queue: INCOMPLETE_PTR, + }; + if init() { + guard.new_queue = COMPLETE_PTR; + } + return; + } + (INCOMPLETE, None) | (RUNNING, _) => { + wait(&queue, curr_queue); + curr_queue = queue.load(Ordering::Acquire); + } + _ => debug_assert!(false), + } + } +} + +fn wait(queue: &AtomicPtr, mut curr_queue: *mut Waiter) { + let curr_state = strict::addr(curr_queue) & STATE_MASK; + loop { + let node = Waiter { + thread: Cell::new(Some(thread::current())), + signaled: AtomicBool::new(false), + next: strict::map_addr(curr_queue, |q| q & !STATE_MASK), + }; + let me = &node as *const Waiter as *mut Waiter; + + let exchange = queue.compare_exchange( + curr_queue, + strict::map_addr(me, |q| q | curr_state), + Ordering::Release, + Ordering::Relaxed, + ); + if let Err(new_queue) = exchange { + if strict::addr(new_queue) & STATE_MASK != curr_state { + return; + } + curr_queue = new_queue; + continue; + } + + while !node.signaled.load(Ordering::Acquire) { + thread::park(); + } + break; + } +} + +// Polyfill of strict provenance from https://crates.io/crates/sptr. +// +// Use free-standing function rather than a trait to keep things simple and +// avoid any potential conflicts with future stabile std API. +mod strict { + #[must_use] + #[inline] + pub(crate) fn addr(ptr: *mut T) -> usize + where + T: Sized, + { + // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. + // SAFETY: Pointer-to-integer transmutes are valid (if you are okay with losing the + // provenance). + unsafe { core::mem::transmute(ptr) } + } + + #[must_use] + #[inline] + pub(crate) fn with_addr(ptr: *mut T, addr: usize) -> *mut T + where + T: Sized, + { + // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. + // + // In the mean-time, this operation is defined to be "as if" it was + // a wrapping_offset, so we can emulate it as such. This should properly + // restore pointer provenance even under today's compiler. + let self_addr = self::addr(ptr) as isize; + let dest_addr = addr as isize; + let offset = dest_addr.wrapping_sub(self_addr); + + // This is the canonical desugarring of this operation, + // but `pointer::cast` was only stabilized in 1.38. + // self.cast::().wrapping_offset(offset).cast::() + (ptr as *mut u8).wrapping_offset(offset) as *mut T + } + + #[must_use] + #[inline] + pub(crate) fn map_addr(ptr: *mut T, f: impl FnOnce(usize) -> usize) -> *mut T + where + T: Sized, + { + self::with_addr(ptr, f(addr(ptr))) + } +} + +// These test are snatched from std as well. +#[cfg(test)] +mod tests { + use std::panic; + use std::{sync::mpsc::channel, thread}; + + use super::OnceCell; + + impl OnceCell { + fn init(&self, f: impl FnOnce() -> T) { + enum Void {} + let _ = self.initialize(|| Ok::(f())); + } + } + + #[test] + fn smoke_once() { + static O: OnceCell<()> = OnceCell::new(); + let mut a = 0; + O.init(|| a += 1); + assert_eq!(a, 1); + O.init(|| a += 1); + assert_eq!(a, 1); + } + + #[test] + fn stampede_once() { + static O: OnceCell<()> = OnceCell::new(); + static mut RUN: bool = false; + + let (tx, rx) = channel(); + for _ in 0..10 { + let tx = tx.clone(); + thread::spawn(move || { + for _ in 0..4 { + thread::yield_now() + } + unsafe { + O.init(|| { + assert!(!RUN); + RUN = true; + }); + assert!(RUN); + } + tx.send(()).unwrap(); + }); + } + + unsafe { + O.init(|| { + assert!(!RUN); + RUN = true; + }); + assert!(RUN); + } + + for _ in 0..10 { + rx.recv().unwrap(); + } + } + + #[test] + fn poison_bad() { + static O: OnceCell<()> = OnceCell::new(); + + // poison the once + let t = panic::catch_unwind(|| { + O.init(|| panic!()); + }); + assert!(t.is_err()); + + // we can subvert poisoning, however + let mut called = false; + O.init(|| { + called = true; + }); + assert!(called); + + // once any success happens, we stop propagating the poison + O.init(|| {}); + } + + #[test] + fn wait_for_force_to_finish() { + static O: OnceCell<()> = OnceCell::new(); + + // poison the once + let t = panic::catch_unwind(|| { + O.init(|| panic!()); + }); + assert!(t.is_err()); + + // make sure someone's waiting inside the once via a force + let (tx1, rx1) = channel(); + let (tx2, rx2) = channel(); + let t1 = thread::spawn(move || { + O.init(|| { + tx1.send(()).unwrap(); + rx2.recv().unwrap(); + }); + }); + + rx1.recv().unwrap(); + + // put another waiter on the once + let t2 = thread::spawn(|| { + let mut called = false; + O.init(|| { + called = true; + }); + assert!(!called); + }); + + tx2.send(()).unwrap(); + + assert!(t1.join().is_ok()); + assert!(t2.join().is_ok()); + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn test_size() { + use std::mem::size_of; + + assert_eq!(size_of::>(), 4 * size_of::()); + } +} diff --git a/tokio/src/util/once_cell/mod.rs b/tokio/src/util/once_cell/mod.rs new file mode 100644 index 00000000000..5feb4151772 --- /dev/null +++ b/tokio/src/util/once_cell/mod.rs @@ -0,0 +1,695 @@ +//! # Overview +//! +//! Vendored from [here](https://github.com/matklad/once_cell/tree/v1.14.0) +//! +//! `once_cell` provides two new cell-like types, [`unsync::OnceCell`] and [`sync::OnceCell`]. A `OnceCell` +//! might store arbitrary non-`Copy` types, can be assigned to at most once and provides direct access +//! to the stored contents. The core API looks *roughly* like this (and there's much more inside, read on!): +//! +//! ```rust,ignore +//! impl OnceCell { +//! const fn new() -> OnceCell { ... } +//! fn set(&self, value: T) -> Result<(), T> { ... } +//! fn get(&self) -> Option<&T> { ... } +//! } +//! ``` +//! +//! Note that, like with [`RefCell`] and [`Mutex`], the `set` method requires only a shared reference. +//! Because of the single assignment restriction `get` can return a `&T` instead of `Ref` +//! or `MutexGuard`. +//! +//! The `sync` flavor is thread-safe (that is, implements the [`Sync`] trait), while the `unsync` one is not. +//! +//! [`unsync::OnceCell`]: unsync/struct.OnceCell.html +//! [`sync::OnceCell`]: sync/struct.OnceCell.html +//! [`RefCell`]: https://doc.rust-lang.org/std/cell/struct.RefCell.html +//! [`Mutex`]: https://doc.rust-lang.org/std/sync/struct.Mutex.html +//! [`Sync`]: https://doc.rust-lang.org/std/marker/trait.Sync.html +//! +//! # Recipes +//! +//! `OnceCell` might be useful for a variety of patterns. +//! +//! ## Safe Initialization of Global Data +//! +//! ```rust +//! use std::{env, io}; +//! +//! use once_cell::sync::OnceCell; +//! +//! #[derive(Debug)] +//! pub struct Logger { +//! // ... +//! } +//! static INSTANCE: OnceCell = OnceCell::new(); +//! +//! impl Logger { +//! pub fn global() -> &'static Logger { +//! INSTANCE.get().expect("logger is not initialized") +//! } +//! +//! fn from_cli(args: env::Args) -> Result { +//! // ... +//! # Ok(Logger {}) +//! } +//! } +//! +//! fn main() { +//! let logger = Logger::from_cli(env::args()).unwrap(); +//! INSTANCE.set(logger).unwrap(); +//! // use `Logger::global()` from now on +//! } +//! ``` +//! +//! ## Lazy Initialized Global Data +//! +//! This is essentially the `lazy_static!` macro, but without a macro. +//! +//! ```rust +//! use std::{sync::Mutex, collections::HashMap}; +//! +//! use once_cell::sync::OnceCell; +//! +//! fn global_data() -> &'static Mutex> { +//! static INSTANCE: OnceCell>> = OnceCell::new(); +//! INSTANCE.get_or_init(|| { +//! let mut m = HashMap::new(); +//! m.insert(13, "Spica".to_string()); +//! m.insert(74, "Hoyten".to_string()); +//! Mutex::new(m) +//! }) +//! } +//! ``` +//! +//! There are also the [`sync::Lazy`] and [`unsync::Lazy`] convenience types to streamline this pattern: +//! +//! ```rust +//! use std::{sync::Mutex, collections::HashMap}; +//! use once_cell::sync::Lazy; +//! +//! static GLOBAL_DATA: Lazy>> = Lazy::new(|| { +//! let mut m = HashMap::new(); +//! m.insert(13, "Spica".to_string()); +//! m.insert(74, "Hoyten".to_string()); +//! Mutex::new(m) +//! }); +//! +//! fn main() { +//! println!("{:?}", GLOBAL_DATA.lock().unwrap()); +//! } +//! ``` +//! +//! Note that the variable that holds `Lazy` is declared as `static`, *not* +//! `const`. This is important: using `const` instead compiles, but works wrong. +//! +//! [`sync::Lazy`]: sync/struct.Lazy.html +//! [`unsync::Lazy`]: unsync/struct.Lazy.html +//! +//! ## General purpose lazy evaluation +//! +//! Unlike `lazy_static!`, `Lazy` works with local variables. +//! +//! ```rust +//! use once_cell::unsync::Lazy; +//! +//! fn main() { +//! let ctx = vec![1, 2, 3]; +//! let thunk = Lazy::new(|| { +//! ctx.iter().sum::() +//! }); +//! assert_eq!(*thunk, 6); +//! } +//! ``` +//! +//! If you need a lazy field in a struct, you probably should use `OnceCell` +//! directly, because that will allow you to access `self` during initialization. +//! +//! ```rust +//! use std::{fs, path::PathBuf}; +//! +//! use once_cell::unsync::OnceCell; +//! +//! struct Ctx { +//! config_path: PathBuf, +//! config: OnceCell, +//! } +//! +//! impl Ctx { +//! pub fn get_config(&self) -> Result<&str, std::io::Error> { +//! let cfg = self.config.get_or_try_init(|| { +//! fs::read_to_string(&self.config_path) +//! })?; +//! Ok(cfg.as_str()) +//! } +//! } +//! ``` +//! +//! ## Lazily Compiled Regex +//! +//! This is a `regex!` macro which takes a string literal and returns an +//! *expression* that evaluates to a `&'static Regex`: +//! +//! ``` +//! macro_rules! regex { +//! ($re:literal $(,)?) => {{ +//! static RE: once_cell::sync::OnceCell = once_cell::sync::OnceCell::new(); +//! RE.get_or_init(|| regex::Regex::new($re).unwrap()) +//! }}; +//! } +//! ``` +//! +//! This macro can be useful to avoid the "compile regex on every loop iteration" problem. +//! +//! ## Runtime `include_bytes!` +//! +//! The `include_bytes` macro is useful to include test resources, but it slows +//! down test compilation a lot. An alternative is to load the resources at +//! runtime: +//! +//! ``` +//! use std::path::Path; +//! +//! use once_cell::sync::OnceCell; +//! +//! pub struct TestResource { +//! path: &'static str, +//! cell: OnceCell>, +//! } +//! +//! impl TestResource { +//! pub const fn new(path: &'static str) -> TestResource { +//! TestResource { path, cell: OnceCell::new() } +//! } +//! pub fn bytes(&self) -> &[u8] { +//! self.cell.get_or_init(|| { +//! let dir = std::env::var("CARGO_MANIFEST_DIR").unwrap(); +//! let path = Path::new(dir.as_str()).join(self.path); +//! std::fs::read(&path).unwrap_or_else(|_err| { +//! panic!("failed to load test resource: {}", path.display()) +//! }) +//! }).as_slice() +//! } +//! } +//! +//! static TEST_IMAGE: TestResource = TestResource::new("test_data/lena.png"); +//! +//! #[test] +//! fn test_sobel_filter() { +//! let rgb: &[u8] = TEST_IMAGE.bytes(); +//! // ... +//! # drop(rgb); +//! } +//! ``` +//! +//! ## `lateinit` +//! +//! `LateInit` type for delayed initialization. It is reminiscent of Kotlin's +//! `lateinit` keyword and allows construction of cyclic data structures: +//! +//! +//! ``` +//! use once_cell::sync::OnceCell; +//! +//! #[derive(Debug)] +//! pub struct LateInit { cell: OnceCell } +//! +//! impl LateInit { +//! pub fn init(&self, value: T) { +//! assert!(self.cell.set(value).is_ok()) +//! } +//! } +//! +//! impl Default for LateInit { +//! fn default() -> Self { LateInit { cell: OnceCell::default() } } +//! } +//! +//! impl std::ops::Deref for LateInit { +//! type Target = T; +//! fn deref(&self) -> &T { +//! self.cell.get().unwrap() +//! } +//! } +//! +//! #[derive(Default, Debug)] +//! struct A<'a> { +//! b: LateInit<&'a B<'a>>, +//! } +//! +//! #[derive(Default, Debug)] +//! struct B<'a> { +//! a: LateInit<&'a A<'a>> +//! } +//! +//! fn build_cycle() { +//! let a = A::default(); +//! let b = B::default(); +//! a.b.init(&b); +//! b.a.init(&a); +//! println!("{:?}", a.b.a.b.a); +//! } +//! ``` +//! +//! # Comparison with std +//! +//! |`!Sync` types | Access Mode | Drawbacks | +//! |----------------------|------------------------|-----------------------------------------------| +//! |`Cell` | `T` | requires `T: Copy` for `get` | +//! |`RefCell` | `RefMut` / `Ref` | may panic at runtime | +//! |`unsync::OnceCell` | `&T` | assignable only once | +//! +//! |`Sync` types | Access Mode | Drawbacks | +//! |----------------------|------------------------|-----------------------------------------------| +//! |`AtomicT` | `T` | works only with certain `Copy` types | +//! |`Mutex` | `MutexGuard` | may deadlock at runtime, may block the thread | +//! |`sync::OnceCell` | `&T` | assignable only once, may block the thread | +//! +//! Technically, calling `get_or_init` will also cause a panic or a deadlock if it recursively calls +//! itself. However, because the assignment can happen only once, such cases should be more rare than +//! equivalents with `RefCell` and `Mutex`. +//! +//! # Minimum Supported `rustc` Version +//! +//! This crate's minimum supported `rustc` version is `1.36.0`. +//! +//! If only the `std` feature is enabled, MSRV will be updated conservatively. +//! When using other features, like `parking_lot`, MSRV might be updated more frequently, up to the latest stable. +//! In both cases, increasing MSRV is *not* considered a semver-breaking change. +//! +//! # Implementation details +//! +//! The implementation is based on the [`lazy_static`](https://github.com/rust-lang-nursery/lazy-static.rs/) +//! and [`lazy_cell`](https://github.com/indiv0/lazycell/) crates and [`std::sync::Once`]. In some sense, +//! `once_cell` just streamlines and unifies those APIs. +//! +//! To implement a sync flavor of `OnceCell`, this crates uses either a custom +//! re-implementation of `std::sync::Once` or `parking_lot::Mutex`. This is +//! controlled by the `parking_lot` feature (disabled by default). Performance +//! is the same for both cases, but the `parking_lot` based `OnceCell` is +//! smaller by up to 16 bytes. +//! +//! This crate uses `unsafe`. +//! +//! [`std::sync::Once`]: https://doc.rust-lang.org/std/sync/struct.Once.html +//! +//! # F.A.Q. +//! +//! **Should I use lazy_static or once_cell?** +//! +//! To the first approximation, `once_cell` is both more flexible and more convenient than `lazy_static` +//! and should be preferred. +//! +//! Unlike `once_cell`, `lazy_static` supports spinlock-based implementation of blocking which works with +//! `#![no_std]`. +//! +//! `lazy_static` has received significantly more real world testing, but `once_cell` is also a widely +//! used crate. +//! +//! **Should I use the sync or unsync flavor?** +//! +//! Because Rust compiler checks thread safety for you, it's impossible to accidentally use `unsync` where +//! `sync` is required. So, use `unsync` in single-threaded code and `sync` in multi-threaded. It's easy +//! to switch between the two if code becomes multi-threaded later. +//! +//! At the moment, `unsync` has an additional benefit that reentrant initialization causes a panic, which +//! might be easier to debug than a deadlock. +//! +//! **Does this crate support async?** +//! +//! No, but you can use [`async_once_cell`](https://crates.io/crates/async_once_cell) instead. +//! +//! # Related crates +//! +//! * [double-checked-cell](https://github.com/niklasf/double-checked-cell) +//! * [lazy-init](https://crates.io/crates/lazy-init) +//! * [lazycell](https://crates.io/crates/lazycell) +//! * [mitochondria](https://crates.io/crates/mitochondria) +//! * [lazy_static](https://crates.io/crates/lazy_static) +//! * [async_once_cell](https://crates.io/crates/async_once_cell) +//! +//! Most of this crate's functionality is available in `std` in nightly Rust. +//! See the [tracking issue](https://github.com/rust-lang/rust/issues/74465). + +#[cfg(feature = "parking_lot")] +#[path = "imp_pl.rs"] +mod imp; + +#[cfg(not(feature = "parking_lot"))] +#[path = "imp_std.rs"] +mod imp; + +/// Thread-safe, blocking version of `OnceCell`. +pub(crate) mod sync { + use std::{ + cell::Cell, + fmt, + ops::{Deref, DerefMut}, + panic::RefUnwindSafe, + }; + + use super::imp::OnceCell as Imp; + + /// A thread-safe cell which can be written to only once. + /// + /// `OnceCell` provides `&` references to the contents without RAII guards. + /// + /// Reading a non-`None` value out of `OnceCell` establishes a + /// happens-before relationship with a corresponding write. For example, if + /// thread A initializes the cell with `get_or_init(f)`, and thread B + /// subsequently reads the result of this call, B also observes all the side + /// effects of `f`. + /// + /// # Example + /// ``` + /// use once_cell::sync::OnceCell; + /// + /// static CELL: OnceCell = OnceCell::new(); + /// assert!(CELL.get().is_none()); + /// + /// std::thread::spawn(|| { + /// let value: &String = CELL.get_or_init(|| { + /// "Hello, World!".to_string() + /// }); + /// assert_eq!(value, "Hello, World!"); + /// }).join().unwrap(); + /// + /// let value: Option<&String> = CELL.get(); + /// assert!(value.is_some()); + /// assert_eq!(value.unwrap().as_str(), "Hello, World!"); + /// ``` + pub(crate) struct OnceCell(Imp); + + impl Default for OnceCell { + fn default() -> OnceCell { + OnceCell::new() + } + } + + impl fmt::Debug for OnceCell { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.get() { + Some(v) => f.debug_tuple("OnceCell").field(v).finish(), + None => f.write_str("OnceCell(Uninit)"), + } + } + } + + impl Clone for OnceCell { + fn clone(&self) -> OnceCell { + match self.get() { + Some(value) => Self::with_value(value.clone()), + None => Self::new(), + } + } + + fn clone_from(&mut self, source: &Self) { + match (self.get_mut(), source.get()) { + (Some(this), Some(source)) => this.clone_from(source), + _ => *self = source.clone(), + } + } + } + + impl From for OnceCell { + fn from(value: T) -> Self { + Self::with_value(value) + } + } + + impl PartialEq for OnceCell { + fn eq(&self, other: &OnceCell) -> bool { + self.get() == other.get() + } + } + + impl Eq for OnceCell {} + + impl OnceCell { + /// Creates a new empty cell. + pub(crate) const fn new() -> OnceCell { + OnceCell(Imp::new()) + } + + /// Creates a new initialized cell. + pub(crate) const fn with_value(value: T) -> OnceCell { + OnceCell(Imp::with_value(value)) + } + + /// Gets the reference to the underlying value. + /// + /// Returns `None` if the cell is empty, or being initialized. This + /// method never blocks. + pub(crate) fn get(&self) -> Option<&T> { + if self.0.is_initialized() { + // Safe b/c value is initialized. + Some(unsafe { self.get_unchecked() }) + } else { + None + } + } + + /// Gets the mutable reference to the underlying value. + /// + /// Returns `None` if the cell is empty. + /// + /// This method is allowed to violate the invariant of writing to a `OnceCell` + /// at most once because it requires `&mut` access to `self`. As with all + /// interior mutability, `&mut` access permits arbitrary modification: + /// + /// ``` + /// use once_cell::sync::OnceCell; + /// + /// let mut cell: OnceCell = OnceCell::new(); + /// cell.set(92).unwrap(); + /// cell = OnceCell::new(); + /// ``` + pub(crate) fn get_mut(&mut self) -> Option<&mut T> { + self.0.get_mut() + } + + /// Get the reference to the underlying value, without checking if the + /// cell is initialized. + /// + /// # Safety + /// + /// Caller must ensure that the cell is in initialized state, and that + /// the contents are acquired by (synchronized to) this thread. + pub(crate) unsafe fn get_unchecked(&self) -> &T { + self.0.get_unchecked() + } + + /// Gets the contents of the cell, initializing it with `f` if the cell + /// was empty. + /// + /// Many threads may call `get_or_init` concurrently with different + /// initializing functions, but it is guaranteed that only one function + /// will be executed. + /// + /// # Panics + /// + /// If `f` panics, the panic is propagated to the caller, and the cell + /// remains uninitialized. + /// + /// It is an error to reentrantly initialize the cell from `f`. The + /// exact outcome is unspecified. Current implementation deadlocks, but + /// this may be changed to a panic in the future. + /// + /// # Example + /// ``` + /// use once_cell::sync::OnceCell; + /// + /// let cell = OnceCell::new(); + /// let value = cell.get_or_init(|| 92); + /// assert_eq!(value, &92); + /// let value = cell.get_or_init(|| unreachable!()); + /// assert_eq!(value, &92); + /// ``` + pub(crate) fn get_or_init(&self, f: F) -> &T + where + F: FnOnce() -> T, + { + enum Void {} + match self.get_or_try_init(|| Ok::(f())) { + Ok(val) => val, + Err(void) => match void {}, + } + } + + /// Gets the contents of the cell, initializing it with `f` if + /// the cell was empty. If the cell was empty and `f` failed, an + /// error is returned. + /// + /// # Panics + /// + /// If `f` panics, the panic is propagated to the caller, and + /// the cell remains uninitialized. + /// + /// It is an error to reentrantly initialize the cell from `f`. + /// The exact outcome is unspecified. Current implementation + /// deadlocks, but this may be changed to a panic in the future. + /// + /// # Example + /// ``` + /// use once_cell::sync::OnceCell; + /// + /// let cell = OnceCell::new(); + /// assert_eq!(cell.get_or_try_init(|| Err(())), Err(())); + /// assert!(cell.get().is_none()); + /// let value = cell.get_or_try_init(|| -> Result { + /// Ok(92) + /// }); + /// assert_eq!(value, Ok(&92)); + /// assert_eq!(cell.get(), Some(&92)) + /// ``` + pub(crate) fn get_or_try_init(&self, f: F) -> Result<&T, E> + where + F: FnOnce() -> Result, + { + // Fast path check + if let Some(value) = self.get() { + return Ok(value); + } + self.0.initialize(f)?; + + // Safe b/c value is initialized. + debug_assert!(self.0.is_initialized()); + Ok(unsafe { self.get_unchecked() }) + } + } + + /// A value which is initialized on the first access. + /// + /// This type is thread-safe and can be used in statics. + /// + /// # Example + /// + /// ``` + /// use std::collections::HashMap; + /// + /// use once_cell::sync::Lazy; + /// + /// static HASHMAP: Lazy> = Lazy::new(|| { + /// println!("initializing"); + /// let mut m = HashMap::new(); + /// m.insert(13, "Spica".to_string()); + /// m.insert(74, "Hoyten".to_string()); + /// m + /// }); + /// + /// fn main() { + /// println!("ready"); + /// std::thread::spawn(|| { + /// println!("{:?}", HASHMAP.get(&13)); + /// }).join().unwrap(); + /// println!("{:?}", HASHMAP.get(&74)); + /// + /// // Prints: + /// // ready + /// // initializing + /// // Some("Spica") + /// // Some("Hoyten") + /// } + /// ``` + pub(crate) struct Lazy T> { + cell: OnceCell, + init: Cell>, + } + + impl fmt::Debug for Lazy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Lazy") + .field("cell", &self.cell) + .field("init", &"..") + .finish() + } + } + + // We never create a `&F` from a `&Lazy` so it is fine to not impl + // `Sync` for `F`. We do create a `&mut Option` in `force`, but this is + // properly synchronized, so it only happens once so it also does not + // contribute to this impl. + unsafe impl Sync for Lazy where OnceCell: Sync {} + // auto-derived `Send` impl is OK. + + impl RefUnwindSafe for Lazy where OnceCell: RefUnwindSafe {} + + impl Lazy { + /// Creates a new lazy value with the given initializing + /// function. + pub(crate) const fn new(f: F) -> Lazy { + Lazy { + cell: OnceCell::new(), + init: Cell::new(Some(f)), + } + } + } + + impl T> Lazy { + /// Forces the evaluation of this lazy value and + /// returns a reference to the result. This is equivalent + /// to the `Deref` impl, but is explicit. + /// + /// # Example + /// ``` + /// use once_cell::sync::Lazy; + /// + /// let lazy = Lazy::new(|| 92); + /// + /// assert_eq!(Lazy::force(&lazy), &92); + /// assert_eq!(&*lazy, &92); + /// ``` + pub(crate) fn force(this: &Lazy) -> &T { + this.cell.get_or_init(|| match this.init.take() { + Some(f) => f(), + None => panic!("Lazy instance has previously been poisoned"), + }) + } + } + + impl T> Deref for Lazy { + type Target = T; + fn deref(&self) -> &T { + Lazy::force(self) + } + } + + impl T> DerefMut for Lazy { + fn deref_mut(&mut self) -> &mut T { + Lazy::force(self); + self.cell.get_mut().unwrap_or_else(|| unreachable!()) + } + } + + impl Default for Lazy { + /// Creates a new lazy value using `Default` as the initializing function. + fn default() -> Lazy { + Lazy::new(T::default) + } + } + + /// ```compile_fail + /// struct S(*mut ()); + /// unsafe impl Sync for S {} + /// + /// fn share(_: &T) {} + /// share(&once_cell::sync::OnceCell::::new()); + /// ``` + /// + /// ```compile_fail + /// struct S(*mut ()); + /// unsafe impl Sync for S {} + /// + /// fn share(_: &T) {} + /// share(&once_cell::sync::Lazy::::new(|| unimplemented!())); + /// ``` + fn _dummy() {} +} + +unsafe fn take_unchecked(val: &mut Option) -> T { + match val.take() { + Some(it) => it, + None => { + debug_assert!(false); + std::hint::unreachable_unchecked() + } + } +}