diff --git a/Cargo.toml b/Cargo.toml index 3226a40152643..d5e77bc04ad87 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1600,6 +1600,9 @@ target_sdk_version = 31 icon = "@mipmap/ic_launcher" label = "Bevy Example" +[profile.release] +lto = true + [profile.wasm-release] inherits = "release" opt-level = "z" diff --git a/benches/Cargo.toml b/benches/Cargo.toml index 165fbce1cb25e..76e65bf947c53 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -17,6 +17,10 @@ bevy_reflect = { path = "../crates/bevy_reflect" } bevy_tasks = { path = "../crates/bevy_tasks" } bevy_utils = { path = "../crates/bevy_utils" } +[profile.release] +opt-level = 3 +lto = true + [[bench]] name = "ecs" path = "benches/bevy_ecs/benches.rs" diff --git a/crates/bevy_ecs/src/query/fetch.rs b/crates/bevy_ecs/src/query/fetch.rs index d87f2f959a2b2..ad6598bcc6e0d 100644 --- a/crates/bevy_ecs/src/query/fetch.rs +++ b/crates/bevy_ecs/src/query/fetch.rs @@ -3,7 +3,7 @@ use crate::{ change_detection::Ticks, component::{Component, ComponentId, ComponentStorage, ComponentTicks, StorageType}, entity::Entity, - query::{debug_checked_unreachable, Access, FilteredAccess}, + query::{Access, DebugCheckedUnwrap, FilteredAccess}, storage::{ComponentSparseSet, Table}, world::{Mut, World}, }; @@ -552,7 +552,7 @@ unsafe impl WorldQuery for &T { .storages() .sparse_sets .get(component_id) - .unwrap_or_else(|| debug_checked_unreachable()) + .debug_checked_unwrap() }), } } @@ -585,7 +585,7 @@ unsafe impl WorldQuery for &T { fetch.table_components = Some( table .get_column(component_id) - .unwrap_or_else(|| debug_checked_unreachable()) + .debug_checked_unwrap() .get_data_slice() .into(), ); @@ -600,14 +600,14 @@ unsafe impl WorldQuery for &T { match T::Storage::STORAGE_TYPE { StorageType::Table => fetch .table_components - .unwrap_or_else(|| debug_checked_unreachable()) + .debug_checked_unwrap() .get(table_row) .deref(), StorageType::SparseSet => fetch .sparse_set - .unwrap_or_else(|| debug_checked_unreachable()) + .debug_checked_unwrap() .get(entity) - .unwrap_or_else(|| debug_checked_unreachable()) + .debug_checked_unwrap() .deref(), } } @@ -696,7 +696,7 @@ unsafe impl<'__w, T: Component> WorldQuery for &'__w mut T { .storages() .sparse_sets .get(component_id) - .unwrap_or_else(|| debug_checked_unreachable()) + .debug_checked_unwrap() }), last_change_tick, change_tick, @@ -730,9 +730,7 @@ unsafe impl<'__w, T: Component> WorldQuery for &'__w mut T { &component_id: &ComponentId, table: &'w Table, ) { - let column = table - .get_column(component_id) - .unwrap_or_else(|| debug_checked_unreachable()); + let column = table.get_column(component_id).debug_checked_unwrap(); fetch.table_data = Some(( column.get_data_slice().into(), column.get_ticks_slice().into(), @@ -747,9 +745,7 @@ unsafe impl<'__w, T: Component> WorldQuery for &'__w mut T { ) -> Self::Item<'w> { match T::Storage::STORAGE_TYPE { StorageType::Table => { - let (table_components, table_ticks) = fetch - .table_data - .unwrap_or_else(|| debug_checked_unreachable()); + let (table_components, table_ticks) = fetch.table_data.debug_checked_unwrap(); Mut { value: table_components.get(table_row).deref_mut(), ticks: Ticks { @@ -762,9 +758,9 @@ unsafe impl<'__w, T: Component> WorldQuery for &'__w mut T { StorageType::SparseSet => { let (component, component_ticks) = fetch .sparse_set - .unwrap_or_else(|| debug_checked_unreachable()) + .debug_checked_unwrap() .get_with_ticks(entity) - .unwrap_or_else(|| debug_checked_unreachable()); + .debug_checked_unwrap(); Mut { value: component.assert_unique().deref_mut(), ticks: Ticks { @@ -1038,7 +1034,7 @@ unsafe impl WorldQuery for ChangeTrackers { .storages() .sparse_sets .get(component_id) - .unwrap_or_else(|| debug_checked_unreachable()) + .debug_checked_unwrap() }), marker: PhantomData, last_change_tick, @@ -1077,7 +1073,7 @@ unsafe impl WorldQuery for ChangeTrackers { fetch.table_ticks = Some( table .get_column(id) - .unwrap_or_else(|| debug_checked_unreachable()) + .debug_checked_unwrap() .get_ticks_slice() .into(), ); @@ -1092,9 +1088,7 @@ unsafe impl WorldQuery for ChangeTrackers { match T::Storage::STORAGE_TYPE { StorageType::Table => ChangeTrackers { component_ticks: { - let table_ticks = fetch - .table_ticks - .unwrap_or_else(|| debug_checked_unreachable()); + let table_ticks = fetch.table_ticks.debug_checked_unwrap(); table_ticks.get(table_row).read() }, marker: PhantomData, @@ -1104,9 +1098,9 @@ unsafe impl WorldQuery for ChangeTrackers { StorageType::SparseSet => ChangeTrackers { component_ticks: *fetch .sparse_set - .unwrap_or_else(|| debug_checked_unreachable()) + .debug_checked_unwrap() .get_ticks(entity) - .unwrap_or_else(|| debug_checked_unreachable()) + .debug_checked_unwrap() .get(), marker: PhantomData, last_change_tick: fetch.last_change_tick, diff --git a/crates/bevy_ecs/src/query/filter.rs b/crates/bevy_ecs/src/query/filter.rs index 984aac1178711..2cf2fc3e1c10b 100644 --- a/crates/bevy_ecs/src/query/filter.rs +++ b/crates/bevy_ecs/src/query/filter.rs @@ -2,7 +2,7 @@ use crate::{ archetype::{Archetype, ArchetypeComponentId}, component::{Component, ComponentId, ComponentStorage, ComponentTicks, StorageType}, entity::Entity, - query::{debug_checked_unreachable, Access, FilteredAccess, WorldQuery}, + query::{Access, DebugCheckedUnwrap, FilteredAccess, WorldQuery}, storage::{ComponentSparseSet, Table}, world::World, }; @@ -439,7 +439,7 @@ macro_rules! impl_tick_filter { world.storages() .sparse_sets .get(id) - .unwrap_or_else(|| debug_checked_unreachable()) + .debug_checked_unwrap() }), marker: PhantomData, last_change_tick, @@ -476,7 +476,7 @@ macro_rules! impl_tick_filter { ) { fetch.table_ticks = Some( table.get_column(component_id) - .unwrap_or_else(|| debug_checked_unreachable()) + .debug_checked_unwrap() .get_ticks_slice() .into() ); @@ -504,7 +504,7 @@ macro_rules! impl_tick_filter { StorageType::Table => { $is_detected(&*( fetch.table_ticks - .unwrap_or_else(|| debug_checked_unreachable()) + .debug_checked_unwrap() .get(table_row)) .deref(), fetch.last_change_tick, @@ -514,9 +514,9 @@ macro_rules! impl_tick_filter { StorageType::SparseSet => { let ticks = &*fetch .sparse_set - .unwrap_or_else(|| debug_checked_unreachable()) + .debug_checked_unwrap() .get_ticks(entity) - .unwrap_or_else(|| debug_checked_unreachable()) + .debug_checked_unwrap() .get(); $is_detected(ticks, fetch.last_change_tick, fetch.change_tick) } diff --git a/crates/bevy_ecs/src/query/iter.rs b/crates/bevy_ecs/src/query/iter.rs index 83c4bc270eadd..8924c54829a2d 100644 --- a/crates/bevy_ecs/src/query/iter.rs +++ b/crates/bevy_ecs/src/query/iter.rs @@ -2,7 +2,7 @@ use crate::{ archetype::{ArchetypeEntity, ArchetypeId, Archetypes}, entity::{Entities, Entity}, prelude::World, - query::{ArchetypeFilter, QueryState, WorldQuery}, + query::{ArchetypeFilter, DebugCheckedUnwrap, QueryState, WorldQuery}, storage::{TableId, Tables}, }; use std::{borrow::Borrow, iter::FusedIterator, marker::PhantomData, mem::MaybeUninit}; @@ -153,8 +153,11 @@ where continue; } - let archetype = &self.archetypes[location.archetype_id]; - let table = &self.tables[archetype.table_id()]; + let archetype = self + .archetypes + .get(location.archetype_id) + .debug_checked_unwrap(); + let table = self.tables.get(archetype.table_id()).debug_checked_unwrap(); // SAFETY: `archetype` is from the world that `fetch/filter` were created for, // `fetch_state`/`filter_state` are the states that `fetch/filter` were initialized with @@ -586,7 +589,7 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, // we are on the beginning of the query, or finished processing a table, so skip to the next if self.current_index == self.current_len { let table_id = self.table_id_iter.next()?; - let table = &tables[*table_id]; + let table = tables.get(*table_id).debug_checked_unwrap(); // SAFETY: `table` is from the world that `fetch/filter` were created for, // `fetch_state`/`filter_state` are the states that `fetch/filter` were initialized with Q::set_table(&mut self.fetch, &query_state.fetch_state, table); @@ -616,10 +619,10 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, loop { if self.current_index == self.current_len { let archetype_id = self.archetype_id_iter.next()?; - let archetype = &archetypes[*archetype_id]; + let archetype = archetypes.get(*archetype_id).debug_checked_unwrap(); // SAFETY: `archetype` and `tables` are from the world that `fetch/filter` were created for, // `fetch_state`/`filter_state` are the states that `fetch/filter` were initialized with - let table = &tables[archetype.table_id()]; + let table = tables.get(archetype.table_id()).debug_checked_unwrap(); Q::set_archetype(&mut self.fetch, &query_state.fetch_state, archetype, table); F::set_archetype( &mut self.filter, diff --git a/crates/bevy_ecs/src/query/mod.rs b/crates/bevy_ecs/src/query/mod.rs index 44147b98d55db..06b401822eb4c 100644 --- a/crates/bevy_ecs/src/query/mod.rs +++ b/crates/bevy_ecs/src/query/mod.rs @@ -10,11 +10,49 @@ pub use filter::*; pub use iter::*; pub use state::*; -#[allow(unreachable_code)] -pub(crate) unsafe fn debug_checked_unreachable() -> ! { - #[cfg(debug_assertions)] - unreachable!(); - std::hint::unreachable_unchecked(); +/// A debug checked version of [`Option::unwrap_unchecked`]. Will panic in +/// debug modes if unwrapping a `None` or `Err` value in debug mode, but is +/// equivalent to `Option::unwrap_uncheched` or `Result::unwrap_unchecked` +/// in release mode. +pub(crate) trait DebugCheckedUnwrap { + type Item; + /// # Panics + /// Panics if the value is `None` or `Err`, only in debug mode. + /// + /// # Safety + /// This must never be called on a `None` or `Err` value. This can + /// only be called on `Some` or `Ok` values. + unsafe fn debug_checked_unwrap(self) -> Self::Item; +} + +// Thes two impls are explicitly split to ensure that the unreachable! macro +// does not cause inlining to fail when compiling in release mode. +#[cfg(debug_assertions)] +impl DebugCheckedUnwrap for Option { + type Item = T; + + #[inline(always)] + unsafe fn debug_checked_unwrap(self) -> Self::Item { + if let Some(inner) = self { + inner + } else { + unreachable!() + } + } +} + +#[cfg(not(debug_assertions))] +impl DebugCheckedUnwrap for Option { + type Item = T; + + #[inline(always)] + unsafe fn debug_checked_unwrap(self) -> Self::Item { + if let Some(inner) = self { + inner + } else { + std::hint::unreachable_unchecked() + } + } } #[cfg(test)] diff --git a/crates/bevy_ecs/src/query/state.rs b/crates/bevy_ecs/src/query/state.rs index 7e4e14acde5fa..18874fd107fae 100644 --- a/crates/bevy_ecs/src/query/state.rs +++ b/crates/bevy_ecs/src/query/state.rs @@ -3,7 +3,9 @@ use crate::{ component::ComponentId, entity::Entity, prelude::FromWorld, - query::{Access, FilteredAccess, QueryCombinationIter, QueryIter, WorldQuery}, + query::{ + Access, DebugCheckedUnwrap, FilteredAccess, QueryCombinationIter, QueryIter, WorldQuery, + }, storage::TableId, world::{World, WorldId}, }; @@ -409,11 +411,18 @@ impl QueryState { { return Err(QueryEntityError::QueryDoesNotMatch(entity)); } - let archetype = &world.archetypes[location.archetype_id]; + let archetype = world + .archetypes + .get(location.archetype_id) + .debug_checked_unwrap(); let mut fetch = Q::init_fetch(world, &self.fetch_state, last_change_tick, change_tick); let mut filter = F::init_fetch(world, &self.filter_state, last_change_tick, change_tick); - let table = &world.storages().tables[archetype.table_id()]; + let table = world + .storages() + .tables + .get(archetype.table_id()) + .debug_checked_unwrap(); Q::set_archetype(&mut fetch, &self.fetch_state, archetype, table); F::set_archetype(&mut filter, &self.filter_state, archetype, table); @@ -930,7 +939,7 @@ impl QueryState { let tables = &world.storages().tables; if Q::IS_DENSE && F::IS_DENSE { for table_id in &self.matched_table_ids { - let table = &tables[*table_id]; + let table = tables.get(*table_id).debug_checked_unwrap(); Q::set_table(&mut fetch, &self.fetch_state, table); F::set_table(&mut filter, &self.filter_state, table); @@ -946,8 +955,8 @@ impl QueryState { } else { let archetypes = &world.archetypes; for archetype_id in &self.matched_archetype_ids { - let archetype = &archetypes[*archetype_id]; - let table = &tables[archetype.table_id()]; + let archetype = archetypes.get(*archetype_id).debug_checked_unwrap(); + let table = tables.get(archetype.table_id()).debug_checked_unwrap(); Q::set_archetype(&mut fetch, &self.fetch_state, archetype, table); F::set_archetype(&mut filter, &self.filter_state, archetype, table); @@ -1025,7 +1034,7 @@ impl QueryState { change_tick, ); let tables = &world.storages().tables; - let table = &tables[*table_id]; + let table = tables.get(*table_id).debug_checked_unwrap(); let entities = table.entities(); Q::set_table(&mut fetch, &self.fetch_state, table); F::set_table(&mut filter, &self.filter_state, table); @@ -1076,8 +1085,9 @@ impl QueryState { change_tick, ); let tables = &world.storages().tables; - let archetype = &world.archetypes[*archetype_id]; - let table = &tables[archetype.table_id()]; + let archetype = + world.archetypes.get(*archetype_id).debug_checked_unwrap(); + let table = tables.get(archetype.table_id()).debug_checked_unwrap(); Q::set_archetype(&mut fetch, &self.fetch_state, archetype, table); F::set_archetype(&mut filter, &self.filter_state, archetype, table); diff --git a/crates/bevy_ecs/src/storage/table.rs b/crates/bevy_ecs/src/storage/table.rs index 6e1e67f0abdee..adb684469a6d6 100644 --- a/crates/bevy_ecs/src/storage/table.rs +++ b/crates/bevy_ecs/src/storage/table.rs @@ -1,7 +1,7 @@ use crate::{ component::{ComponentId, ComponentInfo, ComponentTicks, Components}, entity::Entity, - query::debug_checked_unreachable, + query::DebugCheckedUnwrap, storage::{blob_vec::BlobVec, SparseSet}, }; use bevy_ptr::{OwningPtr, Ptr, PtrMut}; @@ -386,7 +386,7 @@ impl Table { for (component_id, column) in self.columns.iter_mut() { new_table .get_column_mut(*component_id) - .unwrap_or_else(|| debug_checked_unreachable()) + .debug_checked_unwrap() .initialize_from_unchecked(column, row, new_row); } TableMoveResult {