Support arbitrary RenderTarget texture formats (bevyengine#6380)

# Objective

Currently, Bevy only supports rendering to the current "surface texture format". This means that "render to texture" scenarios must use the exact format the primary window's surface uses, or Bevy will crash. This is even harder than it used to be now that we detect preferred surface formats at runtime instead of using hard coded BevyDefault values.

## Solution

1. Look up and store each window surface's texture format alongside other extracted window information
2. Specialize the upscaling pass on the current `RenderTarget`'s texture format, now that we can cheaply correlate render targets to their current texture format
3. Remove the old `SurfaceTextureFormat` and `AvailableTextureFormats`: these are now redundant with the information stored on each extracted window, and probably should not have been globals in the first place (as in theory each surface could have a different format). 

This means you can now use any texture format you want when rendering to a texture! For example, changing the `render_to_texture` example to use `R16Float` now doesn't crash / properly only stores the red component:
![image](https://user-images.githubusercontent.com/2694663/198140125-c606dd0e-6fdf-4544-b93d-dbbd10dbadd2.png)

crates/bevy_core_pipeline/src/upscaling/mod.rs

@@ -5,8 +5,8 @@ pub use node::UpscalingNode;
 use bevy_app::prelude::*;
 use bevy_asset::{load_internal_asset, HandleUntyped};
 use bevy_ecs::prelude::*;
-use bevy_render::renderer::{RenderDevice, SurfaceTextureFormat};
-use bevy_render::view::ExtractedView;
+use bevy_render::renderer::RenderDevice;
+use bevy_render::view::ViewTarget;
 use bevy_render::{render_resource::*, RenderApp, RenderStage};
 
 use bevy_reflect::TypeUuid;
@@ -39,13 +39,11 @@ impl Plugin for UpscalingPlugin {
 #[derive(Resource)]
 pub struct UpscalingPipeline {
     ldr_texture_bind_group: BindGroupLayout,
-    surface_texture_format: TextureFormat,
 }
 
 impl FromWorld for UpscalingPipeline {
     fn from_world(render_world: &mut World) -> Self {
         let render_device = render_world.resource::<RenderDevice>();
-        let surface_texture_format = render_world.resource::<SurfaceTextureFormat>().0;
 
         let ldr_texture_bind_group =
             render_device.create_bind_group_layout(&BindGroupLayoutDescriptor {
@@ -72,50 +70,26 @@ impl FromWorld for UpscalingPipeline {
 
         UpscalingPipeline {
             ldr_texture_bind_group,
-            surface_texture_format,
         }
     }
 }
 
-#[repr(u8)]
+#[derive(PartialEq, Eq, Hash, Clone, Copy)]
 pub enum UpscalingMode {
-    Filtering = 0,
-    Nearest = 1,
+    Filtering,
+    Nearest,
 }
 
-bitflags::bitflags! {
-    #[repr(transparent)]
-    pub struct UpscalingPipelineKey: u32 {
-        const NONE                         = 0;
-        const UPSCALING_MODE_RESERVED_BITS = UpscalingPipelineKey::UPSCALING_MODE_MASK_BITS << UpscalingPipelineKey::UPSCALING_MODE_SHIFT_BITS;
-    }
-}
-
-impl UpscalingPipelineKey {
-    const UPSCALING_MODE_MASK_BITS: u32 = 0b1111; // enough for 16 different modes
-    const UPSCALING_MODE_SHIFT_BITS: u32 = 32 - 4;
-
-    pub fn from_upscaling_mode(upscaling_mode: UpscalingMode) -> Self {
-        let upscaling_mode_bits = ((upscaling_mode as u32) & Self::UPSCALING_MODE_MASK_BITS)
-            << Self::UPSCALING_MODE_SHIFT_BITS;
-        UpscalingPipelineKey::from_bits(upscaling_mode_bits).unwrap()
-    }
-
-    pub fn upscaling_mode(&self) -> UpscalingMode {
-        let upscaling_mode_bits =
-            (self.bits >> Self::UPSCALING_MODE_SHIFT_BITS) & Self::UPSCALING_MODE_MASK_BITS;
-        match upscaling_mode_bits {
-            0 => UpscalingMode::Filtering,
-            1 => UpscalingMode::Nearest,
-            other => panic!("invalid upscaling mode bits in UpscalingPipelineKey: {other}"),
-        }
-    }
+#[derive(PartialEq, Eq, Hash, Clone, Copy)]
+pub struct UpscalingPipelineKey {
+    upscaling_mode: UpscalingMode,
+    texture_format: TextureFormat,
 }
 
 impl SpecializedRenderPipeline for UpscalingPipeline {
     type Key = UpscalingPipelineKey;
 
-    fn specialize(&self, _: Self::Key) -> RenderPipelineDescriptor {
+    fn specialize(&self, key: Self::Key) -> RenderPipelineDescriptor {
         RenderPipelineDescriptor {
             label: Some("upscaling pipeline".into()),
             layout: Some(vec![self.ldr_texture_bind_group.clone()]),
@@ -125,7 +99,7 @@ impl SpecializedRenderPipeline for UpscalingPipeline {
                 shader_defs: vec![],
                 entry_point: "fs_main".into(),
                 targets: vec![Some(ColorTargetState {
-                    format: self.surface_texture_format,
+                    format: key.texture_format,
                     blend: None,
                     write_mask: ColorWrites::ALL,
                 })],
@@ -147,10 +121,13 @@ fn queue_upscaling_bind_groups(
     mut pipeline_cache: ResMut<PipelineCache>,
     mut pipelines: ResMut<SpecializedRenderPipelines<UpscalingPipeline>>,
     upscaling_pipeline: Res<UpscalingPipeline>,
-    view_targets: Query<Entity, With<ExtractedView>>,
+    view_targets: Query<(Entity, &ViewTarget)>,
 ) {
-    for entity in view_targets.iter() {
-        let key = UpscalingPipelineKey::from_upscaling_mode(UpscalingMode::Filtering);
+    for (entity, view_target) in view_targets.iter() {
+        let key = UpscalingPipelineKey {
+            upscaling_mode: UpscalingMode::Filtering,
+            texture_format: view_target.out_texture_format,
+        };
         let pipeline = pipelines.specialize(&mut pipeline_cache, &upscaling_pipeline, key);
 
         commands.entity(entity).insert(UpscalingTarget { pipeline });

crates/bevy_render/src/camera/camera.rs

@@ -24,7 +24,7 @@ use bevy_transform::components::GlobalTransform;
 use bevy_utils::HashSet;
 use bevy_window::{WindowCreated, WindowId, WindowResized, Windows};
 use std::{borrow::Cow, ops::Range};
-use wgpu::Extent3d;
+use wgpu::{Extent3d, TextureFormat};
 
 /// Render viewport configuration for the [`Camera`] component.
 ///
@@ -326,6 +326,22 @@ impl RenderTarget {
         }
     }
 
+    /// Retrieves the [`TextureFormat`] of this render target, if it exists.
+    pub fn get_texture_format<'a>(
+        &self,
+        windows: &'a ExtractedWindows,
+        images: &'a RenderAssets<Image>,
+    ) -> Option<TextureFormat> {
+        match self {
+            RenderTarget::Window(window_id) => windows
+                .get(window_id)
+                .and_then(|window| window.swap_chain_texture_format),
+            RenderTarget::Image(image_handle) => {
+                images.get(image_handle).map(|image| image.texture_format)
+            }
+        }
+    }
+
     pub fn get_render_target_info(
         &self,
         windows: &Windows,

crates/bevy_render/src/lib.rs

@@ -39,7 +39,6 @@ pub mod prelude {
 use globals::GlobalsPlugin;
 pub use once_cell;
 use prelude::ComputedVisibility;
-use wgpu::TextureFormat;
 
 use crate::{
     camera::CameraPlugin,
@@ -48,8 +47,7 @@ use crate::{
     primitives::{CubemapFrusta, Frustum},
     render_graph::RenderGraph,
     render_resource::{PipelineCache, Shader, ShaderLoader},
-    renderer::{render_system, RenderInstance, SurfaceTextureFormat},
-    texture::BevyDefault,
+    renderer::{render_system, RenderInstance},
     view::{ViewPlugin, WindowRenderPlugin},
 };
 use bevy_app::{App, AppLabel, Plugin};
@@ -160,26 +158,15 @@ impl Plugin for RenderPlugin {
                 compatible_surface: surface.as_ref(),
                 ..Default::default()
             };
-            let (device, queue, adapter_info, render_adapter, available_texture_formats) =
-                futures_lite::future::block_on(renderer::initialize_renderer(
-                    &instance,
-                    &options,
-                    &request_adapter_options,
-                ));
-            let texture_format = SurfaceTextureFormat(
-                available_texture_formats
-                    .get(0)
-                    .cloned()
-                    .unwrap_or_else(TextureFormat::bevy_default),
+            let (device, queue, adapter_info, render_adapter) = futures_lite::future::block_on(
+                renderer::initialize_renderer(&instance, &options, &request_adapter_options),
             );
             debug!("Configured wgpu adapter Limits: {:#?}", device.limits());
             debug!("Configured wgpu adapter Features: {:#?}", device.features());
             app.insert_resource(device.clone())
                 .insert_resource(queue.clone())
                 .insert_resource(adapter_info.clone())
                 .insert_resource(render_adapter.clone())
-                .insert_resource(available_texture_formats.clone())
-                .insert_resource(texture_format.clone())
                 .init_resource::<ScratchMainWorld>()
                 .register_type::<Frustum>()
                 .register_type::<CubemapFrusta>();
@@ -222,8 +209,6 @@ impl Plugin for RenderPlugin {
                 .insert_resource(device)
                 .insert_resource(queue)
                 .insert_resource(render_adapter)
-                .insert_resource(available_texture_formats)
-                .insert_resource(texture_format)
                 .insert_resource(adapter_info)
                 .insert_resource(pipeline_cache)
                 .insert_resource(asset_server);

crates/bevy_render/src/renderer/mod.rs

@@ -102,16 +102,6 @@ pub struct RenderInstance(pub Instance);
 #[derive(Resource, Clone, Deref, DerefMut)]
 pub struct RenderAdapterInfo(pub AdapterInfo);
 
-/// The [`TextureFormat`](wgpu::TextureFormat) used for rendering to window surfaces.
-/// Initially it's the first element in `AvailableTextureFormats`, or Bevy default format.
-#[derive(Resource, Clone, Deref, DerefMut)]
-pub struct SurfaceTextureFormat(pub wgpu::TextureFormat);
-
-/// The available [`TextureFormat`](wgpu::TextureFormat)s on the [`RenderAdapter`].
-/// Will be inserted as a `Resource` after the renderer is initialized.
-#[derive(Resource, Clone, Deref, DerefMut)]
-pub struct AvailableTextureFormats(pub Arc<Vec<wgpu::TextureFormat>>);
-
 const GPU_NOT_FOUND_ERROR_MESSAGE: &str = if cfg!(target_os = "linux") {
     "Unable to find a GPU! Make sure you have installed required drivers! For extra information, see: https://github.com/bevyengine/bevy/blob/latest/docs/linux_dependencies.md"
 } else {
@@ -124,13 +114,7 @@ pub async fn initialize_renderer(
     instance: &Instance,
     options: &WgpuSettings,
     request_adapter_options: &RequestAdapterOptions<'_>,
-) -> (
-    RenderDevice,
-    RenderQueue,
-    RenderAdapterInfo,
-    RenderAdapter,
-    AvailableTextureFormats,
-) {
+) -> (RenderDevice, RenderQueue, RenderAdapterInfo, RenderAdapter) {
     let adapter = instance
         .request_adapter(request_adapter_options)
         .await
@@ -281,17 +265,11 @@ pub async fn initialize_renderer(
     let device = Arc::new(device);
     let queue = Arc::new(queue);
     let adapter = Arc::new(adapter);
-    let mut available_texture_formats = Vec::new();
-    if let Some(s) = request_adapter_options.compatible_surface {
-        available_texture_formats = s.get_supported_formats(&adapter);
-    };
-    let available_texture_formats = Arc::new(available_texture_formats);
     (
         RenderDevice::from(device),
         RenderQueue(queue),
         RenderAdapterInfo(adapter_info),
         RenderAdapter(adapter),
-        AvailableTextureFormats(available_texture_formats),
     )
 }
 

crates/bevy_render/src/view/mod.rs

@@ -143,6 +143,7 @@ impl ViewMainTexture {
 pub struct ViewTarget {
     pub main_texture: ViewMainTexture,
     pub out_texture: TextureView,
+    pub out_texture_format: TextureFormat,
 }
 
 impl ViewTarget {
@@ -242,7 +243,10 @@ fn prepare_view_targets(
     let mut textures = HashMap::default();
     for (entity, camera, view) in cameras.iter() {
         if let Some(target_size) = camera.physical_target_size {
-            if let Some(texture_view) = camera.target.get_texture_view(&windows, &images) {
+            if let (Some(texture_view), Some(texture_format)) = (
+                camera.target.get_texture_view(&windows, &images),
+                camera.target.get_texture_format(&windows, &images),
+            ) {
                 let size = Extent3d {
                     width: target_size.x,
                     height: target_size.y,
@@ -319,6 +323,7 @@ fn prepare_view_targets(
                 commands.entity(entity).insert(ViewTarget {
                     main_texture: main_texture.clone(),
                     out_texture: texture_view.clone(),
+                    out_texture_format: texture_format,
                 });
             }
         }

crates/bevy_render/src/view/window.rs

@@ -10,6 +10,7 @@ use bevy_window::{
     CompositeAlphaMode, PresentMode, RawHandleWrapper, WindowClosed, WindowId, Windows,
 };
 use std::ops::{Deref, DerefMut};
+use wgpu::TextureFormat;
 
 /// Token to ensure a system runs on the main thread.
 #[derive(Resource, Default)]
@@ -45,6 +46,7 @@ pub struct ExtractedWindow {
     pub physical_height: u32,
     pub present_mode: PresentMode,
     pub swap_chain_texture: Option<TextureView>,
+    pub swap_chain_texture_format: Option<TextureFormat>,
     pub size_changed: bool,
     pub present_mode_changed: bool,
     pub alpha_mode: CompositeAlphaMode,
@@ -91,6 +93,7 @@ fn extract_windows(
                     physical_height: new_height,
                     present_mode: window.present_mode(),
                     swap_chain_texture: None,
+                    swap_chain_texture_format: None,
                     size_changed: false,
                     present_mode_changed: false,
                     alpha_mode: window.alpha_mode(),
@@ -127,9 +130,14 @@ fn extract_windows(
     }
 }
 
+struct SurfaceData {
+    surface: wgpu::Surface,
+    format: TextureFormat,
+}
+
 #[derive(Resource, Default)]
 pub struct WindowSurfaces {
-    surfaces: HashMap<WindowId, wgpu::Surface>,
+    surfaces: HashMap<WindowId, SurfaceData>,
     /// List of windows that we have already called the initial `configure_surface` for
     configured_windows: HashSet<WindowId>,
 }
@@ -172,25 +180,27 @@ pub fn prepare_windows(
         .filter(|x| x.raw_handle.is_some())
     {
         let window_surfaces = window_surfaces.deref_mut();
-        let surface = window_surfaces
+        let surface_data = window_surfaces
             .surfaces
             .entry(window.id)
             .or_insert_with(|| unsafe {
                 // NOTE: On some OSes this MUST be called from the main thread.
-                render_instance.create_surface(&window.raw_handle.as_ref().unwrap().get_handle())
+                let surface = render_instance
+                    .create_surface(&window.raw_handle.as_ref().unwrap().get_handle());
+                let format = *surface
+                    .get_supported_formats(&render_adapter)
+                    .get(0)
+                    .unwrap_or_else(|| {
+                        panic!(
+                            "No supported formats found for surface {:?} on adapter {:?}",
+                            surface, render_adapter
+                        )
+                    });
+                SurfaceData { surface, format }
             });
 
-        // Creates a closure to avoid calling this logic unnecessarily
-        let create_swap_chain_descriptor = || wgpu::SurfaceConfiguration {
-            format: *surface
-                .get_supported_formats(&render_adapter)
-                .get(0)
-                .unwrap_or_else(|| {
-                    panic!(
-                        "No supported formats found for surface {:?} on adapter {:?}",
-                        surface, render_adapter
-                    )
-                }),
+        let surface_configuration = wgpu::SurfaceConfiguration {
+            format: surface_data.format,
             width: window.physical_width,
             height: window.physical_height,
             usage: wgpu::TextureUsages::RENDER_ATTACHMENT,
@@ -216,16 +226,18 @@ pub fn prepare_windows(
             || window.size_changed
             || window.present_mode_changed
         {
-            render_device.configure_surface(surface, &create_swap_chain_descriptor());
-            surface
+            render_device.configure_surface(&surface_data.surface, &surface_configuration);
+            surface_data
+                .surface
                 .get_current_texture()
                 .expect("Error configuring surface")
         } else {
-            match surface.get_current_texture() {
+            match surface_data.surface.get_current_texture() {
                 Ok(swap_chain_frame) => swap_chain_frame,
                 Err(wgpu::SurfaceError::Outdated) => {
-                    render_device.configure_surface(surface, &create_swap_chain_descriptor());
-                    surface
+                    render_device.configure_surface(&surface_data.surface, &surface_configuration);
+                    surface_data
+                        .surface
                         .get_current_texture()
                         .expect("Error reconfiguring surface")
                 }
@@ -234,5 +246,6 @@ pub fn prepare_windows(
         };
 
         window.swap_chain_texture = Some(TextureView::from(frame));
+        window.swap_chain_texture_format = Some(surface_data.format);
     }
 }