Add Skybox Motion Vectors (#13617)

# Objective

- Add motion vector support to the skybox
- This fixes the last remaining "gap" to complete the motion blur
feature

## Solution

- Add a pipeline for the skybox to write motion vectors to the prepass

## Testing

- Used examples to test motion vectors using motion blur


https://github.com/bevyengine/bevy/assets/2632925/74c0778a-7e77-4e68-8111-05791e4bfdd2

---------

Co-authored-by: Patrick Walton <pcwalton@mimiga.net>

crates/bevy_core_pipeline/src/prepass/mod.rs

@@ -31,6 +31,7 @@ use std::ops::Range;
 
 use bevy_asset::AssetId;
 use bevy_ecs::prelude::*;
+use bevy_math::Mat4;
 use bevy_reflect::Reflect;
 use bevy_render::{
     mesh::Mesh,
@@ -38,10 +39,15 @@ use bevy_render::{
         BinnedPhaseItem, CachedRenderPipelinePhaseItem, DrawFunctionId, PhaseItem,
         PhaseItemExtraIndex,
     },
-    render_resource::{BindGroupId, CachedRenderPipelineId, Extent3d, TextureFormat, TextureView},
+    render_resource::{
+        BindGroupId, CachedRenderPipelineId, ColorTargetState, ColorWrites, DynamicUniformBuffer,
+        Extent3d, ShaderType, TextureFormat, TextureView,
+    },
     texture::ColorAttachment,
 };
 
+use crate::deferred::{DEFERRED_LIGHTING_PASS_ID_FORMAT, DEFERRED_PREPASS_FORMAT};
+
 pub const NORMAL_PREPASS_FORMAT: TextureFormat = TextureFormat::Rgb10a2Unorm;
 pub const MOTION_VECTOR_PREPASS_FORMAT: TextureFormat = TextureFormat::Rg16Float;
 
@@ -63,6 +69,22 @@ pub struct MotionVectorPrepass;
 #[derive(Component, Default, Reflect)]
 pub struct DeferredPrepass;
 
+#[derive(Component, ShaderType, Clone)]
+pub struct PreviousViewData {
+    pub inverse_view: Mat4,
+    pub view_proj: Mat4,
+}
+
+#[derive(Resource, Default)]
+pub struct PreviousViewUniforms {
+    pub uniforms: DynamicUniformBuffer<PreviousViewData>,
+}
+
+#[derive(Component)]
+pub struct PreviousViewUniformOffset {
+    pub offset: u32,
+}
+
 /// Textures that are written to by the prepass.
 ///
 /// This component will only be present if any of the relevant prepass components are also present.
@@ -270,3 +292,32 @@ impl CachedRenderPipelinePhaseItem for AlphaMask3dPrepass {
         self.key.pipeline
     }
 }
+
+pub fn prepass_target_descriptors(
+    normal_prepass: bool,
+    motion_vector_prepass: bool,
+    deferred_prepass: bool,
+) -> Vec<Option<ColorTargetState>> {
+    vec![
+        normal_prepass.then_some(ColorTargetState {
+            format: NORMAL_PREPASS_FORMAT,
+            blend: None,
+            write_mask: ColorWrites::ALL,
+        }),
+        motion_vector_prepass.then_some(ColorTargetState {
+            format: MOTION_VECTOR_PREPASS_FORMAT,
+            blend: None,
+            write_mask: ColorWrites::ALL,
+        }),
+        deferred_prepass.then_some(ColorTargetState {
+            format: DEFERRED_PREPASS_FORMAT,
+            blend: None,
+            write_mask: ColorWrites::ALL,
+        }),
+        deferred_prepass.then_some(ColorTargetState {
+            format: DEFERRED_LIGHTING_PASS_ID_FORMAT,
+            blend: None,
+            write_mask: ColorWrites::ALL,
+        }),
+    ]
+}

crates/bevy_core_pipeline/src/prepass/node.rs

@@ -5,14 +5,19 @@ use bevy_render::{
     diagnostic::RecordDiagnostics,
     render_graph::{NodeRunError, RenderGraphContext, ViewNode},
     render_phase::{TrackedRenderPass, ViewBinnedRenderPhases},
-    render_resource::{CommandEncoderDescriptor, RenderPassDescriptor, StoreOp},
+    render_resource::{CommandEncoderDescriptor, PipelineCache, RenderPassDescriptor, StoreOp},
     renderer::RenderContext,
-    view::ViewDepthTexture,
+    view::{ViewDepthTexture, ViewUniformOffset},
 };
 #[cfg(feature = "trace")]
 use bevy_utils::tracing::info_span;
 
-use super::{AlphaMask3dPrepass, DeferredPrepass, Opaque3dPrepass, ViewPrepassTextures};
+use crate::skybox::prepass::{RenderSkyboxPrepassPipeline, SkyboxPrepassBindGroup};
+
+use super::{
+    AlphaMask3dPrepass, DeferredPrepass, Opaque3dPrepass, PreviousViewUniformOffset,
+    ViewPrepassTextures,
+};
 
 /// Render node used by the prepass.
 ///
@@ -26,17 +31,28 @@ impl ViewNode for PrepassNode {
         &'static ExtractedCamera,
         &'static ViewDepthTexture,
         &'static ViewPrepassTextures,
+        &'static ViewUniformOffset,
         Option<&'static DeferredPrepass>,
+        Option<&'static RenderSkyboxPrepassPipeline>,
+        Option<&'static SkyboxPrepassBindGroup>,
+        Option<&'static PreviousViewUniformOffset>,
     );
 
     fn run<'w>(
         &self,
         graph: &mut RenderGraphContext,
         render_context: &mut RenderContext<'w>,
-        (view, camera, view_depth_texture, view_prepass_textures, deferred_prepass): QueryItem<
+        (
-            'w,
+            view,
-            Self::ViewQuery,
+            camera,
-        >,
+            view_depth_texture,
+            view_prepass_textures,
+            view_uniform_offset,
+            deferred_prepass,
+            skybox_prepass_pipeline,
+            skybox_prepass_bind_group,
+            view_prev_uniform_offset,
+        ): QueryItem<'w, Self::ViewQuery>,
         world: &'w World,
     ) -> Result<(), NodeRunError> {
         let (Some(opaque_prepass_phases), Some(alpha_mask_prepass_phases)) = (
@@ -119,6 +135,30 @@ impl ViewNode for PrepassNode {
                 alpha_mask_prepass_phase.render(&mut render_pass, world, view_entity);
             }
 
+            // Skybox draw using a fullscreen triangle
+            if let (
+                Some(skybox_prepass_pipeline),
+                Some(skybox_prepass_bind_group),
+                Some(view_prev_uniform_offset),
+            ) = (
+                skybox_prepass_pipeline,
+                skybox_prepass_bind_group,
+                view_prev_uniform_offset,
+            ) {
+                let pipeline_cache = world.resource::<PipelineCache>();
+                if let Some(pipeline) =
+                    pipeline_cache.get_render_pipeline(skybox_prepass_pipeline.0)
+                {
+                    render_pass.set_render_pipeline(pipeline);
+                    render_pass.set_bind_group(
+                        0,
+                        &skybox_prepass_bind_group.0,
+                        &[view_uniform_offset.offset, view_prev_uniform_offset.offset],
+                    );
+                    render_pass.draw(0..3, 0..1);
+                }
+            }
+
             pass_span.end(&mut render_pass);
             drop(render_pass);
 

crates/bevy_core_pipeline/src/skybox/mod.rs

@@ -22,16 +22,25 @@ use bevy_render::{
     view::{ExtractedView, Msaa, ViewTarget, ViewUniform, ViewUniforms},
     Render, RenderApp, RenderSet,
 };
+use prepass::{SkyboxPrepassPipeline, SKYBOX_PREPASS_SHADER_HANDLE};
 
 use crate::core_3d::CORE_3D_DEPTH_FORMAT;
 
 const SKYBOX_SHADER_HANDLE: Handle<Shader> = Handle::weak_from_u128(55594763423201);
 
+pub mod prepass;
+
 pub struct SkyboxPlugin;
 
 impl Plugin for SkyboxPlugin {
     fn build(&self, app: &mut App) {
         load_internal_asset!(app, SKYBOX_SHADER_HANDLE, "skybox.wgsl", Shader::from_wgsl);
+        load_internal_asset!(
+            app,
+            SKYBOX_PREPASS_SHADER_HANDLE,
+            "skybox_prepass.wgsl",
+            Shader::from_wgsl
+        );
 
         app.add_plugins((
             ExtractComponentPlugin::<Skybox>::default(),
@@ -43,11 +52,15 @@ impl Plugin for SkyboxPlugin {
         };
         render_app
             .init_resource::<SpecializedRenderPipelines<SkyboxPipeline>>()
+            .init_resource::<SpecializedRenderPipelines<SkyboxPrepassPipeline>>()
             .add_systems(
                 Render,
                 (
                     prepare_skybox_pipelines.in_set(RenderSet::Prepare),
+                    prepass::prepare_skybox_prepass_pipelines.in_set(RenderSet::Prepare),
                     prepare_skybox_bind_groups.in_set(RenderSet::PrepareBindGroups),
+                    prepass::prepare_skybox_prepass_bind_groups
+                        .in_set(RenderSet::PrepareBindGroups),
                 ),
             );
     }
@@ -57,7 +70,9 @@ impl Plugin for SkyboxPlugin {
             return;
         };
         let render_device = render_app.world().resource::<RenderDevice>().clone();
-        render_app.insert_resource(SkyboxPipeline::new(&render_device));
+        render_app
+            .insert_resource(SkyboxPipeline::new(&render_device))
+            .init_resource::<SkyboxPrepassPipeline>();
     }
 }
 

crates/bevy_core_pipeline/src/skybox/prepass.rs

@@ -0,0 +1,165 @@
+#![warn(missing_docs)]
+
+//! Adds motion vector support to skyboxes. See [`SkyboxPrepassPipeline`] for details.
+
+use bevy_asset::Handle;
+use bevy_ecs::{
+    component::Component,
+    entity::Entity,
+    query::{Has, With},
+    system::{Commands, Query, Res, ResMut, Resource},
+    world::{FromWorld, World},
+};
+use bevy_render::{
+    render_resource::{
+        binding_types::uniform_buffer, BindGroup, BindGroupEntries, BindGroupLayout,
+        BindGroupLayoutEntries, CachedRenderPipelineId, CompareFunction, DepthStencilState,
+        FragmentState, MultisampleState, PipelineCache, RenderPipelineDescriptor, Shader,
+        ShaderStages, SpecializedRenderPipeline, SpecializedRenderPipelines,
+    },
+    renderer::RenderDevice,
+    view::{Msaa, ViewUniform, ViewUniforms},
+};
+use bevy_utils::prelude::default;
+
+use crate::{
+    core_3d::CORE_3D_DEPTH_FORMAT,
+    prepass::{
+        prepass_target_descriptors, MotionVectorPrepass, NormalPrepass, PreviousViewData,
+        PreviousViewUniforms,
+    },
+    Skybox,
+};
+
+pub const SKYBOX_PREPASS_SHADER_HANDLE: Handle<Shader> = Handle::weak_from_u128(376510055324461154);
+
+/// This pipeline writes motion vectors to the prepass for all [`Skybox`]es.
+///
+/// This allows features like motion blur and TAA to work correctly on the skybox. Without this, for
+/// example, motion blur would not be applied to the skybox when the camera is rotated and motion
+/// blur is enabled.
+#[derive(Resource)]
+pub struct SkyboxPrepassPipeline {
+    bind_group_layout: BindGroupLayout,
+}
+
+/// Used to specialize the [`SkyboxPrepassPipeline`].
+#[derive(PartialEq, Eq, Hash, Clone, Copy)]
+pub struct SkyboxPrepassPipelineKey {
+    samples: u32,
+    normal_prepass: bool,
+}
+
+/// Stores the ID for a camera's specialized pipeline, so it can be retrieved from the
+/// [`PipelineCache`].
+#[derive(Component)]
+pub struct RenderSkyboxPrepassPipeline(pub CachedRenderPipelineId);
+
+/// Stores the [`SkyboxPrepassPipeline`] bind group for a camera. This is later used by the prepass
+/// render graph node to add this binding to the prepass's render pass.
+#[derive(Component)]
+pub struct SkyboxPrepassBindGroup(pub BindGroup);
+
+impl FromWorld for SkyboxPrepassPipeline {
+    fn from_world(world: &mut World) -> Self {
+        let render_device = world.resource::<RenderDevice>();
+
+        Self {
+            bind_group_layout: render_device.create_bind_group_layout(
+                "skybox_prepass_bind_group_layout",
+                &BindGroupLayoutEntries::sequential(
+                    ShaderStages::FRAGMENT,
+                    (
+                        uniform_buffer::<ViewUniform>(true),
+                        uniform_buffer::<PreviousViewData>(true),
+                    ),
+                ),
+            ),
+        }
+    }
+}
+
+impl SpecializedRenderPipeline for SkyboxPrepassPipeline {
+    type Key = SkyboxPrepassPipelineKey;
+
+    fn specialize(&self, key: Self::Key) -> RenderPipelineDescriptor {
+        RenderPipelineDescriptor {
+            label: Some("skybox_prepass_pipeline".into()),
+            layout: vec![self.bind_group_layout.clone()],
+            push_constant_ranges: vec![],
+            vertex: crate::fullscreen_vertex_shader::fullscreen_shader_vertex_state(),
+            primitive: default(),
+            depth_stencil: Some(DepthStencilState {
+                format: CORE_3D_DEPTH_FORMAT,
+                depth_write_enabled: false,
+                depth_compare: CompareFunction::GreaterEqual,
+                stencil: default(),
+                bias: default(),
+            }),
+            multisample: MultisampleState {
+                count: key.samples,
+                mask: !0,
+                alpha_to_coverage_enabled: false,
+            },
+            fragment: Some(FragmentState {
+                shader: SKYBOX_PREPASS_SHADER_HANDLE,
+                shader_defs: vec![],
+                entry_point: "fragment".into(),
+                targets: prepass_target_descriptors(key.normal_prepass, true, false),
+            }),
+        }
+    }
+}
+
+/// Specialize and cache the [`SkyboxPrepassPipeline`] for each camera with a [`Skybox`].
+pub fn prepare_skybox_prepass_pipelines(
+    mut commands: Commands,
+    pipeline_cache: Res<PipelineCache>,
+    mut pipelines: ResMut<SpecializedRenderPipelines<SkyboxPrepassPipeline>>,
+    msaa: Res<Msaa>,
+    pipeline: Res<SkyboxPrepassPipeline>,
+    views: Query<(Entity, Has<NormalPrepass>), (With<Skybox>, With<MotionVectorPrepass>)>,
+) {
+    for (entity, normal_prepass) in &views {
+        let pipeline_key = SkyboxPrepassPipelineKey {
+            samples: msaa.samples(),
+            normal_prepass,
+        };
+
+        let render_skybox_prepass_pipeline =
+            pipelines.specialize(&pipeline_cache, &pipeline, pipeline_key);
+        commands
+            .entity(entity)
+            .insert(RenderSkyboxPrepassPipeline(render_skybox_prepass_pipeline));
+    }
+}
+
+/// Creates the required bind groups for the [`SkyboxPrepassPipeline`]. This binds the view uniforms
+/// from the CPU for access in the prepass shader on the GPU, allowing us to compute camera motion
+/// between frames. This is then stored in the [`SkyboxPrepassBindGroup`] component on the camera.
+pub fn prepare_skybox_prepass_bind_groups(
+    mut commands: Commands,
+    pipeline: Res<SkyboxPrepassPipeline>,
+    view_uniforms: Res<ViewUniforms>,
+    prev_view_uniforms: Res<PreviousViewUniforms>,
+    render_device: Res<RenderDevice>,
+    views: Query<Entity, (With<Skybox>, With<MotionVectorPrepass>)>,
+) {
+    for entity in &views {
+        let (Some(view_uniforms), Some(prev_view_uniforms)) = (
+            view_uniforms.uniforms.binding(),
+            prev_view_uniforms.uniforms.binding(),
+        ) else {
+            continue;
+        };
+        let bind_group = render_device.create_bind_group(
+            "skybox_prepass_bind_group",
+            &pipeline.bind_group_layout,
+            &BindGroupEntries::sequential((view_uniforms, prev_view_uniforms)),
+        );
+
+        commands
+            .entity(entity)
+            .insert(SkyboxPrepassBindGroup(bind_group));
+    }
+}

crates/bevy_core_pipeline/src/skybox/skybox_prepass.wgsl

@@ -0,0 +1,21 @@
+#import bevy_render::view::View
+#import bevy_core_pipeline::fullscreen_vertex_shader::FullscreenVertexOutput
+#import bevy_pbr::view_transformations::uv_to_ndc
+
+struct PreviousViewUniforms {
+    inverse_view: mat4x4<f32>,
+    view_proj: mat4x4<f32>,
+}
+
+@group(0) @binding(0) var<uniform> view: View;
+@group(0) @binding(1) var<uniform> previous_view: PreviousViewUniforms;
+
+@fragment
+fn fragment(in: FullscreenVertexOutput) -> @location(1) vec4<f32> {
+    let clip_pos = uv_to_ndc(in.uv); // Convert from uv to clip space
+    let world_pos = view.inverse_view_proj * vec4(clip_pos, 0.0, 1.0);
+    let prev_clip_pos = (previous_view.view_proj * world_pos).xy;
+    let velocity = (clip_pos - prev_clip_pos) * vec2(0.5, -0.5); // Copied from mesh motion vectors
+
+    return vec4(velocity.x, velocity.y, 0.0, 1.0);
+}

crates/bevy_pbr/src/meshlet/gpu_scene.rs

@@ -4,11 +4,13 @@ use super::{
 };
 use crate::{
     Material, MeshFlags, MeshTransforms, MeshUniform, NotShadowCaster, NotShadowReceiver,
-    PreviousGlobalTransform, PreviousViewData, PreviousViewUniforms, RenderMaterialInstances,
+    PreviousGlobalTransform, RenderMaterialInstances, ShadowView,
-    ShadowView,
 };
 use bevy_asset::{AssetEvent, AssetId, AssetServer, Assets, Handle, UntypedAssetId};
-use bevy_core_pipeline::core_3d::Camera3d;
+use bevy_core_pipeline::{
+    core_3d::Camera3d,
+    prepass::{PreviousViewData, PreviousViewUniforms},
+};
 use bevy_ecs::{
     component::Component,
     entity::{Entity, EntityHashMap},

crates/bevy_pbr/src/meshlet/material_draw_nodes.rs

@@ -7,10 +7,10 @@ use super::{
     MeshletGpuScene,
 };
 use crate::{
-    MeshViewBindGroup, PrepassViewBindGroup, PreviousViewUniformOffset, ViewFogUniformOffset,
+    MeshViewBindGroup, PrepassViewBindGroup, ViewFogUniformOffset, ViewLightProbesUniformOffset,
-    ViewLightProbesUniformOffset, ViewLightsUniformOffset, ViewScreenSpaceReflectionsUniformOffset,
+    ViewLightsUniformOffset, ViewScreenSpaceReflectionsUniformOffset,
 };
-use bevy_core_pipeline::prepass::ViewPrepassTextures;
+use bevy_core_pipeline::prepass::{PreviousViewUniformOffset, ViewPrepassTextures};
 use bevy_ecs::{query::QueryItem, world::World};
 use bevy_render::{
     camera::ExtractedCamera,

crates/bevy_pbr/src/meshlet/visibility_buffer_raster_node.rs

@@ -2,8 +2,9 @@ use super::{
     gpu_scene::{MeshletViewBindGroups, MeshletViewResources},
     pipelines::MeshletPipelines,
 };
-use crate::{LightEntity, PreviousViewUniformOffset, ShadowView, ViewLightEntities};
+use crate::{LightEntity, ShadowView, ViewLightEntities};
 use bevy_color::LinearRgba;
+use bevy_core_pipeline::prepass::PreviousViewUniformOffset;
 use bevy_ecs::{
     query::QueryState,
     world::{FromWorld, World},

crates/bevy_pbr/src/prepass/mod.rs

@@ -15,7 +15,7 @@ use bevy_ecs::{
         SystemParamItem,
     },
 };
-use bevy_math::{Affine3A, Mat4};
+use bevy_math::Affine3A;
 use bevy_render::{
     globals::{GlobalsBuffer, GlobalsUniform},
     prelude::{Camera, Mesh},
@@ -194,12 +194,6 @@ where
 #[derive(Resource)]
 struct AnyPrepassPluginLoaded;
 
-#[derive(Component, ShaderType, Clone)]
-pub struct PreviousViewData {
-    pub inverse_view: Mat4,
-    pub view_proj: Mat4,
-}
-
 #[cfg(not(feature = "meshlet"))]
 type PreviousViewFilter = (With<Camera3d>, With<MotionVectorPrepass>);
 #[cfg(feature = "meshlet")]
@@ -472,39 +466,12 @@ where
         let vertex_buffer_layout = layout.0.get_layout(&vertex_attributes)?;
 
         // Setup prepass fragment targets - normals in slot 0 (or None if not needed), motion vectors in slot 1
-        let mut targets = vec![
+        let mut targets = prepass_target_descriptors(
+            key.mesh_key.contains(MeshPipelineKey::NORMAL_PREPASS),
             key.mesh_key
-                .contains(MeshPipelineKey::NORMAL_PREPASS)
+                .contains(MeshPipelineKey::MOTION_VECTOR_PREPASS),
-                .then_some(ColorTargetState {
+            key.mesh_key.contains(MeshPipelineKey::DEFERRED_PREPASS),
-                    format: NORMAL_PREPASS_FORMAT,
+        );
-                    // BlendState::REPLACE is not needed here, and None will be potentially much faster in some cases.
-                    blend: None,
-                    write_mask: ColorWrites::ALL,
-                }),
-            key.mesh_key
-                .contains(MeshPipelineKey::MOTION_VECTOR_PREPASS)
-                .then_some(ColorTargetState {
-                    format: MOTION_VECTOR_PREPASS_FORMAT,
-                    // BlendState::REPLACE is not needed here, and None will be potentially much faster in some cases.
-                    blend: None,
-                    write_mask: ColorWrites::ALL,
-                }),
-            key.mesh_key
-                .contains(MeshPipelineKey::DEFERRED_PREPASS)
-                .then_some(ColorTargetState {
-                    format: DEFERRED_PREPASS_FORMAT,
-                    // BlendState::REPLACE is not needed here, and None will be potentially much faster in some cases.
-                    blend: None,
-                    write_mask: ColorWrites::ALL,
-                }),
-            key.mesh_key
-                .contains(MeshPipelineKey::DEFERRED_PREPASS)
-                .then_some(ColorTargetState {
-                    format: DEFERRED_LIGHTING_PASS_ID_FORMAT,
-                    blend: None,
-                    write_mask: ColorWrites::ALL,
-                }),
-        ];
 
         if targets.iter().all(Option::is_none) {
             // if no targets are required then clear the list, so that no fragment shader is required
@@ -623,16 +590,6 @@ pub fn extract_camera_previous_view_data(
     }
 }
 
-#[derive(Resource, Default)]
-pub struct PreviousViewUniforms {
-    pub uniforms: DynamicUniformBuffer<PreviousViewData>,
-}
-
-#[derive(Component)]
-pub struct PreviousViewUniformOffset {
-    pub offset: u32,
-}
-
 pub fn prepare_previous_view_uniforms(
     mut commands: Commands,
     render_device: Res<RenderDevice>,