Smarter testbeds (#17573)

# Objective

- Improve CI when testing rendering by having smarter testbeds

## Solution

- CI testing no longer need a config file and will run with a default
config if not found
- It is now possible to give a name to a screenshot instead of just a
frame number
- 2d and 3d testbeds are now driven from code
  - a new system in testbed will watch for state changed
- on state changed, trigger a screenshot 100 frames after (so that the
scene has time to render) with the name of the scene
- when the screenshot is taken (`Captured` component has been removed),
switch scene
- this means less setup to run a testbed (no need for a config file),
screenshots have better names, and it's faster as we don't wait 100
frames for the screenshot to be taken

## Testing

- `cargo run --example testbed_2d --features bevy_ci_testing`

.github/example-run/testbed_2d.ron

@@ -1,12 +1,4 @@
 (
     events: [
-        (100, Screenshot),
-        (200, Custom("switch_scene")),
-        (300, Screenshot),
-        (400, Custom("switch_scene")),
-        (500, Screenshot),
-        (600, Custom("switch_scene")),
-        (700, Screenshot),
-        (800, AppExit),
     ]
 )

.github/example-run/testbed_3d.ron

@@ -1,12 +1,4 @@
 (
     events: [
-        (100, Screenshot),
-        (200, Custom("switch_scene")),
-        (300, Screenshot),
-        (400, Custom("switch_scene")),
-        (500, Screenshot),
-        (600, Custom("switch_scene")),
-        (700, Screenshot),
-        (800, AppExit),
     ]
 )

crates/bevy_dev_tools/src/ci_testing/config.rs

@@ -6,7 +6,7 @@ use serde::Deserialize;
 /// It gets used when the `bevy_ci_testing` feature is enabled to automatically
 /// exit a Bevy app when run through the CI. This is needed because otherwise
 /// Bevy apps would be stuck in the game loop and wouldn't allow the CI to progress.
-#[derive(Deserialize, Resource, PartialEq, Debug)]
+#[derive(Deserialize, Resource, PartialEq, Debug, Default)]
 pub struct CiTestingConfig {
     /// The setup for this test.
     #[serde(default)]
@@ -37,6 +37,9 @@ pub enum CiTestingEvent {
     /// Takes a screenshot of the entire screen, and saves the results to
     /// `screenshot-{current_frame}.png`.
     Screenshot,
+    /// Takes a screenshot of the entire screen, and saves the results to
+    /// `screenshot-{name}.png`.
+    NamedScreenshot(String),
     /// Stops the program by sending [`AppExit::Success`].
     ///
     /// [`AppExit::Success`]: bevy_app::AppExit::Success

crates/bevy_dev_tools/src/ci_testing/mod.rs

@@ -30,11 +30,12 @@ impl Plugin for CiTestingPlugin {
         let config: CiTestingConfig = {
             let filename = std::env::var("CI_TESTING_CONFIG")
                 .unwrap_or_else(|_| "ci_testing_config.ron".to_string());
-            ron::from_str(
-                &std::fs::read_to_string(filename)
-                    .expect("error reading CI testing configuration file"),
-            )
-            .expect("error deserializing CI testing configuration file")
+            std::fs::read_to_string(filename)
+                .map(|content| {
+                    ron::from_str(&content)
+                        .expect("error deserializing CI testing configuration file")
+                })
+                .unwrap_or_default()
         };
 
         #[cfg(target_arch = "wasm32")]

crates/bevy_dev_tools/src/ci_testing/systems.rs

@@ -28,6 +28,16 @@ pub(crate) fn send_events(world: &mut World, mut current_frame: Local<u32>) {
                     .observe(save_to_disk(path));
                 info!("Took a screenshot at frame {}.", *current_frame);
             }
+            CiTestingEvent::NamedScreenshot(name) => {
+                let path = format!("./screenshot-{}.png", name);
+                world
+                    .spawn(Screenshot::primary_window())
+                    .observe(save_to_disk(path));
+                info!(
+                    "Took a screenshot at frame {} for {}.",
+                    *current_frame, name
+                );
+            }
             // Custom events are forwarded to the world.
             CiTestingEvent::Custom(event_string) => {
                 world.send_event(CiTestingCustomEvent(event_string));

examples/testbed/2d.rs

@@ -2,9 +2,10 @@
 //!
 //! You can switch scene by pressing the spacebar
 
-#[cfg(feature = "bevy_ci_testing")]
-use bevy::dev_tools::ci_testing::CiTestingCustomEvent;
+mod helpers;
+
 use bevy::prelude::*;
+use helpers::Next;
 
 fn main() {
     let mut app = App::new();
@@ -15,6 +16,10 @@ fn main() {
         .add_systems(OnEnter(Scene::Text), text::setup)
         .add_systems(OnEnter(Scene::Sprite), sprite::setup)
         .add_systems(Update, switch_scene);
+
+    #[cfg(feature = "bevy_ci_testing")]
+    app.add_systems(Update, helpers::switch_scene_in_ci::<Scene>);
+
     app.run();
 }
 
@@ -28,28 +33,25 @@ enum Scene {
     Sprite,
 }
 
-fn switch_scene(
-    keyboard: Res<ButtonInput<KeyCode>>,
-    #[cfg(feature = "bevy_ci_testing")] mut ci_events: EventReader<CiTestingCustomEvent>,
-    scene: Res<State<Scene>>,
-    mut next_scene: ResMut<NextState<Scene>>,
-) {
-    let mut should_switch = false;
-    should_switch |= keyboard.just_pressed(KeyCode::Space);
-    #[cfg(feature = "bevy_ci_testing")]
-    {
-        should_switch |= ci_events.read().any(|event| match event {
-            CiTestingCustomEvent(event) => event == "switch_scene",
-        });
-    }
-    if should_switch {
-        info!("Switching scene");
-        next_scene.set(match scene.get() {
+impl Next for Scene {
+    fn next(&self) -> Self {
+        match self {
             Scene::Shapes => Scene::Bloom,
             Scene::Bloom => Scene::Text,
             Scene::Text => Scene::Sprite,
             Scene::Sprite => Scene::Shapes,
-        });
+        }
+    }
+}
+
+fn switch_scene(
+    keyboard: Res<ButtonInput<KeyCode>>,
+    scene: Res<State<Scene>>,
+    mut next_scene: ResMut<NextState<Scene>>,
+) {
+    if keyboard.just_pressed(KeyCode::Space) {
+        info!("Switching scene");
+        next_scene.set(scene.get().next());
     }
 }
 

examples/testbed/3d.rs

@@ -2,9 +2,10 @@
 //!
 //! You can switch scene by pressing the spacebar
 
-#[cfg(feature = "bevy_ci_testing")]
-use bevy::dev_tools::ci_testing::CiTestingCustomEvent;
+mod helpers;
+
 use bevy::prelude::*;
+use helpers::Next;
 
 fn main() {
     let mut app = App::new();
@@ -19,6 +20,9 @@ fn main() {
     app.add_systems(OnEnter(Scene::Bloom), bloom::setup)
         .add_systems(OnEnter(Scene::Gltf), gltf::setup);
 
+    #[cfg(feature = "bevy_ci_testing")]
+    app.add_systems(Update, helpers::switch_scene_in_ci::<Scene>);
+
     app.run();
 }
 
@@ -32,28 +36,25 @@ enum Scene {
     Animation,
 }
 
-fn switch_scene(
-    keyboard: Res<ButtonInput<KeyCode>>,
-    #[cfg(feature = "bevy_ci_testing")] mut ci_events: EventReader<CiTestingCustomEvent>,
-    scene: Res<State<Scene>>,
-    mut next_scene: ResMut<NextState<Scene>>,
-) {
-    let mut should_switch = false;
-    should_switch |= keyboard.just_pressed(KeyCode::Space);
-    #[cfg(feature = "bevy_ci_testing")]
-    {
-        should_switch |= ci_events.read().any(|event| match event {
-            CiTestingCustomEvent(event) => event == "switch_scene",
-        });
-    }
-    if should_switch {
-        info!("Switching scene");
-        next_scene.set(match scene.get() {
+impl Next for Scene {
+    fn next(&self) -> Self {
+        match self {
             Scene::Light => Scene::Bloom,
             Scene::Bloom => Scene::Gltf,
             Scene::Gltf => Scene::Animation,
             Scene::Animation => Scene::Light,
-        });
+        }
+    }
+}
+
+fn switch_scene(
+    keyboard: Res<ButtonInput<KeyCode>>,
+    scene: Res<State<Scene>>,
+    mut next_scene: ResMut<NextState<Scene>>,
+) {
+    if keyboard.just_pressed(KeyCode::Space) {
+        info!("Switching scene");
+        next_scene.set(scene.get().next());
     }
 }
 

examples/testbed/helpers.rs

@@ -0,0 +1,45 @@
+#[cfg(feature = "bevy_ci_testing")]
+use bevy::{
+    dev_tools::ci_testing::{CiTestingConfig, CiTestingEvent, CiTestingEventOnFrame},
+    diagnostic::FrameCount,
+    platform_support::collections::HashSet,
+    prelude::*,
+    render::view::screenshot::Captured,
+    state::state::FreelyMutableState,
+};
+
+#[cfg(feature = "bevy_ci_testing")]
+pub fn switch_scene_in_ci<Scene: States + FreelyMutableState + Next>(
+    mut ci_config: ResMut<CiTestingConfig>,
+    scene: Res<State<Scene>>,
+    mut next_scene: ResMut<NextState<Scene>>,
+    mut scenes_visited: Local<HashSet<Scene>>,
+    frame_count: Res<FrameCount>,
+    captured: RemovedComponents<Captured>,
+) {
+    if scene.is_changed() {
+        // Changed scene! trigger a screenshot in 100 frames
+        ci_config.events.push(CiTestingEventOnFrame(
+            frame_count.0 + 100,
+            CiTestingEvent::NamedScreenshot(format!("{:?}", scene.get())),
+        ));
+        if scenes_visited.contains(scene.get()) {
+            // Exit once all scenes have been screenshotted
+            ci_config.events.push(CiTestingEventOnFrame(
+                frame_count.0 + 1,
+                CiTestingEvent::AppExit,
+            ));
+        }
+        return;
+    }
+
+    if !captured.is_empty() {
+        // Screenshot taken! Switch to the next scene
+        scenes_visited.insert(scene.get().clone());
+        next_scene.set(scene.get().next());
+    }
+}
+
+pub trait Next {
+    fn next(&self) -> Self;
+}