new format for ci config file (#13154)

# Objective

- Current config file is hard to extend

## Solution

- Instead of an hard coded list of field, the file now has a list of
`(frame, event)`, and will deal with know events (exiting or taking a
screenshot), or send an event for others that can be dealt by third
party plugins

.github/example-run/alien_cake_addict.ron

@@ -1,3 +1,5 @@
 (
-    exit_after: Some(300)
+    events: [
+        (300, AppExit),
+    ]
 )

.github/example-run/breakout.ron

@@ -1,5 +1,9 @@
 (
-    exit_after: Some(900),
+    setup: (
-    frame_time: Some(0.03),
+        fixed_frame_time: Some(0.03),
-    screenshot_frames: [200],
+    ),
+    events: [
+        (200, Screenshot),
+        (900, AppExit),
+    ]
 )

.github/example-run/contributors.ron

@@ -1,3 +1,5 @@
 (
-    exit_after: Some(900)
+    events: [
+        (900, AppExit),
+    ]
 )

.github/example-run/load_gltf.ron

@@ -1,5 +1,9 @@
 (
-    exit_after: Some(300),
+    setup: (
-    frame_time: Some(0.03),
+        frame_time: Some(0.03),
-    screenshot_frames: [100],
+    ),
+    events: [
+        (100, Screenshot),
+        (300, AppExit),
+    ]
 )

.github/example-run/no_renderer.ron

@@ -1,3 +1,5 @@
 (
-    exit_after: Some(100)
+    events: [
+        (100, AppExit),
+    ]
 )

.github/example-run/scene.ron

@@ -1,3 +1,5 @@
 (
-    exit_after: Some(100)
+    events: [
+        (100, AppExit),
+    ]
 )

crates/bevy_dev_tools/src/ci_testing.rs

@@ -1,16 +1,13 @@
 //! Utilities for testing in CI environments.
 
 use bevy_app::{App, AppExit, Update};
-use bevy_ecs::{
+use bevy_ecs::prelude::*;
-    entity::Entity,
-    prelude::{resource_exists, Resource},
-    query::With,
-    schedule::IntoSystemConfigs,
-    system::{Local, Query, Res, ResMut},
-};
 use bevy_render::view::screenshot::ScreenshotManager;
 use bevy_time::TimeUpdateStrategy;
-use bevy_utils::{tracing::info, Duration};
+use bevy_utils::{
+    tracing::{debug, info, warn},
+    Duration,
+};
 use bevy_window::PrimaryWindow;
 use serde::Deserialize;
 
@@ -20,30 +17,39 @@ use serde::Deserialize;
 /// exit a Bevy app when run through the CI. This is needed because otherwise
 /// Bevy apps would be stuck in the game loop and wouldn't allow the CI to progress.
 #[derive(Deserialize, Resource)]
-pub struct CiTestingConfig {
+struct CiTestingConfig {
-    /// The number of frames after which Bevy should exit.
+    /// The setup for this test.
-    pub exit_after: Option<u32>,
-    /// The time in seconds to update for each frame.
-    pub frame_time: Option<f32>,
-    /// Frames at which to capture a screenshot.
     #[serde(default)]
-    pub screenshot_frames: Vec<u32>,
+    setup: CiTestingSetup,
+    /// Events to send, with their associated frame.
+    #[serde(default)]
+    events: Vec<CiTestingEventOnFrame>,
 }
 
-fn ci_testing_exit_after(
+/// Setup for a test.
-    mut current_frame: bevy_ecs::prelude::Local<u32>,
+#[derive(Deserialize, Default)]
-    ci_testing_config: bevy_ecs::prelude::Res<CiTestingConfig>,
+struct CiTestingSetup {
-    mut app_exit_events: bevy_ecs::event::EventWriter<AppExit>,
+    /// The time in seconds to update for each frame.
-) {
+    /// Set with the `TimeUpdateStrategy::ManualDuration(f32)` resource.
-    if let Some(exit_after) = ci_testing_config.exit_after {
+    pub fixed_frame_time: Option<f32>,
-        if *current_frame > exit_after {
-            app_exit_events.send(AppExit::Success);
-            info!("Exiting after {} frames. Test successful!", exit_after);
-        }
-    }
-    *current_frame += 1;
 }
 
+/// An event to send at a given frame, used for CI testing.
+#[derive(Deserialize)]
+pub struct CiTestingEventOnFrame(u32, CiTestingEvent);
+
+/// An event to send, used for CI testing.
+#[derive(Deserialize, Debug)]
+enum CiTestingEvent {
+    Screenshot,
+    AppExit,
+    Custom(String),
+}
+
+/// A custom event that can be configured from a configuration file for CI testing.
+#[derive(Event)]
+pub struct CiTestingCustomEvent(pub String);
+
 pub(crate) fn setup_app(app: &mut App) -> &mut App {
     #[cfg(not(target_arch = "wasm32"))]
     let config: CiTestingConfig = {
@@ -61,39 +67,60 @@ pub(crate) fn setup_app(app: &mut App) -> &mut App {
         ron::from_str(config).expect("error deserializing CI testing configuration file")
     };
 
-    if let Some(frame_time) = config.frame_time {
+    if let Some(fixed_frame_time) = config.setup.fixed_frame_time {
         app.world_mut()
             .insert_resource(TimeUpdateStrategy::ManualDuration(Duration::from_secs_f32(
-                frame_time,
+                fixed_frame_time,
             )));
     }
 
-    app.insert_resource(config).add_systems(
+    app.add_event::<CiTestingCustomEvent>()
-        Update,
+        .insert_resource(config)
-        (
+        .add_systems(Update, send_events);
-            ci_testing_exit_after,
-            ci_testing_screenshot_at.run_if(resource_exists::<ScreenshotManager>),
-        ),
-    );
 
     app
 }
 
-fn ci_testing_screenshot_at(
+fn send_events(world: &mut World, mut current_frame: Local<u32>) {
-    mut current_frame: Local<u32>,
+    let mut config = world.resource_mut::<CiTestingConfig>();
-    ci_testing_config: Res<CiTestingConfig>,
+
-    mut screenshot_manager: ResMut<ScreenshotManager>,
+    let events = std::mem::take(&mut config.events);
-    main_window: Query<Entity, With<PrimaryWindow>>,
+    let (to_run, remaining): (Vec<_>, _) = events
-) {
+        .into_iter()
-    if ci_testing_config
+        .partition(|event| event.0 == *current_frame);
-        .screenshot_frames
+    config.events = remaining;
-        .contains(&*current_frame)
+
-    {
+    for CiTestingEventOnFrame(_, event) in to_run {
-        info!("Taking a screenshot at frame {}.", *current_frame);
+        debug!("Handling event: {:?}", event);
-        let path = format!("./screenshot-{}.png", *current_frame);
+        match event {
-        screenshot_manager
+            CiTestingEvent::AppExit => {
-            .save_screenshot_to_disk(main_window.single(), path)
+                world.send_event(AppExit::Success);
-            .unwrap();
+                info!("Exiting after {} frames. Test successful!", *current_frame);
+            }
+            CiTestingEvent::Screenshot => {
+                let mut primary_window_query =
+                    world.query_filtered::<Entity, With<PrimaryWindow>>();
+                let Ok(main_window) = primary_window_query.get_single(world) else {
+                    warn!("Requesting screenshot, but PrimaryWindow is not available");
+                    continue;
+                };
+                let Some(mut screenshot_manager) = world.get_resource_mut::<ScreenshotManager>()
+                else {
+                    warn!("Requesting screenshot, but ScreenshotManager is not available");
+                    continue;
+                };
+                let path = format!("./screenshot-{}.png", *current_frame);
+                screenshot_manager
+                    .save_screenshot_to_disk(main_window, path)
+                    .unwrap();
+                info!("Took a screenshot at frame {}.", *current_frame);
+            }
+            // Custom events are forwarded to the world.
+            CiTestingEvent::Custom(event_string) => {
+                world.send_event(CiTestingCustomEvent(event_string));
+            }
+        }
     }
+
     *current_frame += 1;
 }

tools/build-wasm-example/src/main.rs

@@ -50,7 +50,7 @@ fn main() {
     let mut features: Vec<&str> = cli.features.iter().map(|f| f.as_str()).collect();
     if let Some(frames) = cli.frames {
         let mut file = File::create("ci_testing_config.ron").unwrap();
-        file.write_fmt(format_args!("(exit_after: Some({frames}))"))
+        file.write_fmt(format_args!("(events: [({frames}, AppExit)])"))
             .unwrap();
         features.push("bevy_ci_testing");
     }

tools/example-showcase/src/main.rs

@@ -168,7 +168,7 @@ fn main() {
                 (true, true) => {
                     let mut file = File::create("example_showcase_config.ron").unwrap();
                     file.write_all(
-                        b"(exit_after: None, frame_time: Some(0.05), screenshot_frames: [100])",
+                        b"(setup: (fixed_frame_time: Some(0.05)), events: [(100, Screenshot)])",
                     )
                     .unwrap();
                     extra_parameters.push("--features");
@@ -178,7 +178,7 @@ fn main() {
                 (false, true) => {
                     let mut file = File::create("example_showcase_config.ron").unwrap();
                     file.write_all(
-                        b"(exit_after: Some(250), frame_time: Some(0.05), screenshot_frames: [100])",
+                        b"(setup: (fixed_frame_time: Some(0.05)), events: [(100, Screenshot), (250, AppExit)])",
                     )
                     .unwrap();
                     extra_parameters.push("--features");
@@ -186,7 +186,7 @@ fn main() {
                 }
                 (false, false) => {
                     let mut file = File::create("example_showcase_config.ron").unwrap();
-                    file.write_all(b"(exit_after: Some(250))").unwrap();
+                    file.write_all(b"(events: [(250, AppExit)])").unwrap();
                     extra_parameters.push("--features");
                     extra_parameters.push("bevy_ci_testing");
                 }