From 39ea1bb9b7f03e9803648b2c28c269ce365e56e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Mon, 6 Jun 2022 20:22:51 +0000 Subject: [PATCH] run examples in wasm in CI (#4818) # Objective - Run examples in WASM in CI - Fix #4817 ## Solution - on feature `bevy_ci_testing` - add an extra log message before exiting - when building for wasm, read CI config file at compile time - add a simple [playwright](https://playwright.dev) test script that opens the browser then waits for the success log, and takes a screenshot - add a CI job that runs the playwright test for Chromium and Firefox on one example (lighting) and save the screenshots - Firefox screenshot is good (with some clusters visible) - Chromium screenshot is gray, I don't know why but it's logging `GPU stall due to ReadPixels` - Webkit is not enabled for now, to revisit once https://bugs.webkit.org/show_bug.cgi?id=234926 is fixed or worked around - the CI job only runs on bors validation example run: https://github.com/mockersf/bevy/actions/runs/2361673465. The screenshots can be downloaded --- .github/bors.toml | 1 + .github/start-wasm-example/.gitignore | 4 + .github/start-wasm-example/package-lock.json | 76 +++++++++++++ .github/start-wasm-example/package.json | 16 +++ .../start-wasm-example/playwright.config.ts | 107 ++++++++++++++++++ .../tests/wasm_example.spec.ts | 51 +++++++++ .github/workflows/validation-jobs.yml | 58 ++++++++++ crates/bevy_app/src/ci_testing.rs | 25 +++- tools/build-wasm-example/Cargo.toml | 1 + tools/build-wasm-example/src/main.rs | 78 ++++++++++--- 10 files changed, 396 insertions(+), 21 deletions(-) create mode 100644 .github/start-wasm-example/.gitignore create mode 100644 .github/start-wasm-example/package-lock.json create mode 100644 .github/start-wasm-example/package.json create mode 100644 .github/start-wasm-example/playwright.config.ts create mode 100644 .github/start-wasm-example/tests/wasm_example.spec.ts diff --git a/.github/bors.toml b/.github/bors.toml index daaf76ad1e..d1735c98b9 100644 --- a/.github/bors.toml +++ b/.github/bors.toml @@ -8,6 +8,7 @@ status = [ "build-android", "markdownlint", "run-examples", + "run-examples-on-wasm", "check-doc", "check-missing-examples-in-docs", "check-unused-dependencies", diff --git a/.github/start-wasm-example/.gitignore b/.github/start-wasm-example/.gitignore new file mode 100644 index 0000000000..75e854d8dc --- /dev/null +++ b/.github/start-wasm-example/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +/test-results/ +/playwright-report/ +/playwright/.cache/ diff --git a/.github/start-wasm-example/package-lock.json b/.github/start-wasm-example/package-lock.json new file mode 100644 index 0000000000..4a39b73a25 --- /dev/null +++ b/.github/start-wasm-example/package-lock.json @@ -0,0 +1,76 @@ +{ + "name": "start-wasm-example", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "start-wasm-example", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "dotenv": "^16.0.1" + }, + "devDependencies": { + "@playwright/test": "^1.22.1" + } + }, + "node_modules/@playwright/test": { + "version": "1.22.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.22.1.tgz", + "integrity": "sha512-8ouMBUboYslHom41W8bnSEn0TwlAMHhCACwOZeuiAgzukj7KobpZ+UBwrGE0jJ0UblJbKAQNRHXL+z7sDSkb6g==", + "dev": true, + "dependencies": { + "playwright-core": "1.22.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/dotenv": { + "version": "16.0.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.1.tgz", + "integrity": "sha512-1K6hR6wtk2FviQ4kEiSjFiH5rpzEVi8WW0x96aztHVMhEspNpc4DVOUTEHtEva5VThQ8IaBX1Pe4gSzpVVUsKQ==", + "engines": { + "node": ">=12" + } + }, + "node_modules/playwright-core": { + "version": "1.22.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.22.1.tgz", + "integrity": "sha512-H+ZUVYnceWNXrRf3oxTEKAr81QzFsCKu5Fp//fEjQvqgKkfA1iX3E9DBrPJpPNOrgVzcE+IqeI0fDmYJe6Ynnw==", + "dev": true, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=14" + } + } + }, + "dependencies": { + "@playwright/test": { + "version": "1.22.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.22.1.tgz", + "integrity": "sha512-8ouMBUboYslHom41W8bnSEn0TwlAMHhCACwOZeuiAgzukj7KobpZ+UBwrGE0jJ0UblJbKAQNRHXL+z7sDSkb6g==", + "dev": true, + "requires": { + "playwright-core": "1.22.1" + } + }, + "dotenv": { + "version": "16.0.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.1.tgz", + "integrity": "sha512-1K6hR6wtk2FviQ4kEiSjFiH5rpzEVi8WW0x96aztHVMhEspNpc4DVOUTEHtEva5VThQ8IaBX1Pe4gSzpVVUsKQ==" + }, + "playwright-core": { + "version": "1.22.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.22.1.tgz", + "integrity": "sha512-H+ZUVYnceWNXrRf3oxTEKAr81QzFsCKu5Fp//fEjQvqgKkfA1iX3E9DBrPJpPNOrgVzcE+IqeI0fDmYJe6Ynnw==", + "dev": true + } + } +} diff --git a/.github/start-wasm-example/package.json b/.github/start-wasm-example/package.json new file mode 100644 index 0000000000..9e10e8134e --- /dev/null +++ b/.github/start-wasm-example/package.json @@ -0,0 +1,16 @@ +{ + "name": "start-wasm-example", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": {}, + "keywords": [], + "author": "", + "license": "ISC", + "devDependencies": { + "@playwright/test": "^1.22.1" + }, + "dependencies": { + "dotenv": "^16.0.1" + } +} diff --git a/.github/start-wasm-example/playwright.config.ts b/.github/start-wasm-example/playwright.config.ts new file mode 100644 index 0000000000..f5988c74a2 --- /dev/null +++ b/.github/start-wasm-example/playwright.config.ts @@ -0,0 +1,107 @@ +import type { PlaywrightTestConfig } from '@playwright/test'; +import { devices } from '@playwright/test'; + +/** + * Read environment variables from file. + * https://github.com/motdotla/dotenv + */ +require('dotenv').config(); + +/** + * See https://playwright.dev/docs/test-configuration. + */ +const config: PlaywrightTestConfig = { + testDir: './tests', + /* Maximum time one test can run for. */ + timeout: 300_000, + expect: { + /** + * Maximum time expect() should wait for the condition to be met. + * For example in `await expect(locator).toHaveText();` + */ + timeout: 5000 + }, + /* Run tests in files in parallel */ + fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: 0, + /* Opt out of parallel tests on CI. */ + workers: 1, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: 'list', + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */ + actionTimeout: 0, + /* Base URL to use in actions like `await page.goto('/')`. */ + // baseURL: 'http://localhost:3000', + + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: 'on-first-retry', + }, + + /* Configure projects for major browsers */ + projects: [ + { + name: 'chromium', + use: { + ...devices['Desktop Chrome'], + }, + }, + + { + name: 'firefox', + use: { + ...devices['Desktop Firefox'], + }, + }, + + { + name: 'webkit', + use: { + ...devices['Desktop Safari'], + }, + }, + + /* Test against mobile viewports. */ + // { + // name: 'Mobile Chrome', + // use: { + // ...devices['Pixel 5'], + // }, + // }, + // { + // name: 'Mobile Safari', + // use: { + // ...devices['iPhone 12'], + // }, + // }, + + /* Test against branded browsers. */ + // { + // name: 'Microsoft Edge', + // use: { + // channel: 'msedge', + // }, + // }, + // { + // name: 'Google Chrome', + // use: { + // channel: 'chrome', + // }, + // }, + ], + + /* Folder for test artifacts such as screenshots, videos, traces, etc. */ + // outputDir: 'test-results/', + + /* Run your local dev server before starting the tests */ + // webServer: { + // command: 'npm run start', + // port: 3000, + // }, +}; + +export default config; diff --git a/.github/start-wasm-example/tests/wasm_example.spec.ts b/.github/start-wasm-example/tests/wasm_example.spec.ts new file mode 100644 index 0000000000..32bb4d9d2c --- /dev/null +++ b/.github/start-wasm-example/tests/wasm_example.spec.ts @@ -0,0 +1,51 @@ +import { test, expect, Page } from '@playwright/test'; + +test.beforeEach(async ({ page }) => { + await page.goto('http://localhost:8000/'); +}); + +const MAX_TIMEOUT_FOR_TEST = 300_000; + +test.describe('WASM example', () => { + test('Wait for success', async ({ page }, test_info) => { + let start = new Date().getTime(); + + let found = false; + while (new Date().getTime() - start < MAX_TIMEOUT_FOR_TEST) { + let msg = await promise_with_timeout(100, on_console(page), "no log found"); + if (msg.includes("no log found")) { + continue; + } + console.log(msg); + if (msg.includes("Test successful")) { + let prefix = process.env.SCREENSHOT_PREFIX === undefined ? "screenshot" : process.env.SCREENSHOT_PREFIX; + await page.screenshot({ path: `${prefix}-${test_info.project.name}.png`, fullPage: true }); + found = true; + break; + } + } + + expect(found).toBe(true); + }); + +}); + +function on_console(page) { + return new Promise(resolve => { + page.on('console', msg => resolve(msg.text())); + }); +} + +async function promise_with_timeout(time_limit, task, failure_value) { + let timeout; + const timeout_promise = new Promise((resolve, reject) => { + timeout = setTimeout(() => { + resolve(failure_value); + }, time_limit); + }); + const response = await Promise.race([task, timeout_promise]); + if (timeout) { + clearTimeout(timeout); + } + return response; +} \ No newline at end of file diff --git a/.github/workflows/validation-jobs.yml b/.github/workflows/validation-jobs.yml index 533d20b7ca..ec060da4af 100644 --- a/.github/workflows/validation-jobs.yml +++ b/.github/workflows/validation-jobs.yml @@ -97,3 +97,61 @@ jobs: time CI_TESTING_CONFIG=$example cargo run --example $example_name --features "bevy_ci_testing" sleep 10 done + + run-examples-on-wasm: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v3 + + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + target: wasm32-unknown-unknown + override: true + + - uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + ~/.github/start-wasm-example/node_modules + target/ + key: ${{ runner.os }}-wasm-run-examples-${{ hashFiles('**/Cargo.toml') }} + + - name: install xvfb, llvmpipe and lavapipe + run: | + sudo apt-get update -y -qq + sudo add-apt-repository ppa:oibaf/graphics-drivers -y + sudo apt-get update + sudo apt install -y xvfb libegl1-mesa libgl1-mesa-dri libxcb-xfixes0-dev mesa-vulkan-drivers + + - name: Install wasm-bindgen + run: cargo install --force wasm-bindgen-cli + + - name: Setup playwright + run: | + cd .github/start-wasm-example + npm install + npx playwright install --with-deps + cd ../.. + + - name: First WASM build + run: | + cargo build --release --example ui --target wasm32-unknown-unknown + + - name: Run examples + shell: bash + run: | + # start a webserver + python3 -m http.server --directory examples/wasm & + + xvfb-run cargo run -p build-wasm-example -- --browsers chromium --browsers firefox --frames 25 --test shapes lighting text_debug breakout + + - name: Save screenshots + uses: actions/upload-artifact@v3 + with: + name: screenshots + path: .github/start-wasm-example/screenshot-*.png diff --git a/crates/bevy_app/src/ci_testing.rs b/crates/bevy_app/src/ci_testing.rs index b319f04a91..66206d9109 100644 --- a/crates/bevy_app/src/ci_testing.rs +++ b/crates/bevy_app/src/ci_testing.rs @@ -1,6 +1,8 @@ use crate::{app::AppExit, App}; use serde::Deserialize; +use bevy_utils::tracing::info; + /// A configuration struct for automated CI testing. /// /// It gets used when the `bevy_ci_testing` feature is enabled to automatically @@ -20,18 +22,29 @@ fn ci_testing_exit_after( if let Some(exit_after) = ci_testing_config.exit_after { if *current_frame > exit_after { app_exit_events.send(AppExit); + info!("Exiting after {} frames. Test successful!", exit_after); } } *current_frame += 1; } pub(crate) fn setup_app(app: &mut App) -> &mut App { - let filename = - std::env::var("CI_TESTING_CONFIG").unwrap_or_else(|_| "ci_testing_config.ron".to_string()); - let config: CiTestingConfig = ron::from_str( - &std::fs::read_to_string(filename).expect("error reading CI testing configuration file"), - ) - .expect("error deserializing CI testing configuration file"); + #[cfg(not(target_arch = "wasm32"))] + let config: CiTestingConfig = { + let filename = std::env::var("CI_TESTING_CONFIG") + .unwrap_or_else(|_| "ci_testing_config.ron".to_string()); + ron::from_str( + &std::fs::read_to_string(filename) + .expect("error reading CI testing configuration file"), + ) + .expect("error deserializing CI testing configuration file") + }; + #[cfg(target_arch = "wasm32")] + let config: CiTestingConfig = { + let config = include_str!("../../../ci_testing_config.ron"); + ron::from_str(config).expect("error deserializing CI testing configuration file") + }; + app.insert_resource(config) .add_system(ci_testing_exit_after); diff --git a/tools/build-wasm-example/Cargo.toml b/tools/build-wasm-example/Cargo.toml index fdd3199a7f..998d3e800b 100644 --- a/tools/build-wasm-example/Cargo.toml +++ b/tools/build-wasm-example/Cargo.toml @@ -9,3 +9,4 @@ license = "MIT OR Apache-2.0" [dependencies] xshell = "0.2" +clap = { version = "3.1.12", features = ["derive"] } diff --git a/tools/build-wasm-example/src/main.rs b/tools/build-wasm-example/src/main.rs index 39d43e15a6..8d89f817c4 100644 --- a/tools/build-wasm-example/src/main.rs +++ b/tools/build-wasm-example/src/main.rs @@ -1,18 +1,66 @@ +use std::{fs::File, io::Write}; + +use clap::Parser; use xshell::{cmd, Shell}; -fn main() { - let example = std::env::args().nth(1).expect("abbb"); - let sh = Shell::new().unwrap(); - cmd!( - sh, - "cargo build --release --target wasm32-unknown-unknown --example {example}" - ) - .run() - .expect("Error building example"); - cmd!( - sh, - "wasm-bindgen --out-dir examples/wasm/target --out-name wasm_example --target web target/wasm32-unknown-unknown/release/examples/{example}.wasm" - ) - .run() - .expect("Error creating wasm binding"); +#[derive(Parser, Debug)] +struct Args { + /// Examples to build + examples: Vec, + + #[clap(short, long)] + /// Run tests + test: bool, + + #[clap(short, long)] + /// Run on the given browsers. By default, chromium, firefox, webkit + browsers: Vec, + + #[clap(short, long)] + /// Stop after this number of frames + frames: Option, +} + +fn main() { + let cli = Args::parse(); + eprintln!("{:?}", cli); + + assert!(!cli.examples.is_empty(), "must have at least one example"); + + let mut bevy_ci_testing = vec![]; + if let Some(frames) = cli.frames { + let mut file = File::create("ci_testing_config.ron").unwrap(); + file.write_fmt(format_args!("(exit_after: Some({}))", frames)) + .unwrap(); + bevy_ci_testing = vec!["--features", "bevy_ci_testing"]; + } + + for example in cli.examples { + let sh = Shell::new().unwrap(); + let bevy_ci_testing = bevy_ci_testing.clone(); + cmd!( + sh, + "cargo build {bevy_ci_testing...} --release --target wasm32-unknown-unknown --example {example}" + ) + .run() + .expect("Error building example"); + cmd!( + sh, + "wasm-bindgen --out-dir examples/wasm/target --out-name wasm_example --target web target/wasm32-unknown-unknown/release/examples/{example}.wasm" + ) + .run() + .expect("Error creating wasm binding"); + + if cli.test { + let _dir = sh.push_dir(".github/start-wasm-example"); + let mut browsers = cli.browsers.clone(); + if !browsers.is_empty() { + browsers.insert(0, "--project".to_string()); + } + cmd!(sh, "npx playwright test --headed {browsers...}") + .env("SCREENSHOT_PREFIX", format!("screenshot-{example}")) + .run() + .expect("Error running playwright test"); + } + } }