forestia/bevy
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Speed up ECS benchmarks by limiting variations (#18659)

Browse Source 
## Objective

Reduce the time spent on ECS benchmarks without significantly
compromising coverage.

## Background

A `cargo bench -p benches --bench ecs` takes about 45 minutes. I'm
guessing this bench is mainly used to check for regressions after ECS
changes, and requiring 2x45 minute tests means that most people will
skip benchmarking entirely.

I noticed that some benches are repeated with sizes from long linear
progressions (10, 20, ..., 100). This might be nice for detailed
profiling, but seems too much for a overall regression check.

## Solution

The PR follows the principles of "three or four different sizes is fine"
and "powers of ten where it fits". The number of benches is reduced from
394 to 238 (-40%), and time from 46.2 minutes to 32.8 (-30%).

While some coverage is lost, I think it's reasonable for anyone doing
detailed profiling of a particular feature to temporarily add more
benches.

There's a couple of changes to avoid leading zeroes. I felt that `0010,
0100, 1000` is harder to read than `10, 100, 1000`.

## Is That Enough?

32 minutes is still too much. Possible future options:

- Reduce measurement and warmup times. I suspect the current times
(mostly 4-5 seconds total) are too conservative, and 1 second would be
fine for spotting significant regressions.
- Split the bench into quick and detailed variants.

## Testing

```
cargo bench -p benches --bench ecs
```
This commit is contained in:
Greeble 2025-05-06 01:07:18 +01:00 committed by GitHub
parent 023b502153
commit 49f1827633
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 42 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
benches/benches/bevy_ecs/components/archetype_updates.rs
View File

@ -51,8 +51,7 @@ fn add_archetypes(world: &mut World, count: u16) {
pub fn no_archetypes(criterion: &mut Criterion) {
let mut group = criterion.benchmark_group("no_archetypes");
for i in 0..=5 {
let system_count = i * 20;
for system_count in [0, 10, 100] {
let (mut world, mut schedule) = setup(system_count);
group.bench_with_input(
BenchmarkId::new("system_count", system_count),
@ -69,7 +68,7 @@ pub fn no_archetypes(criterion: &mut Criterion) {
pub fn added_archetypes(criterion: &mut Criterion) {
const SYSTEM_COUNT: usize = 100;
let mut group = criterion.benchmark_group("added_archetypes");
for archetype_count in [100, 200, 500, 1000, 2000, 5000, 10000] {
for archetype_count in [100, 1_000, 10_000] {
group.bench_with_input(
BenchmarkId::new("archetype_count", archetype_count),
&archetype_count,

6
benches/benches/bevy_ecs/empty_archetypes.rs
View File

@ -155,7 +155,7 @@ fn add_archetypes(world: &mut World, count: u16) {
fn empty_archetypes(criterion: &mut Criterion) {
let mut group = criterion.benchmark_group("empty_archetypes");
for archetype_count in [10, 100, 500, 1000, 2000, 5000, 10000] {
for archetype_count in [10, 100, 1_000, 10_000] {
let (mut world, mut schedule) = setup(true, |schedule| {
schedule.add_systems(iter);
});
@ -186,7 +186,7 @@ fn empty_archetypes(criterion: &mut Criterion) {
},
);
}
for archetype_count in [10, 100, 500, 1000, 2000, 5000, 10000] {
for archetype_count in [10, 100, 1_000, 10_000] {
let (mut world, mut schedule) = setup(true, |schedule| {
schedule.add_systems(for_each);
});
@ -217,7 +217,7 @@ fn empty_archetypes(criterion: &mut Criterion) {
},
);
}
for archetype_count in [10, 100, 500, 1000, 2000, 5000, 10000] {
for archetype_count in [10, 100, 1_000, 10_000] {
let (mut world, mut schedule) = setup(true, |schedule| {
schedule.add_systems(par_for_each);
});

12
benches/benches/bevy_ecs/events/mod.rs
View File

@ -9,19 +9,19 @@ fn send(c: &mut Criterion) {
let mut group = c.benchmark_group("events_send");
group.warm_up_time(core::time::Duration::from_millis(500));
group.measurement_time(core::time::Duration::from_secs(4));
for count in [100, 1000, 10000, 50000] {
for count in [100, 1_000, 10_000] {
group.bench_function(format!("size_4_events_{}", count), |b| {
let mut bench = send::Benchmark::<4>::new(count);
b.iter(move || bench.run());
});
}
for count in [100, 1000, 10000, 50000] {
for count in [100, 1_000, 10_000] {
group.bench_function(format!("size_16_events_{}", count), |b| {
let mut bench = send::Benchmark::<16>::new(count);
b.iter(move || bench.run());
});
}
for count in [100, 1000, 10000, 50000] {
for count in [100, 1_000, 10_000] {
group.bench_function(format!("size_512_events_{}", count), |b| {
let mut bench = send::Benchmark::<512>::new(count);
b.iter(move || bench.run());
@ -34,19 +34,19 @@ fn iter(c: &mut Criterion) {
let mut group = c.benchmark_group("events_iter");
group.warm_up_time(core::time::Duration::from_millis(500));
group.measurement_time(core::time::Duration::from_secs(4));
for count in [100, 1000, 10000, 50000] {
for count in [100, 1_000, 10_000] {
group.bench_function(format!("size_4_events_{}", count), |b| {
let mut bench = iter::Benchmark::<4>::new(count);
b.iter(move || bench.run());
});
}
for count in [100, 1000, 10000, 50000] {
for count in [100, 1_000, 10_000] {
group.bench_function(format!("size_16_events_{}", count), |b| {
let mut bench = iter::Benchmark::<4>::new(count);
b.iter(move || bench.run());
});
}
for count in [100, 1000, 10000, 50000] {
for count in [100, 1_000, 10_000] {
group.bench_function(format!("size_512_events_{}", count), |b| {
let mut bench = iter::Benchmark::<512>::new(count);
b.iter(move || bench.run());

28
benches/benches/bevy_ecs/scheduling/run_condition.rs
View File

@ -17,15 +17,14 @@ pub fn run_condition_yes(criterion: &mut Criterion) {
group.warm_up_time(core::time::Duration::from_millis(500));
group.measurement_time(core::time::Duration::from_secs(3));
fn empty() {}
for amount in 0..21 {
for amount in [10, 100, 1_000] {
let mut schedule = Schedule::default();
schedule.add_systems(empty.run_if(yes));
for _ in 0..amount {
for _ in 0..(amount / 5) {
schedule.add_systems((empty, empty, empty, empty, empty).distributive_run_if(yes));
}
// run once to initialize systems
schedule.run(&mut world);
group.bench_function(format!("{:03}_systems", 5 * amount + 1), |bencher| {
group.bench_function(format!("{}_systems", amount), |bencher| {
bencher.iter(|| {
schedule.run(&mut world);
});
@ -40,15 +39,14 @@ pub fn run_condition_no(criterion: &mut Criterion) {
group.warm_up_time(core::time::Duration::from_millis(500));
group.measurement_time(core::time::Duration::from_secs(3));
fn empty() {}
for amount in 0..21 {
for amount in [10, 100, 1_000] {
let mut schedule = Schedule::default();
schedule.add_systems(empty.run_if(no));
for _ in 0..amount {
for _ in 0..(amount / 5) {
schedule.add_systems((empty, empty, empty, empty, empty).distributive_run_if(no));
}
// run once to initialize systems
schedule.run(&mut world);
group.bench_function(format!("{:03}_systems", 5 * amount + 1), |bencher| {
group.bench_function(format!("{}_systems", amount), |bencher| {
bencher.iter(|| {
schedule.run(&mut world);
});
@ -70,17 +68,16 @@ pub fn run_condition_yes_with_query(criterion: &mut Criterion) {
fn yes_with_query(query: Single<&TestBool>) -> bool {
query.0
}
for amount in 0..21 {
for amount in [10, 100, 1_000] {
let mut schedule = Schedule::default();
schedule.add_systems(empty.run_if(yes_with_query));
for _ in 0..amount {
for _ in 0..(amount / 5) {
schedule.add_systems(
(empty, empty, empty, empty, empty).distributive_run_if(yes_with_query),
);
}
// run once to initialize systems
schedule.run(&mut world);
group.bench_function(format!("{:03}_systems", 5 * amount + 1), |bencher| {
group.bench_function(format!("{}_systems", amount), |bencher| {
bencher.iter(|| {
schedule.run(&mut world);
});
@ -99,17 +96,16 @@ pub fn run_condition_yes_with_resource(criterion: &mut Criterion) {
fn yes_with_resource(res: Res<TestBool>) -> bool {
res.0
}
for amount in 0..21 {
for amount in [10, 100, 1_000] {
let mut schedule = Schedule::default();
schedule.add_systems(empty.run_if(yes_with_resource));
for _ in 0..amount {
for _ in 0..(amount / 5) {
schedule.add_systems(
(empty, empty, empty, empty, empty).distributive_run_if(yes_with_resource),
);
}
// run once to initialize systems
schedule.run(&mut world);
group.bench_function(format!("{:03}_systems", 5 * amount + 1), |bencher| {
group.bench_function(format!("{}_systems", amount), |bencher| {
bencher.iter(|| {
schedule.run(&mut world);
});

30
benches/benches/bevy_ecs/scheduling/running_systems.rs
View File

@ -20,25 +20,25 @@ pub fn empty_systems(criterion: &mut Criterion) {
group.warm_up_time(core::time::Duration::from_millis(500));
group.measurement_time(core::time::Duration::from_secs(3));
fn empty() {}
for amount in 0..5 {
for amount in [0, 2, 4] {
let mut schedule = Schedule::default();
for _ in 0..amount {
schedule.add_systems(empty);
}
schedule.run(&mut world);
group.bench_function(format!("{:03}_systems", amount), |bencher| {
group.bench_function(format!("{}_systems", amount), |bencher| {
bencher.iter(|| {
schedule.run(&mut world);
});
});
}
for amount in 1..21 {
for amount in [10, 100, 1_000] {
let mut schedule = Schedule::default();
for _ in 0..amount {
for _ in 0..(amount / 5) {
schedule.add_systems((empty, empty, empty, empty, empty));
}
schedule.run(&mut world);
group.bench_function(format!("{:03}_systems", 5 * amount), |bencher| {
group.bench_function(format!("{}_systems", amount), |bencher| {
bencher.iter(|| {
schedule.run(&mut world);
});
@ -67,23 +67,21 @@ pub fn busy_systems(criterion: &mut Criterion) {
let mut group = criterion.benchmark_group("busy_systems");
group.warm_up_time(core::time::Duration::from_millis(500));
group.measurement_time(core::time::Duration::from_secs(3));
for entity_bunches in 1..6 {
for entity_bunches in [1, 3, 5] {
world.spawn_batch((0..4 * ENTITY_BUNCH).map(|_| (A(0.0), B(0.0))));
world.spawn_batch((0..4 * ENTITY_BUNCH).map(|_| (A(0.0), B(0.0), C(0.0))));
world.spawn_batch((0..ENTITY_BUNCH).map(|_| (A(0.0), B(0.0), C(0.0), D(0.0))));
world.spawn_batch((0..ENTITY_BUNCH).map(|_| (A(0.0), B(0.0), C(0.0), E(0.0))));
for system_amount in 0..5 {
for system_amount in [3, 9, 15] {
let mut schedule = Schedule::default();
schedule.add_systems((ab, cd, ce));
for _ in 0..system_amount {
for _ in 0..(system_amount / 3) {
schedule.add_systems((ab, cd, ce));
}
schedule.run(&mut world);
group.bench_function(
format!(
"{:02}x_entities_{:02}_systems",
entity_bunches,
3 * system_amount + 3
entity_bunches, system_amount
),
|bencher| {
bencher.iter(|| {
@ -119,22 +117,20 @@ pub fn contrived(criterion: &mut Criterion) {
let mut group = criterion.benchmark_group("contrived");
group.warm_up_time(core::time::Duration::from_millis(500));
group.measurement_time(core::time::Duration::from_secs(3));
for entity_bunches in 1..6 {
for entity_bunches in [1, 3, 5] {
world.spawn_batch((0..ENTITY_BUNCH).map(|_| (A(0.0), B(0.0), C(0.0), D(0.0))));
world.spawn_batch((0..ENTITY_BUNCH).map(|_| (A(0.0), B(0.0))));
world.spawn_batch((0..ENTITY_BUNCH).map(|_| (C(0.0), D(0.0))));
for system_amount in 0..5 {
for system_amount in [3, 9, 15] {
let mut schedule = Schedule::default();
schedule.add_systems((s_0, s_1, s_2));
for _ in 0..system_amount {
for _ in 0..(system_amount / 3) {
schedule.add_systems((s_0, s_1, s_2));
}
schedule.run(&mut world);
group.bench_function(
format!(
"{:02}x_entities_{:02}_systems",
entity_bunches,
3 * system_amount + 3
entity_bunches, system_amount
),
|bencher| {
bencher.iter(|| {

6
benches/benches/bevy_ecs/world/commands.rs
View File

@ -36,7 +36,7 @@ pub fn spawn_commands(criterion: &mut Criterion) {
group.warm_up_time(core::time::Duration::from_millis(500));
group.measurement_time(core::time::Duration::from_secs(4));
for entity_count in (1..5).map(|i| i * 2 * 1000) {
for entity_count in [100, 1_000, 10_000] {
group.bench_function(format!("{}_entities", entity_count), |bencher| {
let mut world = World::default();
let mut command_queue = CommandQueue::default();
@ -136,7 +136,7 @@ pub fn fake_commands(criterion: &mut Criterion) {
group.warm_up_time(core::time::Duration::from_millis(500));
group.measurement_time(core::time::Duration::from_secs(4));
for command_count in (1..5).map(|i| i * 2 * 1000) {
for command_count in [100, 1_000, 10_000] {
group.bench_function(format!("{}_commands", command_count), |bencher| {
let mut world = World::default();
let mut command_queue = CommandQueue::default();
@ -181,7 +181,7 @@ pub fn sized_commands_impl<T: Default + Command>(criterion: &mut Criterion) {
group.warm_up_time(core::time::Duration::from_millis(500));
group.measurement_time(core::time::Duration::from_secs(4));
for command_count in (1..5).map(|i| i * 2 * 1000) {
for command_count in [100, 1_000, 10_000] {
group.bench_function(format!("{}_commands", command_count), |bencher| {
let mut world = World::default();
let mut command_queue = CommandQueue::default();

2
benches/benches/bevy_ecs/world/despawn.rs
View File

@ -12,7 +12,7 @@ pub fn world_despawn(criterion: &mut Criterion) {
group.warm_up_time(core::time::Duration::from_millis(500));
group.measurement_time(core::time::Duration::from_secs(4));
for entity_count in (0..5).map(|i| 10_u32.pow(i)) {
for entity_count in [1, 100, 10_000] {
group.bench_function(format!("{}_entities", entity_count), |bencher| {
bencher.iter_batched_ref(
|| {

2
benches/benches/bevy_ecs/world/despawn_recursive.rs
View File

@ -12,7 +12,7 @@ pub fn world_despawn_recursive(criterion: &mut Criterion) {
group.warm_up_time(core::time::Duration::from_millis(500));
group.measurement_time(core::time::Duration::from_secs(4));
for entity_count in (0..5).map(|i| 10_u32.pow(i)) {
for entity_count in [1, 100, 10_000] {
group.bench_function(format!("{}_entities", entity_count), |bencher| {
bencher.iter_batched_ref(
|| {

2
benches/benches/bevy_ecs/world/spawn.rs
View File

@ -12,7 +12,7 @@ pub fn world_spawn(criterion: &mut Criterion) {
group.warm_up_time(core::time::Duration::from_millis(500));
group.measurement_time(core::time::Duration::from_secs(4));
for entity_count in (0..5).map(|i| 10_u32.pow(i)) {
for entity_count in [1, 100, 10_000] {
group.bench_function(format!("{}_entities", entity_count), |bencher| {
let mut world = World::default();
bencher.iter(|| {