diff --git a/.gitignore b/.gitignore index 0d39edea49..e9a2ae604c 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ Cargo.lock assets/**/*.meta crates/bevy_asset/imported_assets imported_assets +.http-asset-cache # Bevy Examples example_showcase_config.ron diff --git a/Cargo.toml b/Cargo.toml index 0b64a82ed3..bba7c011a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -526,6 +526,15 @@ file_watcher = ["bevy_internal/file_watcher"] # Enables watching in memory asset providers for Bevy Asset hot-reloading embedded_watcher = ["bevy_internal/embedded_watcher"] +# Enables downloading assets from HTTP sources +http = ["bevy_internal/http"] + +# Enables downloading assets from HTTPS sources +https = ["bevy_internal/https"] + +# Enable caching downloaded assets on the filesystem. NOTE: this cache currently never invalidates entries! +http_source_cache = ["bevy_internal/http_source_cache"] + # Enable stepping-based debugging of Bevy systems bevy_debug_stepping = [ "bevy_internal/bevy_debug_stepping", @@ -624,7 +633,7 @@ nonmax = "0.5" smol = "2" smol-macros = "0.1" smol-hyper = "0.1" -ureq = { version = "3.0.8", features = ["json"] } +ureq = { version = "3", features = ["json"] } [target.'cfg(target_arch = "wasm32")'.dev-dependencies] wasm-bindgen = { version = "0.2" } @@ -1913,12 +1922,24 @@ path = "examples/asset/extra_source.rs" doc-scrape-examples = true [package.metadata.example.extra_asset_source] -name = "Extra asset source" +name = "Extra Asset Source" description = "Load an asset from a non-standard asset source" category = "Assets" # Uses non-standard asset path wasm = false +[[example]] +name = "http_source" +path = "examples/asset/http_source.rs" +doc-scrape-examples = true +required-features = ["https"] + +[package.metadata.example.http_source] +name = "HTTP Asset Source" +description = "Load an asset from a http source" +category = "Assets" +wasm = true + [[example]] name = "hot_asset_reloading" path = "examples/asset/hot_asset_reloading.rs" diff --git a/crates/bevy_asset/Cargo.toml b/crates/bevy_asset/Cargo.toml index edf8986130..36aca42343 100644 --- a/crates/bevy_asset/Cargo.toml +++ b/crates/bevy_asset/Cargo.toml @@ -14,6 +14,9 @@ keywords = ["bevy"] file_watcher = ["notify-debouncer-full", "watch", "multi_threaded"] embedded_watcher = ["file_watcher"] multi_threaded = ["bevy_tasks/multi_threaded"] +http = ["ureq"] +https = ["ureq", "ureq/rustls"] +http_source_cache = [] asset_processor = [] watch = [] trace = [] @@ -90,6 +93,8 @@ bevy_reflect = { path = "../bevy_reflect", version = "0.17.0-dev", default-featu [target.'cfg(not(target_arch = "wasm32"))'.dependencies] notify-debouncer-full = { version = "0.5.0", default-features = false, optional = true } +# updating ureq: while ureq is semver stable, it depends on rustls which is not, meaning unlikely but possible breaking changes on minor releases. https://github.com/bevyengine/bevy/pull/16366#issuecomment-2572890794 +ureq = { version = "3", optional = true, default-features = false } [lints] workspace = true diff --git a/crates/bevy_asset/src/http_source.rs b/crates/bevy_asset/src/http_source.rs new file mode 100644 index 0000000000..53f2cd0761 --- /dev/null +++ b/crates/bevy_asset/src/http_source.rs @@ -0,0 +1,271 @@ +use crate::io::{AssetReader, AssetReaderError, Reader}; +use crate::io::{AssetSource, PathStream}; +use crate::AssetApp; +use alloc::boxed::Box; +use bevy_app::{App, Plugin}; +use bevy_tasks::ConditionalSendFuture; +use std::path::{Path, PathBuf}; + +/// Adds the `http` and `https` asset sources to the app. +/// +/// NOTE: Make sure to add this plugin *before* `AssetPlugin` to properly register http asset sources. +/// +/// Any asset path that begins with `http` (when the `http` feature is enabled) or `https` (when the +/// `https` feature is enabled) will be loaded from the web via `fetch`(wasm) or `ureq`(native). +/// +/// By default, `ureq`'s HTTP compression is disabled. To enable gzip and brotli decompression, add +/// the following dependency and features to your Cargo.toml. This will improve bandwidth +/// utilization when its supported by the server. +/// +/// ```toml +/// [target.'cfg(not(target_family = "wasm"))'.dev-dependencies] +/// ureq = { version = "3", default-features = false, features = ["gzip", "brotli"] } +/// ``` +pub struct HttpSourcePlugin; + +impl Plugin for HttpSourcePlugin { + fn build(&self, app: &mut App) { + #[cfg(feature = "http")] + app.register_asset_source( + "http", + AssetSource::build() + .with_reader(|| Box::new(HttpSourceAssetReader::Http)) + .with_processed_reader(|| Box::new(HttpSourceAssetReader::Http)), + ); + + #[cfg(feature = "https")] + app.register_asset_source( + "https", + AssetSource::build() + .with_reader(|| Box::new(HttpSourceAssetReader::Https)) + .with_processed_reader(|| Box::new(HttpSourceAssetReader::Https)), + ); + } +} + +impl Default for HttpSourcePlugin { + fn default() -> Self { + Self + } +} + +/// Asset reader that treats paths as urls to load assets from. +pub enum HttpSourceAssetReader { + /// Unencrypted connections. + Http, + /// Use TLS for setting up connections. + Https, +} + +impl HttpSourceAssetReader { + fn make_uri(&self, path: &Path) -> PathBuf { + PathBuf::from(match self { + Self::Http => "http://", + Self::Https => "https://", + }) + .join(path) + } + + /// See [`crate::io::get_meta_path`] + fn make_meta_uri(&self, path: &Path) -> PathBuf { + let meta_path = crate::io::get_meta_path(path); + self.make_uri(&meta_path) + } +} + +#[cfg(target_arch = "wasm32")] +async fn get<'a>(path: PathBuf) -> Result, AssetReaderError> { + use crate::io::wasm::HttpWasmAssetReader; + + HttpWasmAssetReader::new("") + .fetch_bytes(path) + .await + .map(|r| Box::new(r) as Box) +} + +#[cfg(not(target_arch = "wasm32"))] +async fn get(path: PathBuf) -> Result, AssetReaderError> { + use crate::io::VecReader; + use alloc::{boxed::Box, vec::Vec}; + use bevy_platform::sync::LazyLock; + use std::io::{self, BufReader, Read}; + + let str_path = path.to_str().ok_or_else(|| { + AssetReaderError::Io( + io::Error::other(std::format!("non-utf8 path: {}", path.display())).into(), + ) + })?; + + #[cfg(all(not(target_arch = "wasm32"), feature = "http_source_cache"))] + if let Some(data) = http_asset_cache::try_load_from_cache(str_path).await? { + return Ok(Box::new(VecReader::new(data))); + } + use ureq::Agent; + + static AGENT: LazyLock = LazyLock::new(|| Agent::config_builder().build().new_agent()); + + match AGENT.get(str_path).call() { + Ok(mut response) => { + let mut reader = BufReader::new(response.body_mut().with_config().reader()); + + let mut buffer = Vec::new(); + reader.read_to_end(&mut buffer)?; + + #[cfg(all(not(target_arch = "wasm32"), feature = "http_source_cache"))] + http_asset_cache::save_to_cache(str_path, &buffer).await?; + + Ok(Box::new(VecReader::new(buffer))) + } + // ureq considers all >=400 status codes as errors + Err(ureq::Error::StatusCode(code)) => { + if code == 404 { + Err(AssetReaderError::NotFound(path)) + } else { + Err(AssetReaderError::HttpError(code)) + } + } + Err(err) => Err(AssetReaderError::Io( + io::Error::other(std::format!( + "unexpected error while loading asset {}: {}", + path.display(), + err + )) + .into(), + )), + } +} + +impl AssetReader for HttpSourceAssetReader { + fn read<'a>( + &'a self, + path: &'a Path, + ) -> impl ConditionalSendFuture, AssetReaderError>> { + get(self.make_uri(path)) + } + + async fn read_meta<'a>(&'a self, path: &'a Path) -> Result, AssetReaderError> { + let uri = self.make_meta_uri(path); + get(uri).await + } + + async fn is_directory<'a>(&'a self, _path: &'a Path) -> Result { + Ok(false) + } + + async fn read_directory<'a>( + &'a self, + path: &'a Path, + ) -> Result, AssetReaderError> { + Err(AssetReaderError::NotFound(self.make_uri(path))) + } +} + +/// A naive implementation of an HTTP asset cache that never invalidates. +/// `ureq` currently does not support caching, so this is a simple workaround. +/// It should eventually be replaced by `http-cache` or similar, see [tracking issue](https://github.com/06chaynes/http-cache/issues/91) +#[cfg(all(not(target_arch = "wasm32"), feature = "http_source_cache"))] +mod http_asset_cache { + use alloc::string::String; + use alloc::vec::Vec; + use core::hash::{Hash, Hasher}; + use futures_lite::AsyncWriteExt; + use std::collections::hash_map::DefaultHasher; + use std::io; + use std::path::PathBuf; + + use crate::io::Reader; + + const CACHE_DIR: &str = ".http-asset-cache"; + + fn url_to_hash(url: &str) -> String { + let mut hasher = DefaultHasher::new(); + url.hash(&mut hasher); + std::format!("{:x}", hasher.finish()) + } + + pub async fn try_load_from_cache(url: &str) -> Result>, io::Error> { + let filename = url_to_hash(url); + let cache_path = PathBuf::from(CACHE_DIR).join(&filename); + + if cache_path.exists() { + let mut file = async_fs::File::open(&cache_path).await?; + let mut buffer = Vec::new(); + file.read_to_end(&mut buffer).await?; + Ok(Some(buffer)) + } else { + Ok(None) + } + } + + pub async fn save_to_cache(url: &str, data: &[u8]) -> Result<(), io::Error> { + let filename = url_to_hash(url); + let cache_path = PathBuf::from(CACHE_DIR).join(&filename); + + async_fs::create_dir_all(CACHE_DIR).await.ok(); + + let mut cache_file = async_fs::File::create(&cache_path).await?; + cache_file.write_all(data).await?; + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn make_http_uri() { + assert_eq!( + HttpSourceAssetReader::Http + .make_uri(Path::new("example.com/favicon.png")) + .to_str() + .unwrap(), + "http://example.com/favicon.png" + ); + } + + #[test] + fn make_https_uri() { + assert_eq!( + HttpSourceAssetReader::Https + .make_uri(Path::new("example.com/favicon.png")) + .to_str() + .unwrap(), + "https://example.com/favicon.png" + ); + } + + #[test] + fn make_http_meta_uri() { + assert_eq!( + HttpSourceAssetReader::Http + .make_meta_uri(Path::new("example.com/favicon.png")) + .to_str() + .unwrap(), + "http://example.com/favicon.png.meta" + ); + } + + #[test] + fn make_https_meta_uri() { + assert_eq!( + HttpSourceAssetReader::Https + .make_meta_uri(Path::new("example.com/favicon.png")) + .to_str() + .unwrap(), + "https://example.com/favicon.png.meta" + ); + } + + #[test] + fn make_https_without_extension_meta_uri() { + assert_eq!( + HttpSourceAssetReader::Https + .make_meta_uri(Path::new("example.com/favicon")) + .to_str() + .unwrap(), + "https://example.com/favicon.meta" + ); + } +} diff --git a/crates/bevy_asset/src/io/mod.rs b/crates/bevy_asset/src/io/mod.rs index aa7256bbd9..9fd1c5b2dd 100644 --- a/crates/bevy_asset/src/io/mod.rs +++ b/crates/bevy_asset/src/io/mod.rs @@ -46,7 +46,8 @@ pub enum AssetReaderError { Io(Arc), /// The HTTP request completed but returned an unhandled [HTTP response status code](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status). - /// If the request fails before getting a status code (e.g. request timeout, interrupted connection, etc), expect [`AssetReaderError::Io`]. + /// - If the request returns a 404 error, expect [`AssetReaderError::NotFound`]. + /// - If the request fails before getting a status code (e.g. request timeout, interrupted connection, etc), expect [`AssetReaderError::Io`]. #[error("Encountered HTTP status {0:?} when loading asset")] HttpError(u16), } @@ -762,11 +763,16 @@ impl Reader for SliceReader<'_> { } } -/// Appends `.meta` to the given path. +/// Appends `.meta` to the given path: +/// - `foo` becomes `foo.meta` +/// - `foo.bar` becomes `foo.bar.meta` pub(crate) fn get_meta_path(path: &Path) -> PathBuf { let mut meta_path = path.to_path_buf(); let mut extension = path.extension().unwrap_or_default().to_os_string(); - extension.push(".meta"); + if !extension.is_empty() { + extension.push("."); + } + extension.push("meta"); meta_path.set_extension(extension); meta_path } @@ -783,3 +789,24 @@ impl Stream for EmptyPathStream { Poll::Ready(None) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn get_meta_path_no_extension() { + assert_eq!( + get_meta_path(Path::new("foo")).to_str().unwrap(), + "foo.meta" + ); + } + + #[test] + fn get_meta_path_with_extension() { + assert_eq!( + get_meta_path(Path::new("foo.bar")).to_str().unwrap(), + "foo.bar.meta" + ); + } +} diff --git a/crates/bevy_asset/src/io/wasm.rs b/crates/bevy_asset/src/io/wasm.rs index 4ed7162d2b..165f52c9b8 100644 --- a/crates/bevy_asset/src/io/wasm.rs +++ b/crates/bevy_asset/src/io/wasm.rs @@ -52,7 +52,8 @@ fn js_value_to_err(context: &str) -> impl FnOnce(JsValue) -> std::io::Error + '_ } impl HttpWasmAssetReader { - async fn fetch_bytes(&self, path: PathBuf) -> Result { + // Also used by [`HttpSourceAssetReader`](crate::HttpSourceAssetReader) + pub(crate) async fn fetch_bytes(&self, path: PathBuf) -> Result { // The JS global scope includes a self-reference via a specializing name, which can be used to determine the type of global context available. let global: Global = js_sys::global().unchecked_into(); let promise = if !global.window().is_undefined() { diff --git a/crates/bevy_asset/src/lib.rs b/crates/bevy_asset/src/lib.rs index 8186b6315d..d7907d6341 100644 --- a/crates/bevy_asset/src/lib.rs +++ b/crates/bevy_asset/src/lib.rs @@ -186,6 +186,9 @@ mod reflect; mod render_asset; mod server; +#[cfg(any(feature = "http", feature = "https"))] +pub mod http_source; + pub use assets::*; pub use bevy_asset_macros::Asset; pub use direct_access_ext::DirectAssetAccessExt; diff --git a/crates/bevy_internal/Cargo.toml b/crates/bevy_internal/Cargo.toml index e591803751..d409d18294 100644 --- a/crates/bevy_internal/Cargo.toml +++ b/crates/bevy_internal/Cargo.toml @@ -232,6 +232,15 @@ debug_glam_assert = ["bevy_math/debug_glam_assert"] default_font = ["bevy_text?/default_font"] +# Enables downloading assets from HTTP sources +http = ["bevy_asset?/http"] + +# Enables downloading assets from HTTPS sources +https = ["bevy_asset?/https"] + +# Enable caching downloaded assets on the filesystem. NOTE: this cache currently never invalidates entries! +http_source_cache = ["bevy_asset?/http_source_cache"] + # Enables the built-in asset processor for processed assets. asset_processor = ["bevy_asset?/asset_processor"] diff --git a/crates/bevy_internal/src/default_plugins.rs b/crates/bevy_internal/src/default_plugins.rs index cdb59921dc..807915cd1d 100644 --- a/crates/bevy_internal/src/default_plugins.rs +++ b/crates/bevy_internal/src/default_plugins.rs @@ -21,6 +21,10 @@ plugin_group! { #[cfg(feature = "std")] #[custom(cfg(any(all(unix, not(target_os = "horizon")), windows)))] bevy_app:::TerminalCtrlCHandlerPlugin, + // NOTE: Load this before AssetPlugin to properly register http asset sources. + #[cfg(feature = "bevy_asset")] + #[custom(cfg(any(feature = "http", feature = "https")))] + bevy_asset::http_source:::HttpSourcePlugin, #[cfg(feature = "bevy_asset")] bevy_asset:::AssetPlugin, #[cfg(feature = "bevy_scene")] diff --git a/deny.toml b/deny.toml index d22efdf153..a203dfaaa8 100644 --- a/deny.toml +++ b/deny.toml @@ -23,6 +23,7 @@ allow = [ "BSD-3-Clause", "BSL-1.0", "CC0-1.0", + "CDLA-Permissive-2.0", "ISC", "MIT", "MIT-0", diff --git a/docs/cargo_features.md b/docs/cargo_features.md index 120c461efe..20d6f195d9 100644 --- a/docs/cargo_features.md +++ b/docs/cargo_features.md @@ -93,6 +93,9 @@ The default feature set enables most of the expected features of a game engine, |glam_assert|Enable assertions to check the validity of parameters passed to glam| |gltf_convert_coordinates_default|Enable converting glTF coordinates to Bevy's coordinate system by default. This will be Bevy's default behavior starting in 0.18.| |hotpatching|Enable hotpatching of Bevy systems| +|http|Enables downloading assets from HTTP sources| +|http_source_cache|Enable caching downloaded assets on the filesystem. NOTE: this cache currently never invalidates entries!| +|https|Enables downloading assets from HTTPS sources| |ico|ICO image format support| |jpeg|JPEG image format support| |libm|Uses the `libm` maths library instead of the one provided in `std` and `core`.| diff --git a/examples/README.md b/examples/README.md index 29420e66c5..d19d86694e 100644 --- a/examples/README.md +++ b/examples/README.md @@ -256,7 +256,8 @@ Example | Description [Custom Asset](../examples/asset/custom_asset.rs) | Implements a custom asset loader [Custom Asset IO](../examples/asset/custom_asset_reader.rs) | Implements a custom AssetReader [Embedded Asset](../examples/asset/embedded_asset.rs) | Embed an asset in the application binary and load it -[Extra asset source](../examples/asset/extra_source.rs) | Load an asset from a non-standard asset source +[Extra Asset Source](../examples/asset/extra_source.rs) | Load an asset from a non-standard asset source +[HTTP Asset Source](../examples/asset/http_source.rs) | Load an asset from a http source [Hot Reloading of Assets](../examples/asset/hot_asset_reloading.rs) | Demonstrates automatic reloading of assets when modified on disk [Multi-asset synchronization](../examples/asset/multi_asset_sync.rs) | Demonstrates how to wait for multiple assets to be loaded. [Repeated texture configuration](../examples/asset/repeated_texture.rs) | How to configure the texture to repeat instead of the default clamp to edges diff --git a/examples/asset/http_source.rs b/examples/asset/http_source.rs new file mode 100644 index 0000000000..d1f6cf28a6 --- /dev/null +++ b/examples/asset/http_source.rs @@ -0,0 +1,20 @@ +//! Example usage of the `https` asset source to load assets from the web. +//! +//! Run with the feature `https`, and optionally `http_source_cache` +//! for a simple caching mechanism that never invalidates. +//! +use bevy::prelude::*; + +fn main() { + App::new() + .add_plugins(DefaultPlugins) + .add_systems(Startup, setup) + .run(); +} + +fn setup(mut commands: Commands, asset_server: Res) { + commands.spawn(Camera2d); + let url = "https://raw.githubusercontent.com/bevyengine/bevy/refs/heads/main/assets/branding/bevy_bird_dark.png"; + // Simply use a url where you would normally use an asset folder relative path + commands.spawn(Sprite::from_image(asset_server.load(url))); +} diff --git a/release-content/release-notes/http_source.md b/release-content/release-notes/http_source.md new file mode 100644 index 0000000000..faed8fae19 --- /dev/null +++ b/release-content/release-notes/http_source.md @@ -0,0 +1,19 @@ +--- +title: HTTP Assets +authors: ["@johanhelsing", "@mrchantey", "@jf908"] +pull_requests: [17889] +--- + +Bevy now supports downloading assets over http and https. +Use the new `http` and `https` features to enable `http://` and `https://` URLs as asset paths. +This functionality is powered by the [`ureq`](https://github.com/algesten/ureq) crate on native platforms and the fetch API on wasm. + +```rust +let image = asset_server.load("https://example.com/image.png"); +commands.spawn(Sprite::from_image(image)); +``` + +By default these assets aren’t saved anywhere but you can enable the `http_source` feature to cache assets on your file system. + +The implementation has changed quite a bit but this feature originally started out as an upstreaming of the [`bevy_web_asset`](https://github.com/johanhelsing/bevy_web_asset) crate. +Special thanks to @johanhelsing and bevy_web_asset's contributors!