diff --git a/.cargo/config.toml b/.cargo/config.toml index a20e20d2971e..95a1cb5189af 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -73,4 +73,4 @@ rustflags = [ linker = "arm-linux-gnueabihf-gcc" [target.wasm32-unknown-unknown] -rustflags = ["-Zshare-generics=y", "--cfg", 'getrandom_backend="wasm_js"'] +rustflags = ["-Zshare-generics=y", "-Aunused", "--cfg", 'getrandom_backend="wasm_js"'] diff --git a/Cargo.lock b/Cargo.lock index a06447a33961..9e85e1f0bea0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1318,7 +1318,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" dependencies = [ "lazy_static", - "windows-sys 0.59.0", + "windows-sys 0.48.0", ] [[package]] @@ -1469,6 +1469,15 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "convert_case" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb402b8d4c85569410425650ce3eddc7d698ed96d39a73f941b08fb63082f1e7" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "cooked-waker" version = "5.0.0" @@ -1713,15 +1722,17 @@ dependencies = [ [[package]] name = "crossterm" -version = "0.26.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a84cda67535339806297f1b331d6dd6320470d2a0fe65381e79ee9e156dd3d13" +checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.1", "crossterm_winapi", - "libc", - "mio 0.8.11", + "derive_more 2.0.1", + "document-features", + "mio 1.0.3", "parking_lot", + "rustix 1.0.7", "signal-hook", "signal-hook-mio", "winapi", @@ -2077,6 +2088,7 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" dependencies = [ + "convert_case 0.7.1", "proc-macro2", "quote", "syn 2.0.104", @@ -2134,9 +2146,9 @@ dependencies = [ [[package]] name = "document-features" -version = "0.2.10" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb6969eaabd2421f8a2775cfd2471a2b634372b4a25d41e3bd647b79912850a0" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" dependencies = [ "litrs", ] @@ -3480,6 +3492,17 @@ dependencies = [ "rustversion", ] +[[package]] +name = "io-uring" +version = "0.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd7bddefd0a8833b88a4b68f90dae22c7450d11b354198baee3874fd811b344" +dependencies = [ + "bitflags 2.9.1", + "cfg-if", + "libc", +] + [[package]] name = "ipnet" version = "2.10.1" @@ -3636,9 +3659,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.77" +version = "0.3.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" dependencies = [ "once_cell", "wasm-bindgen", @@ -3868,7 +3891,7 @@ version = "1.0.0-alpha.43" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84c12744d1279367caed41739ef094c325d53fb0ffcd4f9b84a368796f870252" dependencies = [ - "convert_case", + "convert_case 0.6.0", "proc-macro2", "quote", "syn 1.0.109", @@ -3932,9 +3955,9 @@ checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" [[package]] name = "litrs" -version = "0.4.1" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" [[package]] name = "lmdb-rkv" @@ -4289,7 +4312,7 @@ version = "0.110.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "733b723f4e2a9c561b611aed657e6c3938941124fa1cb83b6a3b8e61511ff3c4" dependencies = [ - "convert_case", + "convert_case 0.6.0", "handlebars", "once_cell", "regex", @@ -4367,7 +4390,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cbe2585d8ac223f7d34f13701434b9d5f4eb9c332cccce8dee57ea18ab8ab0c" dependencies = [ "cfg-if", - "convert_case", + "convert_case 0.6.0", "napi-derive-backend", "proc-macro2", "quote", @@ -4380,7 +4403,7 @@ version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1639aaa9eeb76e91c6ae66da8ce3e89e921cd3885e99ec85f4abacae72fc91bf" dependencies = [ - "convert_case", + "convert_case 0.6.0", "once_cell", "proc-macro2", "quote", @@ -5318,9 +5341,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.13" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -6927,12 +6950,13 @@ dependencies = [ [[package]] name = "signal-hook-mio" -version = "0.2.3" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" dependencies = [ "libc", "mio 0.8.11", + "mio 1.0.3", "signal-hook", ] @@ -7055,6 +7079,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "socket2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + [[package]] name = "spin" version = "0.9.8" @@ -8918,28 +8952,50 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.43.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" +version = "1.47.1" +source = "git+https://github.com/utooland/tokio.git#0b9f7720e2b65634e0fb7d7d8be9073fc21d96a1" dependencies = [ "backtrace", "bytes", + "io-uring", "libc", "mio 1.0.3", "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.10", + "slab", + "socket2 0.6.1", "tokio-macros", "tracing", - "windows-sys 0.52.0", + "wasm_thread", + "wasmtimer", + "windows-sys 0.59.0", +] + +[[package]] +name = "tokio-fs-ext" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db902ff0804cc7965a81fec91151b9c27ed3a489c217a5dc7a885064fb0c1571" +dependencies = [ + "bitflags 2.9.1", + "cfg-if", + "futures", + "js-sys", + "notify-types", + "pin-project-lite", + "rustc-hash 2.1.1", + "tokio", + "tokio-stream", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", ] [[package]] name = "tokio-macros" version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +source = "git+https://github.com/utooland/tokio.git#0b9f7720e2b65634e0fb7d7d8be9073fc21d96a1" dependencies = [ "proc-macro2", "quote", @@ -8992,15 +9048,13 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" +version = "0.7.16" +source = "git+https://github.com/utooland/tokio.git#0b9f7720e2b65634e0fb7d7d8be9073fc21d96a1" dependencies = [ "bytes", "futures-core", "futures-sink", "futures-util", - "hashbrown 0.14.5", "pin-project-lite", "tokio", ] @@ -9686,6 +9740,7 @@ dependencies = [ "jsonc-parser", "mime", "notify", + "notify-types", "parking_lot", "rand 0.10.0", "regex", @@ -9698,6 +9753,7 @@ dependencies = [ "tempfile", "thiserror 1.0.69", "tokio", + "tokio-fs-ext", "tracing", "triomphe 0.1.12", "turbo-bincode", @@ -9731,6 +9787,7 @@ version = "0.1.0" dependencies = [ "bincode 2.0.1", "data-encoding", + "getrandom 0.3.3", "sha2", "turbo-tasks-macros", "twox-hash 2.1.2", @@ -9804,7 +9861,6 @@ dependencies = [ "serde", "serde_json", "smallvec", - "tokio", "tracing", "turbo-esregex", "turbo-rcstr", @@ -9877,6 +9933,8 @@ dependencies = [ "bincode 2.0.1", "either", "indoc", + "qstring", + "regex", "serde", "serde_json", "serde_qs", @@ -9887,6 +9945,7 @@ dependencies = [ "turbo-tasks-fs", "turbo-tasks-hash", "turbopack-core", + "turbopack-css", "turbopack-ecmascript", "turbopack-ecmascript-runtime", "turbopack-resolve", @@ -9942,7 +10001,8 @@ name = "turbopack-cli-utils" version = "0.1.0" dependencies = [ "anyhow", - "crossterm 0.26.1", + "clap", + "crossterm 0.29.0", "owo-colors", "rustc-hash 2.1.1", "serde", @@ -10260,12 +10320,14 @@ dependencies = [ "async-trait", "base64 0.21.4", "bincode 2.0.1", + "bytes", "const_format", "dashmap 6.1.0", "either", "futures", "futures-retry", "indoc", + "js-sys", "napi", "napi-derive", "once_cell", @@ -10290,6 +10352,8 @@ dependencies = [ "turbopack-core", "turbopack-ecmascript", "turbopack-resolve", + "wasm-bindgen", + "wasm-bindgen-futures", ] [[package]] @@ -11020,37 +11084,26 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.100" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" dependencies = [ "cfg-if", "once_cell", "rustversion", "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" -dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn 2.0.104", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.50" +version = "0.4.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" dependencies = [ "cfg-if", + "futures-core", + "futures-util", "js-sys", "once_cell", "wasm-bindgen", @@ -11059,9 +11112,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.100" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -11069,22 +11122,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.100" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" dependencies = [ + "bumpalo", "proc-macro2", "quote", "syn 2.0.104", - "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.100" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" dependencies = [ "unicode-ident", ] @@ -11121,6 +11174,17 @@ dependencies = [ "wasmparser 0.244.0", ] +[[package]] +name = "wasm_thread" +version = "0.3.3" +source = "git+https://github.com/utooland/wasm_thread.git#fd53c2f5a20e9f899d9216dee045472f50d97219" +dependencies = [ + "futures", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasmer" version = "6.1.0-rc.3" @@ -11480,6 +11544,20 @@ dependencies = [ "semver", ] +[[package]] +name = "wasmtimer" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c598d6b99ea013e35844697fc4670d08339d5cda15588f193c6beedd12f644b" +dependencies = [ + "futures", + "js-sys", + "parking_lot", + "pin-utils", + "slab", + "wasm-bindgen", +] + [[package]] name = "wast" version = "235.0.0" @@ -11504,9 +11582,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.77" +version = "0.3.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index 526990132a26..13237f81a953 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -428,6 +428,7 @@ napi-derive = "2" napi-build = "2" nohash-hasher = "0.2.0" notify = "8.1.0" +notify-types = "2.0.0" once_cell = "1.21.3" owo-colors = "4.2.2" parcel_selectors = "0.28.2" @@ -441,6 +442,7 @@ qstring = "0.7.2" quick_cache = { version = "0.6.14" } quote = "1.0.23" rand = "0.10.0" +getrandom = "0.3" rayon = "1.10.0" regex = "1.10.6" regress = "0.10.4" @@ -470,8 +472,8 @@ syn = "2.0.100" tempfile = "3.20.0" thiserror = "1.0.48" thread_local = "1.1.8" -tokio = "1.43.0" -tokio-util = { version = "0.7.13", features = ["io", "rt"] } +tokio = "1.47.1" +tokio-util = { version = "0.7.16", features = ["io", "rt"] } tracing = "0.1.37" tracing-subscriber = "0.3.16" triomphe = { git = "https://github.com/sokra/triomphe", branch = "sokra/unstable" } @@ -492,3 +494,5 @@ virtue = { git = "https://github.com/bgw/virtue.git", branch = "bgw/fix-generic- mdxjs = { git = "https://github.com/vercel-labs/mdxjs-rs-turbopack.git", branch = "turbopack" } # Doesn't compile on recent nightly versions because of https://github.com/Michael-F-Bryan/include_dir/pull/117 include_dir = { git = "https://github.com/vercel-labs/include_dir", branch = "turbopack" } +tokio = { git = "https://github.com/utooland/tokio.git", package = "tokio" } +tokio-util = { git = "https://github.com/utooland/tokio.git", package = "tokio-util" } diff --git a/crates/next-core/src/transform_options.rs b/crates/next-core/src/transform_options.rs index 22468bdd74d9..b4a1fffa3902 100644 --- a/crates/next-core/src/transform_options.rs +++ b/crates/next-core/src/transform_options.rs @@ -54,7 +54,7 @@ pub async fn get_typescript_transform_options( ) -> Result> { let tsconfig = get_typescript_options(project_path, tsconfig_path).await?; - let use_define_for_class_fields = if let Some(tsconfig) = tsconfig { + let use_define_for_class_fields = if let Some(ref tsconfig) = tsconfig { read_from_tsconfigs(&tsconfig, |json, _| { json["compilerOptions"]["useDefineForClassFields"].as_bool() }) @@ -63,9 +63,19 @@ pub async fn get_typescript_transform_options( } else { false }; + let verbatim_module_syntax = if let Some(ref tsconfig) = tsconfig { + read_from_tsconfigs(tsconfig, |json, _| { + json["compilerOptions"]["verbatimModuleSyntax"].as_bool() + }) + .await? + .unwrap_or(false) + } else { + false + }; let ts_transform_options = TypescriptTransformOptions { use_define_for_class_fields, + verbatim_module_syntax, }; Ok(ts_transform_options.cell()) diff --git a/packages/next/src/build/swc/index.ts b/packages/next/src/build/swc/index.ts index ba7900daf8f0..e5e996f9e039 100644 --- a/packages/next/src/build/swc/index.ts +++ b/packages/next/src/build/swc/index.ts @@ -41,6 +41,7 @@ import type { } from './types' import { runLoaderWorkerPool } from './loaderWorkerPool' import { throwTurbopackInternalError } from '../../shared/lib/turbopack/internal-error' +import { runLoaderWorkerPool } from './loaderWorkerPool' export enum HmrTarget { Client = 'client', diff --git a/turbopack/crates/turbo-tasks-backend/Cargo.toml b/turbopack/crates/turbo-tasks-backend/Cargo.toml index aa5f4d177f46..363bd4df57a7 100644 --- a/turbopack/crates/turbo-tasks-backend/Cargo.toml +++ b/turbopack/crates/turbo-tasks-backend/Cargo.toml @@ -39,7 +39,7 @@ arc-swap = { version = "1.8.2" } auto-hash-map = { workspace = true } bitfield = { workspace = true } byteorder = { workspace = true } -dashmap = { workspace = true, features = ["raw-api"]} +dashmap = { workspace = true, features = ["raw-api"] } either = { workspace = true } hashbrown = { workspace = true, features = ["raw"] } indexmap = { workspace = true } @@ -58,11 +58,13 @@ tokio = { workspace = true } tracing = { workspace = true } thread_local = { workspace = true } turbo-bincode = { workspace = true } -turbo-persistence = { workspace = true } turbo-rcstr = { workspace = true } turbo-tasks = { workspace = true } turbo-tasks-hash = { workspace = true } +[target.'cfg(not(all(target_family = "wasm", target_os = "unknown")))'.dependencies] +turbo-persistence = { workspace = true } + [dev-dependencies] criterion = { workspace = true, features = ["async_tokio"] } futures = { workspace = true } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs index b47c453ef0c4..b6d38a3d2971 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs @@ -7,7 +7,7 @@ use std::{ ops::{ControlFlow, Deref}, sync::Arc, thread::yield_now, - time::{Duration, Instant}, + time::Duration, }; use anyhow::Result; @@ -16,6 +16,7 @@ use indexmap::map::Entry; use ringmap::RingSet; use rustc_hash::{FxBuildHasher, FxHashMap}; use smallvec::{SmallVec, smallvec}; +use tokio::time::Instant; #[cfg(feature = "trace_aggregation_update_queue")] use tracing::span::Span; #[cfg(any( diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs index 851a7029a837..d2e0eb9871a9 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs @@ -233,6 +233,26 @@ where StorageWriteGuard<'e>, ), ) { + // Fast path: no backing storage (e.g. WASM/NoopKvDb) — process inline, + // skip the Vec allocation needed for batch DB restoration. + #[cfg(all(target_family = "wasm", target_os = "unknown"))] + if !self.backend.should_restore() { + for (id, category) in task_ids { + let mut task = self.backend.storage.access_mut(id); + if !task.flags.is_restored(category) { + task.flags.set_restored(category); + } + if id.is_transient() { + if call_prepared_task_callback_for_transient_tasks { + prepared_task_callback(self, id, category, task); + } + } else { + prepared_task_callback(self, id, category, task); + } + } + return; + } + let mut data_count = 0; let mut meta_count = 0; let mut all_count = 0; diff --git a/turbopack/crates/turbo-tasks-backend/src/database/key_value_database.rs b/turbopack/crates/turbo-tasks-backend/src/database/key_value_database.rs index ab0200cf6801..b203a303c433 100644 --- a/turbopack/crates/turbo-tasks-backend/src/database/key_value_database.rs +++ b/turbopack/crates/turbo-tasks-backend/src/database/key_value_database.rs @@ -1,5 +1,6 @@ use anyhow::Result; use smallvec::SmallVec; +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] use turbo_persistence::{FamilyConfig, FamilyKind}; use crate::database::write_batch::{ @@ -13,6 +14,8 @@ pub enum KeySpace { TaskData = 2, TaskCache = 3, } + +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] impl KeySpace { /// Returns the persistence configuration for this keyspace. pub const fn family_config(&self) -> FamilyConfig { diff --git a/turbopack/crates/turbo-tasks-backend/src/database/mod.rs b/turbopack/crates/turbo-tasks-backend/src/database/mod.rs index 5b8ec3c7c295..012a48a0c089 100644 --- a/turbopack/crates/turbo-tasks-backend/src/database/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/database/mod.rs @@ -12,5 +12,6 @@ pub mod noop_kv; pub mod read_transaction_cache; #[cfg(feature = "lmdb")] pub mod startup_cache; +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] pub mod turbo; pub mod write_batch; diff --git a/turbopack/crates/turbo-tasks-backend/src/lib.rs b/turbopack/crates/turbo-tasks-backend/src/lib.rs index 974c91da16b9..3250f7e054e0 100644 --- a/turbopack/crates/turbo-tasks-backend/src/lib.rs +++ b/turbopack/crates/turbo-tasks-backend/src/lib.rs @@ -15,7 +15,9 @@ use std::path::Path; use anyhow::Result; -use crate::database::{noop_kv::NoopKvDb, turbo::TurboKeyValueDatabase}; +use crate::database::noop_kv::NoopKvDb; +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] +use crate::database::turbo::TurboKeyValueDatabase; pub use crate::{ backend::{BackendOptions, StorageMode, TurboTasksBackend}, backing_storage::BackingStorage, @@ -71,6 +73,7 @@ pub fn lmdb_backing_storage( ) } +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] pub type TurboBackingStorage = KeyValueDatabaseBackingStorage; /// Creates a `BackingStorage` to be passed to [`TurboTasksBackend::new`]. @@ -79,6 +82,7 @@ pub type TurboBackingStorage = KeyValueDatabaseBackingStorage NoopBackingStorage { #[cfg(feature = "lmdb")] pub type DefaultBackingStorage = LmdbBackingStorage; +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] #[cfg(not(feature = "lmdb"))] pub type DefaultBackingStorage = TurboBackingStorage; /// Calls [`turbo_backing_storage`] (recommended) or `lmdb_backing_storage`, depending on if the /// `lmdb` cargo feature is enabled. +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] pub fn default_backing_storage( path: &Path, version_info: &GitVersionInfo, diff --git a/turbopack/crates/turbo-tasks-fs/Cargo.toml b/turbopack/crates/turbo-tasks-fs/Cargo.toml index dc01622369f3..da69fa381718 100644 --- a/turbopack/crates/turbo-tasks-fs/Cargo.toml +++ b/turbopack/crates/turbo-tasks-fs/Cargo.toml @@ -36,7 +36,6 @@ include_dir = { version = "0.7.3", features = ["nightly"] } indexmap = { workspace = true } jsonc-parser = { version = "0.26.3", features = ["serde"] } mime = { workspace = true } -notify = { workspace = true } parking_lot = { workspace = true } regex = { workspace = true } rustc-hash = { workspace = true } @@ -53,6 +52,12 @@ turbo-tasks = { workspace = true } turbo-tasks-hash = { workspace = true } turbo-unix-path = { workspace = true } urlencoding = { workspace = true } +notify-types = { workspace = true } + +notify = { workspace = true } + +[target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies] +tokio-fs-ext = { version = "0.7.8", features = ["opfs_offload", "opfs_watch"] } [dev-dependencies] criterion = { workspace = true, features = ["async_tokio"] } @@ -62,4 +67,3 @@ sha2 = { workspace = true } tempfile = { workspace = true } turbo-tasks-testing = { workspace = true } turbo-tasks-backend = { workspace = true } - diff --git a/turbopack/crates/turbo-tasks-fs/src/embed/dir.rs b/turbopack/crates/turbo-tasks-fs/src/embed/dir.rs index 726e6a3bec5d..71177ac8860b 100644 --- a/turbopack/crates/turbo-tasks-fs/src/embed/dir.rs +++ b/turbopack/crates/turbo-tasks-fs/src/embed/dir.rs @@ -13,6 +13,7 @@ pub async fn directory_from_relative_path( path: RcStr, ) -> Result>> { let disk_fs = DiskFileSystem::new(name, path); + #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] disk_fs.await?.start_watching(None).await?; Ok(Vc::upcast(disk_fs)) diff --git a/turbopack/crates/turbo-tasks-fs/src/embed/file.rs b/turbopack/crates/turbo-tasks-fs/src/embed/file.rs index 6cbc30eaac16..b85e936c38f0 100644 --- a/turbopack/crates/turbo-tasks-fs/src/embed/file.rs +++ b/turbopack/crates/turbo-tasks-fs/src/embed/file.rs @@ -23,6 +23,7 @@ pub async fn content_from_relative_path( root_path.to_string_lossy().into(), root_path.to_string_lossy().into(), ); + #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] disk_fs.await?.start_watching(None).await?; let fs_path = disk_fs.root().await?.join(path)?; diff --git a/turbopack/crates/turbo-tasks-fs/src/lib.rs b/turbopack/crates/turbo-tasks-fs/src/lib.rs index ee602f8c1e25..53225d40badb 100644 --- a/turbopack/crates/turbo-tasks-fs/src/lib.rs +++ b/turbopack/crates/turbo-tasks-fs/src/lib.rs @@ -46,6 +46,9 @@ use std::{ time::Duration, }; +#[cfg(all(target_family = "wasm", target_os = "unknown"))] +pub mod wasm_fs_offload; + use anyhow::{Context, Result, anyhow, bail}; use auto_hash_map::{AutoMap, AutoSet}; use bincode::{Decode, Encode}; @@ -537,14 +540,24 @@ impl DiskFileSystemInner { let root_path = self.root_path().to_path_buf(); // create the directory for the filesystem on disk, if it doesn't exist + #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] retry_blocking(|| std::fs::create_dir_all(&root_path)) .instrument(tracing::info_span!("create root directory", name = ?root_path)) .concurrency_limited(&self.write_semaphore) .await?; + #[cfg(all(target_family = "wasm", target_os = "unknown"))] + wasm_fs_offload::CLIENT.create_dir_all(root_path).await?; + + #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] self.watcher .start_watching(self.clone(), report_invalidation_reason, poll_interval)?; + #[cfg(all(target_family = "wasm", target_os = "unknown"))] + self.watcher + .start_watching(self.clone(), report_invalidation_reason, poll_interval) + .await?; + Ok(()) } @@ -554,10 +567,15 @@ impl DiskFileSystemInner { |fs_context| fs_context.created_directories.contains(directory), ); if !already_created { + #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] retry_blocking(|| std::fs::create_dir_all(directory)) .instrument(tracing::info_span!("create directory", name = ?directory)) .concurrency_limited(&self.write_semaphore) .await?; + + #[cfg(all(target_family = "wasm", target_os = "unknown"))] + wasm_fs_offload::CLIENT.create_dir_all(directory).await?; + ApplyEffectsContext::with(|fs_context: &mut DiskFileSystemApplyContext| { fs_context .created_directories @@ -619,6 +637,7 @@ impl DiskFileSystem { .await } + #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] pub fn stop_watching(&self) { self.inner.watcher.stop_watching(); } @@ -708,7 +727,7 @@ impl DiskFileSystem { /// * `root` - Path to the given filesystem's root. Should be /// [canonicalized][std::fs::canonicalize]. pub fn new(name: RcStr, root: RcStr) -> Vc { - Self::new_internal(name, root, Vec::new()) + Self::new_internal(name, root, Vec::new(), Vec::new()) } /// Create a new instance of `DiskFileSystem`. @@ -727,14 +746,49 @@ impl DiskFileSystem { "denied_path must be normalized: {denied_path:?}" ); } - Self::new_internal(name, root, denied_paths) + Self::new_internal(name, root, denied_paths, Vec::new()) + } + + /// Create a new instance of `DiskFileSystem` with watch ignore patterns. + /// # Arguments + /// + /// * `name` - Name of the filesystem. + /// * `root` - Path to the given filesystem's root. Should be + /// [canonicalized][std::fs::canonicalize]. + /// * `denied_paths` - Paths within this filesystem that are not allowed to be accessed. + /// * `watched_ignored` - Paths that should be ignored when watching for file changes. + pub fn new_with_denied_paths_and_watched_ignored( + name: RcStr, + root: RcStr, + denied_paths: Vec, + watched_ignored: Vec, + ) -> Vc { + for denied_path in &denied_paths { + debug_assert!(!denied_path.is_empty(), "denied_path must not be empty"); + debug_assert!( + normalize_path(denied_path).as_deref() == Some(&**denied_path), + "denied_path must be normalized: {denied_path:?}" + ); + } + Self::new_internal(name, root, denied_paths, watched_ignored) } } #[turbo_tasks::value_impl] impl DiskFileSystem { #[turbo_tasks::function] - fn new_internal(name: RcStr, root: RcStr, denied_paths: Vec) -> Vc { + fn new_internal( + name: RcStr, + root: RcStr, + denied_paths: Vec, + watched_ignored: Vec, + ) -> Vc { + let watcher = if watched_ignored.is_empty() { + DiskWatcher::new() + } else { + DiskWatcher::new_with_ignored_paths(watched_ignored) + }; + let instance = DiskFileSystem { inner: Arc::new(DiskFileSystemInner { name, @@ -745,7 +799,7 @@ impl DiskFileSystem { dir_invalidator_map: InvalidatorMap::new(), read_semaphore: create_read_semaphore(), write_semaphore: create_write_semaphore(), - watcher: DiskWatcher::new(), + watcher, denied_paths, turbo_tasks: turbo_tasks_weak(), tokio_handle: Handle::current(), @@ -777,6 +831,7 @@ impl FileSystem for DiskFileSystem { self.inner.register_read_invalidator(&full_path)?; let _lock = self.inner.lock_path(&full_path).await; + #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] let content = match retry_blocking(|| File::from_path(&full_path)) .instrument(tracing::info_span!("read file", name = ?full_path)) .concurrency_limited(&self.inner.read_semaphore) @@ -788,6 +843,15 @@ impl FileSystem for DiskFileSystem { } Err(e) => return Err(anyhow!(e).context(format!("reading file {full_path:?}"))), }; + + #[cfg(all(target_family = "wasm", target_os = "unknown"))] + let content = match wasm_fs_offload::CLIENT.read(&full_path).await { + Ok(buf) => Ok(FileContent::from(File::from_bytes(buf))), + Err(e) if e.kind() == ErrorKind::NotFound || e.kind() == ErrorKind::InvalidFilename => { + Ok(FileContent::NotFound) + } + Err(e) => Err(anyhow!(e).context(format!("reading file {}", full_path.display()))), + }?; Ok(content.cell()) } @@ -804,6 +868,7 @@ impl FileSystem for DiskFileSystem { self.inner.register_dir_invalidator(&full_path)?; // we use the sync std function here as it's a lot faster (600%) in node-file-trace + #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] let read_dir = match retry_blocking(|| std::fs::read_dir(&full_path)) .instrument(tracing::info_span!("read directory", name = ?full_path)) .concurrency_limited(&self.inner.read_semaphore) @@ -821,6 +886,21 @@ impl FileSystem for DiskFileSystem { return Err(anyhow!(e).context(format!("reading dir {full_path:?}"))); } }; + + #[cfg(all(target_family = "wasm", target_os = "unknown"))] + let read_dir = match wasm_fs_offload::CLIENT.read_dir(&full_path).await { + Ok(dir) => dir, + Err(e) + if e.kind() == ErrorKind::NotFound + || e.kind() == ErrorKind::NotADirectory + || e.kind() == ErrorKind::InvalidFilename => + { + return Ok(RawDirectoryContent::not_found()); + } + Err(e) => { + bail!(anyhow!(e).context(format!("reading dir {}", full_path.display()))) + } + }; let dir_path = fs_path.path.as_str(); let denied_entries: FxHashSet<&str> = self .inner @@ -893,6 +973,7 @@ impl FileSystem for DiskFileSystem { self.inner.register_read_invalidator(&full_path)?; let _lock = self.inner.lock_path(&full_path).await; + #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] let link_path = match retry_blocking(|| std::fs::read_link(&full_path)) .instrument(tracing::info_span!("read symlink", name = ?full_path)) .concurrency_limited(&self.inner.read_semaphore) @@ -901,6 +982,13 @@ impl FileSystem for DiskFileSystem { Ok(res) => res, Err(_) => return Ok(LinkContent::NotFound.cell()), }; + + #[cfg(all(target_family = "wasm", target_os = "unknown"))] + let link_path: PathBuf = { + // TODO: not supported now + todo!() + }; + let is_link_absolute = link_path.is_absolute(); let mut file = link_path.clone(); @@ -1038,56 +1126,83 @@ impl FileSystem for DiskFileSystem { } let content = content.clone(); - retry_blocking(|| { - let mut f = std::fs::File::create(&full_path)?; - let FileContent::Content(file) = &*content else { - unreachable!() - }; - std::io::copy(&mut file.read(), &mut f)?; - #[cfg(unix)] - f.set_permissions(file.meta.permissions.into())?; - f.flush()?; - - static WRITE_VERSION: LazyLock = LazyLock::new(|| { - std::env::var_os("TURBO_ENGINE_WRITE_VERSION") - .is_some_and(|v| v == "1" || v == "true") - }); - if *WRITE_VERSION { - let mut full_path = full_path.clone().into_owned(); - let hash = hash_xxh3_hash64(file); - let ext = full_path.extension(); - let ext = if let Some(ext) = ext { - format!("{:016x}.{}", hash, ext.to_string_lossy()) + { + #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] + { + retry_blocking(|| { + let mut f = std::fs::File::create(&full_path)?; + let FileContent::Content(file) = &*content else { + unreachable!() + }; + std::io::copy(&mut file.read(), &mut f)?; + #[cfg(unix)] + f.set_permissions(file.meta.permissions.into())?; + f.flush()?; + + static WRITE_VERSION: LazyLock = LazyLock::new(|| { + std::env::var_os("TURBO_ENGINE_WRITE_VERSION") + .is_some_and(|v| v == "1" || v == "true") + }); + if *WRITE_VERSION { + let mut full_path = full_path.clone().into_owned(); + let hash = hash_xxh3_hash64(file); + let ext = full_path.extension(); + let ext = if let Some(ext) = ext { + format!("{:016x}.{}", hash, ext.to_string_lossy()) + } else { + format!("{hash:016x}") + }; + full_path.set_extension(ext); + let mut f = std::fs::File::create(&full_path)?; + std::io::copy(&mut file.read(), &mut f)?; + #[cfg(unix)] + f.set_permissions(file.meta.permissions.into())?; + f.flush()?; + } + Ok::<(), io::Error>(()) + }) + .instrument(tracing::info_span!("write file", name = ?full_path)) + .concurrency_limited(&inner.write_semaphore) + } + + #[cfg(all(target_family = "wasm", target_os = "unknown"))] + async { + if let FileContent::Content(file) = &*content { + let mut content_buf = vec![]; + std::io::copy(&mut file.read(), &mut content_buf)?; + wasm_fs_offload::CLIENT + .write(&full_path, &content_buf) + .await } else { - format!("{hash:016x}") - }; - full_path.set_extension(ext); - let mut f = std::fs::File::create(&full_path)?; - std::io::copy(&mut file.read(), &mut f)?; - #[cfg(unix)] - f.set_permissions(file.meta.permissions.into())?; - f.flush()?; + unreachable!() + } } - Ok::<(), io::Error>(()) - }) - .instrument(tracing::info_span!("write file", name = ?full_path)) - .concurrency_limited(&inner.write_semaphore) + } .await .with_context(|| format!("failed to write to {full_path:?}"))?; } FileContent::NotFound => { - retry_blocking(|| std::fs::remove_file(&full_path)) - .instrument(tracing::info_span!("remove file", name = ?full_path)) - .concurrency_limited(&inner.write_semaphore) - .await - .or_else(|err| { - if err.kind() == ErrorKind::NotFound { - Ok(()) - } else { - Err(err) - } - }) - .with_context(|| format!("removing {full_path:?} failed"))?; + { + #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] + { + retry_blocking(|| std::fs::remove_file(full_path.clone())) + .instrument(tracing::info_span!("remove file", name = ?full_path)) + .concurrency_limited(&inner.write_semaphore) + } + #[cfg(all(target_family = "wasm", target_os = "unknown"))] + { + wasm_fs_offload::CLIENT.remove_file(&full_path) + } + } + .await + .or_else(|err| { + if err.kind() == ErrorKind::NotFound { + Ok(()) + } else { + Err(err) + } + }) + .with_context(|| anyhow!("removing {} failed", full_path.display()))?; } } @@ -1101,6 +1216,11 @@ impl FileSystem for DiskFileSystem { #[turbo_tasks::function(fs)] async fn write_link(&self, fs_path: FileSystemPath, target: Vc) -> Result<()> { + #[cfg(all(target_family = "wasm", target_os = "unknown"))] + { + unreachable!() + } + // You might be tempted to use `mark_session_dependent` here, but we purely declare a side // effect and does not need to be re-executed in the next session. All side effects are // re-executed in general. @@ -1116,6 +1236,7 @@ impl FileSystem for DiskFileSystem { let inner = self.inner.clone(); let invalidator = turbo_tasks::get_invalidator(); + #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] effect(async move { let full_path = validate_path_length(&full_path)?; @@ -1177,6 +1298,7 @@ impl FileSystem for DiskFileSystem { Ok(res) => Some((res.is_absolute(), res)), Err(_) => None, }; + let is_equal = match (&os_specific_link_content, &old_content) { ( OsSpecificLinkContent::Link { target, .. }, @@ -1323,12 +1445,18 @@ impl FileSystem for DiskFileSystem { self.inner.register_read_invalidator(&full_path)?; let _lock = self.inner.lock_path(&full_path).await; + #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] let meta = retry_blocking(|| std::fs::metadata(&full_path)) .instrument(tracing::info_span!("read metadata", name = ?full_path)) .concurrency_limited(&self.inner.read_semaphore) .await .with_context(|| format!("reading metadata for {:?}", full_path))?; + #[cfg(all(target_family = "wasm", target_os = "unknown"))] + let meta = wasm_fs_offload::CLIENT + .metadata(&full_path) + .await + .with_context(|| format!("reading metadata for {}", full_path.display()))?; Ok(FileMeta::cell(meta.into())) } } @@ -1509,7 +1637,7 @@ impl FileSystemPath { /// 1. The parent directory, if any; /// 2. The file stem; /// 3. The extension, if any. - fn split_file_stem_extension(&self) -> (Option<&str>, &str, Option<&str>) { + pub fn split_file_stem_extension(&self) -> (Option<&str>, &str, Option<&str>) { let (path_before_extension, extension) = self.split_extension(); if let Some((parent, file_stem)) = path_before_extension.rsplit_once('/') { @@ -1957,6 +2085,7 @@ enum FileComparison { impl FileContent { /// Performs a comparison of self's data against a disk file's streamed /// read. + #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] async fn streaming_compare(&self, path: &Path) -> Result { let old_file = extract_disk_access(retry_blocking(|| std::fs::File::open(path)).await, path)?; @@ -2010,6 +2139,51 @@ impl FileContent { old_contents.consume(len); }) } + + #[cfg(all(target_family = "wasm", target_os = "unknown"))] + async fn streaming_compare(&self, path: &Path) -> Result { + let old_meta = extract_disk_access(wasm_fs_offload::CLIENT.metadata(path).await, path)?; + let Some(old_meta) = old_meta else { + return Ok(FileComparison::Create); + }; + let FileContent::Content(new_file) = self else { + return Ok(FileComparison::NotEqual); + }; + if new_file.meta != old_meta.into() { + return Ok(FileComparison::NotEqual); + } + let mut new_contents = new_file.read(); + let mut old_contents = if let Some(content) = + extract_disk_access(wasm_fs_offload::CLIENT.read(path).await, path)? + { + let cursor = std::io::Cursor::new(content); + BufReader::new(cursor) + } else { + return Ok(FileComparison::Create); + }; + Ok(loop { + let new_chunk = new_contents.fill_buf()?; + let Ok(old_chunk) = old_contents.fill_buf() else { + break FileComparison::NotEqual; + }; + + let len = min(new_chunk.len(), old_chunk.len()); + if len == 0 { + if new_chunk.len() == old_chunk.len() { + break FileComparison::Equal; + } else { + break FileComparison::NotEqual; + } + } + + if new_chunk[0..len] != old_chunk[0..len] { + break FileComparison::NotEqual; + } + + new_contents.consume(len); + old_contents.consume(len); + }) + } } bitflags! { @@ -2232,6 +2406,13 @@ impl DeterministicHash for FileMeta { } } +#[cfg(all(target_family = "wasm", target_os = "unknown"))] +impl From for FileMeta { + fn from(_meta: tokio_fs_ext::Metadata) -> Self { + FileMeta::default() + } +} + impl FileContent { pub fn new(file: File) -> Self { FileContent::Content(file) diff --git a/turbopack/crates/turbo-tasks-fs/src/wasm_fs_offload.rs b/turbopack/crates/turbo-tasks-fs/src/wasm_fs_offload.rs new file mode 100644 index 000000000000..a77aa2e282fb --- /dev/null +++ b/turbopack/crates/turbo-tasks-fs/src/wasm_fs_offload.rs @@ -0,0 +1,20 @@ +use std::{ + path::{Path, PathBuf}, + sync::LazyLock, +}; + +use anyhow::{Context, Result}; +use parking_lot::Mutex; +use tokio::sync::oneshot; +use tokio_fs_ext::offload::{self, FsOffload}; + +static WASM_FS_OFFLOAD: LazyLock<(Mutex, offload::Client)> = LazyLock::new(|| { + let (server, client) = offload::split(); + (Mutex::new(server), client) +}); + +pub static CLIENT: LazyLock = LazyLock::new(|| WASM_FS_OFFLOAD.1.clone()); + +pub async fn server(offload: impl FsOffload) { + WASM_FS_OFFLOAD.0.lock().serve(offload).await +} diff --git a/turbopack/crates/turbo-tasks-fs/src/watcher.rs b/turbopack/crates/turbo-tasks-fs/src/watcher.rs index c82c7d3113f5..34cc7359aacf 100644 --- a/turbopack/crates/turbo-tasks-fs/src/watcher.rs +++ b/turbopack/crates/turbo-tasks-fs/src/watcher.rs @@ -3,7 +3,7 @@ use std::{ collections::BTreeSet, env, fmt, mem::take, - path::{Path, PathBuf}, + path::{Component, Path, PathBuf}, sync::{ Arc, LazyLock, RwLock, RwLockWriteGuard, mpsc::{Receiver, TryRecvError, channel}, @@ -13,16 +13,17 @@ use std::{ use anyhow::{Context, Result}; use bincode::{Decode, Encode}; -use notify::{ - Config, EventKind, PollWatcher, RecommendedWatcher, RecursiveMode, Watcher, - event::{MetadataKind, ModifyKind, RenameMode}, -}; +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] +use notify::{Config, PollWatcher, RecommendedWatcher, RecursiveMode, Watcher}; +use notify_types::event::{EventKind, MetadataKind, ModifyKind, RenameMode}; use rustc_hash::FxHashSet; use tracing::instrument; use turbo_rcstr::RcStr; +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] +use turbo_tasks::spawn_thread; use turbo_tasks::{ FxIndexSet, InvalidationReason, InvalidationReasonKind, Invalidator, TurboTasksApi, parallel, - spawn_thread, util::StaticOrArc, + util::StaticOrArc, }; use crate::{ @@ -32,6 +33,7 @@ use crate::{ path_map::OrderedPathMapExt, }; +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] static WATCH_RECURSIVE_MODE: LazyLock = LazyLock::new(|| { match env::var("TURBO_TASKS_FORCE_WATCH_MODE").as_deref() { Ok("recursive") => { @@ -64,6 +66,9 @@ static WATCH_RECURSIVE_MODE: LazyLock = LazyLock::new(|| { pub(crate) struct DiskWatcher { #[bincode(skip)] state: State, + /// Paths to ignore when watching for file changes + #[bincode(skip)] + ignored_paths: Arc>, } enum State { @@ -86,12 +91,15 @@ enum StateWriteGuard<'a> { impl State { fn new_stopped() -> Self { + #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] match *WATCH_RECURSIVE_MODE { RecursiveMode::Recursive => Self::Recursive(RwLock::new(RecursiveState::Stopped)), RecursiveMode::NonRecursive => { Self::NonRecursive(RwLock::new(NonRecursiveState::Stopped)) } } + #[cfg(all(target_family = "wasm", target_os = "unknown"))] + Self::Recursive(RwLock::new(RecursiveState::Stopped)) } fn write(&self) -> StateWriteGuard<'_> { @@ -110,6 +118,7 @@ enum RecursiveState { Stopped, Watching { /// Hold onto the watcher: When this is dropped, it will cause the channel to disconnect + #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] _notify_watcher: NotifyWatcher, }, } @@ -124,6 +133,7 @@ enum NonRecursiveState { // split out from the `NonRecursiveState` enum because we want to pass this value around struct NonRecursiveWatchingState { + #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] notify_watcher: NotifyWatcher, /// Keeps track of which directories are currently or were previously watched by /// [`Self::notify_watcher`]. @@ -136,11 +146,13 @@ struct NonRecursiveWatchingState { } /// A thin wrapper around [`RecommendedWatcher`] and [`PollWatcher`]. +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] enum NotifyWatcher { Recommended(RecommendedWatcher), Polling(PollWatcher), } +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] impl NotifyWatcher { fn watch(&mut self, path: &Path, recursive_mode: RecursiveMode) -> notify::Result<()> { match self { @@ -150,11 +162,13 @@ impl NotifyWatcher { } } +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] mod non_recursive_helpers { use super::*; use crate::path_map::OrderedPathSetExt; /// Called after a rescan in case a previously watched-but-deleted directory was recreated. + #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] #[instrument(skip_all, level = "trace")] pub fn restore_all_watched_ignore_errors(state: &RwLock, root_path: &Path) { let mut guard = state.write().unwrap(); @@ -173,6 +187,7 @@ mod non_recursive_helpers { /// Called when a new directory is found in a parent directory we're watching. Restores the /// watcher if we were previously watching it. + #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] #[instrument(skip_all, level = "trace")] pub fn restore_if_watched( state: &RwLock, @@ -259,6 +274,7 @@ mod non_recursive_helpers { /// This does not watch any of the parent directories. For that, use /// [`start_watching_dir_and_parents`]. Use this method when iterating over previously-watched /// values in `self.watching`. + #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] fn start_watching_dir( notify_watcher: &mut NotifyWatcher, dir_path: &Path, @@ -287,6 +303,7 @@ mod non_recursive_helpers { /// Watches the given `dir_path` and every parent up to `root_path`. Parents must be recursively /// watched in case any of them change: /// https://docs.rs/notify/latest/notify/#parent-folder-deletion + #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] fn start_watching_dir_and_parents( state: &mut NonRecursiveWatchingState, dir_path: &Path, @@ -333,9 +350,32 @@ mod non_recursive_helpers { impl DiskWatcher { pub fn new() -> Self { + Self::new_with_ignored_paths(vec![]) + } + + pub fn new_with_ignored_paths(ignored_paths: Vec) -> Self { Self { state: State::new_stopped(), + ignored_paths: Arc::new(ignored_paths), + } + } + + /// Check if a path should be ignored based on configured ignore patterns + fn should_ignore_path(&self, path: &Path) -> bool { + if self.ignored_paths.is_empty() { + return false; } + path.components().any(|component| { + if let Component::Normal(name) = component + && let Some(name_str) = name.to_str() + { + return self + .ignored_paths + .iter() + .any(|ignored| ignored.as_str() == name_str); + } + false + }) } /// Create a watcher and start watching by creating `debounced` watcher @@ -353,6 +393,7 @@ impl DiskWatcher { /// - Emits only one Remove event when deleting a directory (inotify) /// - Doesn't emit duplicate create events /// - Doesn't emit Modify events after a Create event + #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] pub fn start_watching( &self, fs_inner: Arc, @@ -457,6 +498,157 @@ impl DiskWatcher { Ok(()) } + #[cfg(all(target_family = "wasm", target_os = "unknown"))] + pub(crate) async fn start_watching( + &self, + fs_inner: Arc, + report_invalidation_reason: bool, + poll_interval: Option, + ) -> Result<()> { + use crate::wasm_fs_offload; + + let fs_inner_arc = fs_inner.clone(); + let root_path = fs_inner_arc.root_path(); + let ignored_paths = self.ignored_paths.clone(); + let watch_dir = wasm_fs_offload::CLIENT + .watch_dir(root_path, true, move |event| { + let paths: Vec = if ignored_paths.is_empty() { + event.paths + } else { + event + .paths + .into_iter() + .filter(|path| { + !path.components().any(|component| { + if let Component::Normal(name) = component { + if let Some(name_str) = name.to_str() { + return ignored_paths + .iter() + .any(|ignored| ignored.as_str() == name_str); + } + } + false + }) + }) + .collect() + }; + + if paths.is_empty() { + return; + } + + let mut batched_invalidate_path = FxHashSet::default(); + let mut batched_invalidate_path_dir = FxHashSet::default(); + let mut batched_invalidate_path_and_children = FxHashSet::default(); + let mut batched_invalidate_path_and_children_dir = FxHashSet::default(); + + match event.kind { + EventKind::Modify( + ModifyKind::Data(_) | ModifyKind::Metadata(MetadataKind::Any), + ) => { + batched_invalidate_path.extend(paths); + } + EventKind::Create(_) => { + batched_invalidate_path_and_children.extend(paths.clone()); + batched_invalidate_path_and_children_dir.extend(paths.clone()); + paths.iter().for_each(|path| { + if let Some(parent) = path.parent() { + batched_invalidate_path_dir.insert(PathBuf::from(parent)); + } + }); + } + EventKind::Remove(_) => { + batched_invalidate_path_and_children.extend(paths.clone()); + batched_invalidate_path_and_children_dir.extend(paths.clone()); + paths.iter().for_each(|path| { + if let Some(parent) = path.parent() { + batched_invalidate_path_dir.insert(PathBuf::from(parent)); + } + }); + } + EventKind::Modify(ModifyKind::Name(RenameMode::Both)) => { + if let [source, destination, ..] = &paths[..] { + batched_invalidate_path_and_children.insert(source.clone()); + if let Some(parent) = source.parent() { + batched_invalidate_path_dir.insert(PathBuf::from(parent)); + } + batched_invalidate_path_and_children.insert(destination.clone()); + if let Some(parent) = destination.parent() { + batched_invalidate_path_dir.insert(PathBuf::from(parent)); + } + } else { + panic!( + "Rename event does not contain source and destination paths \ + {paths:#?}" + ); + } + } + EventKind::Any | EventKind::Modify(ModifyKind::Any | ModifyKind::Name(..)) => { + batched_invalidate_path.extend(paths.clone()); + batched_invalidate_path_and_children.extend(paths.clone()); + batched_invalidate_path_and_children_dir.extend(paths.clone()); + for parent in paths.iter().filter_map(|path| path.parent()) { + batched_invalidate_path_dir.insert(PathBuf::from(parent)); + } + } + EventKind::Modify(ModifyKind::Metadata(..) | ModifyKind::Other) + | EventKind::Access(_) + | EventKind::Other => { + // ignored + } + } + + let Some(turbo_tasks) = fs_inner.turbo_tasks.upgrade() else { + // TurboTasks was dropped, stop watching + // wasm_fs_offload::CLIENT.stop_watching(); + return; + }; + let _guard = fs_inner.tokio_handle.enter(); + // FIXME: this will cause hanging on wasm now + // let _lock = fs_inner.invalidation_lock.blocking_write(); + { + let mut invalidator_map = fs_inner.invalidator_map.lock().unwrap(); + + invalidate_path( + &fs_inner, + &*turbo_tasks, + report_invalidation_reason, + &mut invalidator_map, + batched_invalidate_path.drain(), + ); + + invalidate_path_and_children_execute( + &fs_inner, + &*turbo_tasks, + report_invalidation_reason, + &mut invalidator_map, + batched_invalidate_path_and_children.drain(), + ); + } + { + let mut dir_invalidator_map = fs_inner.dir_invalidator_map.lock().unwrap(); + invalidate_path( + &fs_inner, + &*turbo_tasks, + report_invalidation_reason, + &mut dir_invalidator_map, + batched_invalidate_path_dir.drain(), + ); + + invalidate_path_and_children_execute( + &fs_inner, + &*turbo_tasks, + report_invalidation_reason, + &mut dir_invalidator_map, + batched_invalidate_path_and_children_dir.drain(), + ); + } + }) + .await?; + + Ok(()) + } + pub fn stop_watching(&self) { match &self.state { State::Recursive(state) => *state.write().unwrap() = RecursiveState::Stopped, @@ -470,6 +662,7 @@ impl DiskWatcher { /// and invalidates the cache. /// /// Should only be called once from `start_watching`. + #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] fn watch_thread( &self, rx: Receiver>, @@ -539,7 +732,11 @@ impl DiskWatcher { break; } - let paths: Vec = event.paths; + let paths: Vec = event + .paths + .into_iter() + .filter(|path| !self.should_ignore_path(path)) + .collect(); if paths.is_empty() { // this event isn't useful, but keep trying to process the batch event_result = rx.try_recv(); @@ -737,6 +934,7 @@ impl DiskWatcher { // Watch the parent directory instead of the specified file, since directories also track // their immediate children (even in non-recursive mode), and we need to watch all the // parents anyways. + #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] if let State::NonRecursive(non_recursive) = &self.state && let Some(dir_path) = path.parent() { @@ -746,6 +944,7 @@ impl DiskWatcher { } pub fn ensure_watched_dir(&self, dir_path: &Path, root_path: &Path) -> Result<()> { + #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] if let State::NonRecursive(non_recursive) = &self.state { non_recursive_helpers::ensure_watched(non_recursive, dir_path, root_path)?; } diff --git a/turbopack/crates/turbo-tasks-hash/Cargo.toml b/turbopack/crates/turbo-tasks-hash/Cargo.toml index 4be1293fb4e0..66aea2172322 100644 --- a/turbopack/crates/turbo-tasks-hash/Cargo.toml +++ b/turbopack/crates/turbo-tasks-hash/Cargo.toml @@ -18,3 +18,6 @@ data-encoding = { workspace = true } sha2 = { workspace = true } turbo-tasks-macros = { workspace = true } twox-hash = { workspace = true } + +[target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies] +getrandom = { version = "0.3", features = ["wasm_js"] } diff --git a/turbopack/crates/turbo-tasks-malloc/Cargo.toml b/turbopack/crates/turbo-tasks-malloc/Cargo.toml index 2933ebad133d..b84c0ae1e5c4 100644 --- a/turbopack/crates/turbo-tasks-malloc/Cargo.toml +++ b/turbopack/crates/turbo-tasks-malloc/Cargo.toml @@ -9,17 +9,14 @@ autobenches = false [lib] bench = false -[dependencies] - - -[target.'cfg(not(any(target_os = "linux", target_family = "wasm")))'.dependencies] -mimalloc = { version = "0.1.48", features = [ +[target.'cfg(not(any(target_os = "linux", target_env = "musl")))'.dependencies] +mimalloc = { version = "0.1.48", features = [ "v3", "extended", ], optional = true } -[target.'cfg(all(target_os = "linux", not(target_family = "wasm")))'.dependencies] -mimalloc = { version = "0.1.48", features = [ +[target.'cfg(all(target_os = "linux", not(target_env = "musl")))'.dependencies] +mimalloc = { version ="0.1.48", features = [ "v3", "extended", "local_dynamic_tls", diff --git a/turbopack/crates/turbo-tasks-malloc/src/lib.rs b/turbopack/crates/turbo-tasks-malloc/src/lib.rs index 47410b0e11a1..4af96208c308 100644 --- a/turbopack/crates/turbo-tasks-malloc/src/lib.rs +++ b/turbopack/crates/turbo-tasks-malloc/src/lib.rs @@ -105,17 +105,17 @@ impl TurboMalloc { /// Get the allocator for this platform that we should wrap with TurboMalloc. #[inline] fn base_alloc() -> &'static impl GlobalAlloc { - #[cfg(all(feature = "custom_allocator", not(target_family = "wasm")))] + #[cfg(all(feature = "custom_allocator", not(target_env = "musl")))] return &mimalloc::MiMalloc; - #[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"))))] + #[cfg(any(not(feature = "custom_allocator"), target_env = "musl"))] return &std::alloc::System; } #[allow(unused_variables)] unsafe fn base_alloc_size(ptr: *const u8, layout: Layout) -> usize { - #[cfg(all(feature = "custom_allocator", not(target_family = "wasm")))] + #[cfg(all(feature = "custom_allocator", not(target_env = "musl")))] return unsafe { mimalloc::MiMalloc.usable_size(ptr) }; - #[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"))))] + #[cfg(any(not(feature = "custom_allocator"), target_env = "musl"))] return layout.size(); } diff --git a/turbopack/crates/turbo-tasks/Cargo.toml b/turbopack/crates/turbo-tasks/Cargo.toml index f0ea87d4ee10..957bd34bb50b 100644 --- a/turbopack/crates/turbo-tasks/Cargo.toml +++ b/turbopack/crates/turbo-tasks/Cargo.toml @@ -36,7 +36,7 @@ futures = { workspace = true } indexmap = { workspace = true, features = ["serde"] } inventory = { workspace = true } once_cell = { workspace = true } -parking_lot = { workspace = true, features = ["serde"]} +parking_lot = { workspace = true, features = ["serde"] } pin-project-lite = { workspace = true } rayon = { workspace = true } regex = { workspace = true } @@ -45,7 +45,7 @@ serde = { workspace = true, features = ["rc", "derive"] } serde_json = { workspace = true } shrink-to-fit = { workspace = true, features = ["indexmap", "serde_json", "smallvec", "nightly"] } smallvec = { workspace = true } -tokio = { workspace = true, features = ["full"] } +tokio = { workspace = true, features = ["sync", "time", "macros", "rt-multi-thread"] } tokio-util = { workspace = true } tracing = { workspace = true } triomphe = { workspace = true, features = ["unsize", "unstable"] } @@ -60,7 +60,5 @@ unsize = { workspace = true } [dev-dependencies] criterion = { workspace = true, features = ["async_tokio"] } +tokio = { workspace = true } -[[bench]] -name = "mod" -harness = false diff --git a/turbopack/crates/turbo-tasks/src/duration_span.rs b/turbopack/crates/turbo-tasks/src/duration_span.rs index 8fc73546f81c..313fa90f771f 100644 --- a/turbopack/crates/turbo-tasks/src/duration_span.rs +++ b/turbopack/crates/turbo-tasks/src/duration_span.rs @@ -1,4 +1,4 @@ -use std::time::Instant; +use tokio::time::Instant; /// Guard that emits a tracing event when dropped with the duration of the /// lifetime of the guard. diff --git a/turbopack/crates/turbo-tasks/src/manager.rs b/turbopack/crates/turbo-tasks/src/manager.rs index cc2fd7070dcc..8e4441c3b9c9 100644 --- a/turbopack/crates/turbo-tasks/src/manager.rs +++ b/turbopack/crates/turbo-tasks/src/manager.rs @@ -9,7 +9,7 @@ use std::{ Arc, Mutex, RwLock, Weak, atomic::{AtomicBool, AtomicUsize, Ordering}, }, - time::{Duration, Instant}, + time::Duration, }; use anyhow::{Result, anyhow}; @@ -20,7 +20,7 @@ use futures::stream::FuturesUnordered; use rustc_hash::{FxBuildHasher, FxHasher}; use serde::{Deserialize, Serialize}; use smallvec::SmallVec; -use tokio::{select, sync::mpsc::Receiver, task_local}; +use tokio::{select, sync::mpsc::Receiver, task_local, time::Instant}; use tracing::{Instrument, Span, instrument}; use crate::{ diff --git a/turbopack/crates/turbo-tasks/src/priority_runner.rs b/turbopack/crates/turbo-tasks/src/priority_runner.rs index 8d5e2ad70cf8..cd3cf195e5f7 100644 --- a/turbopack/crates/turbo-tasks/src/priority_runner.rs +++ b/turbopack/crates/turbo-tasks/src/priority_runner.rs @@ -9,12 +9,15 @@ use std::{ atomic::{AtomicUsize, Ordering}, }, task::{Context, Poll}, - time::{Duration, Instant}, + time::Duration, }; use parking_lot::Mutex; use pin_project_lite::pin_project; -use tokio::sync::oneshot::{Receiver, Sender}; +use tokio::{ + sync::oneshot::{Receiver, Sender}, + time::Instant, +}; pub trait Executor: Send + Sync { type Future: Future + Send; diff --git a/turbopack/crates/turbopack-browser/Cargo.toml b/turbopack/crates/turbopack-browser/Cargo.toml index fd634233594b..cbf01d8c3c81 100644 --- a/turbopack/crates/turbopack-browser/Cargo.toml +++ b/turbopack/crates/turbopack-browser/Cargo.toml @@ -28,7 +28,8 @@ serde_json = { workspace = true } serde_qs = { workspace = true } tracing = { workspace = true } urlencoding = { workspace = true } - +regex = { workspace = true } +qstring = { workspace = true } turbo-bincode = { workspace = true } turbo-rcstr = { workspace = true } turbo-tasks = { workspace = true } @@ -37,5 +38,6 @@ turbo-tasks-hash = { workspace = true } turbopack-core = { workspace = true } turbopack-ecmascript = { workspace = true } turbopack-ecmascript-runtime = { workspace = true } +turbopack-css = { workspace = true } turbopack-resolve = { workspace = true } diff --git a/turbopack/crates/turbopack-browser/src/chunking_context.rs b/turbopack/crates/turbopack-browser/src/chunking_context.rs index c1757fef87a9..ee486cfcf2f2 100644 --- a/turbopack/crates/turbopack-browser/src/chunking_context.rs +++ b/turbopack/crates/turbopack-browser/src/chunking_context.rs @@ -1,5 +1,9 @@ +use std::{cmp::min, sync::LazyLock}; + use anyhow::{Context, Result, bail}; use bincode::{Decode, Encode}; +use qstring::QString; +use regex::Regex; use tracing::Instrument; use turbo_rcstr::{RcStr, rcstr}; use turbo_tasks::{ @@ -7,9 +11,9 @@ use turbo_tasks::{ trace::TraceRawVcs, }; use turbo_tasks_fs::FileSystemPath; -use turbo_tasks_hash::{DeterministicHash, HashAlgorithm}; +use turbo_tasks_hash::{DeterministicHash, hash_xxh3_hash64}; use turbopack_core::{ - asset::Asset, + asset::{Asset, AssetContent}, chunk::{ AssetSuffix, Chunk, ChunkGroupResult, ChunkItem, ChunkType, ChunkableModule, ChunkingConfig, ChunkingConfigs, ChunkingContext, EntryChunkGroupResult, EvaluatableAsset, @@ -29,6 +33,7 @@ use turbopack_core::{ }, output::{OutputAsset, OutputAssets}, }; +use turbopack_css::chunk::{CssChunk, source_map::CssChunkSourceMapAsset}; use turbopack_ecmascript::{ async_chunk::module::AsyncLoaderModule, chunk::EcmascriptChunk, @@ -53,6 +58,9 @@ pub enum CurrentChunkMethod { pub const CURRENT_CHUNK_METHOD_DOCUMENT_CURRENT_SCRIPT_EXPR: &str = "typeof document === \"object\" ? document.currentScript : undefined"; +pub const GLOBAL_THIS_FALLBACK_EXPR: &str = + "typeof globalThis !== \"undefined\" ? globalThis : (typeof self !== \"undefined\" ? self : (typeof window !== \"undefined\" ? window : (typeof global !== \"undefined\" ? global : {})))"; + #[derive( Debug, TaskInput, @@ -98,6 +106,11 @@ impl BrowserChunkingContextBuilder { self } + pub fn entry_root_export(mut self, name: Option) -> Self { + self.chunking_context.entry_root_export = name; + self + } + pub fn tracing(mut self, enable_tracing: bool) -> Self { self.chunking_context.enable_tracing = enable_tracing; self @@ -238,6 +251,31 @@ impl BrowserChunkingContextBuilder { self } + pub fn filename(mut self, filename: RcStr) -> Self { + self.chunking_context.filename = Some(filename); + self + } + + pub fn chunk_filename(mut self, chunk_filename: RcStr) -> Self { + self.chunking_context.chunk_filename = Some(chunk_filename); + self + } + + pub fn css_filename(mut self, css_filename: RcStr) -> Self { + self.chunking_context.css_filename = Some(css_filename); + self + } + + pub fn css_chunk_filename(mut self, css_chunk_filename: RcStr) -> Self { + self.chunking_context.css_chunk_filename = Some(css_chunk_filename); + self + } + + pub fn asset_module_filename(mut self, asset_module_filename: RcStr) -> Self { + self.chunking_context.asset_module_filename = Some(asset_module_filename); + self + } + pub fn chunk_loading_global(mut self, chunk_loading_global: RcStr) -> Self { self.chunking_context.chunk_loading_global = Some(chunk_loading_global); self @@ -337,6 +375,21 @@ pub struct BrowserChunkingContext { /// The global variable name used for chunk loading. /// Default: "TURBOPACK" chunk_loading_global: Option, + /// Evaluate chunk filename template + filename: Option, + /// Non evaluate chunk filename template + chunk_filename: Option, + /// Initial css chunk filename template + css_filename: Option, + /// Non initial css chunk filename template + css_chunk_filename: Option, + /// Asset module filename template + asset_module_filename: Option, + /// Expose entry module exports to global scope with the specified name. + /// When set, all named exports from the entry module will be available on + /// `window`/`globalThis` under the specified name. + /// Default: None (no exposure) + entry_root_export: Option, } impl BrowserChunkingContext { @@ -388,6 +441,12 @@ impl BrowserChunkingContext { should_use_absolute_url_references: false, worker_forwarded_globals: vec![], chunk_loading_global: Default::default(), + filename: Default::default(), + chunk_filename: Default::default(), + css_filename: Default::default(), + css_chunk_filename: Default::default(), + asset_module_filename: Default::default(), + entry_root_export: None, }, } } @@ -490,6 +549,12 @@ impl BrowserChunkingContext { self.minify_type.cell() } + /// Returns the entry root export name to expose to global scope. + #[turbo_tasks::function] + pub fn entry_root_export(&self) -> Vc> { + Vc::cell(self.entry_root_export.clone()) + } + /// Returns the chunk path information. #[turbo_tasks::function] fn chunk_path_info(&self) -> Vc { @@ -562,49 +627,101 @@ impl ChunkingContext for BrowserChunkingContext { "`extension` should include the leading '.', got '{extension}'" ); let ChunkPathInfo { - chunk_root_path, - content_hashing, root_path, + chunk_root_path, + content_hashing: _, } = &*self.chunk_path_info().await?; - let name = match *content_hashing { - None => { - ident - .output_name(root_path.clone(), prefix, extension) - .owned() - .await? - } - Some(ContentHashing::Direct { length }) => { - let Some(asset) = asset else { - bail!("chunk_path requires an asset when content hashing is enabled"); + + let output_name = ident + .output_name(root_path.clone(), prefix, extension.clone()) + .owned() + .await?; + + let mut filename = match asset { + Some(asset) => { + let ident = ident.await?; + + let mut evaluate = false; + let mut dev_chunk_list = false; + ident.modifiers.iter().for_each(|m| { + if m.contains("evaluate") { + evaluate = true; + } + if m.contains("dev chunk list") { + dev_chunk_list = true; + } + }); + let query = QString::from(ident.query.as_str()); + let name = if dev_chunk_list { + output_name.as_str() + } else { + query.get("name").unwrap_or(output_name.as_str()) }; - let hash = asset - .content() - .content_hash(HashAlgorithm::Xxh3Hash128Hex) - .await?; - let hash = hash.as_ref().context( - "chunk_path requires an asset with file content when content hashing is \ - enabled", - )?; - let length = length as usize; - let hash = &hash[0..length]; - if let Some(prefix) = prefix { - format!("{prefix}-{hash}{extension}").into() + + let this = self.await?; + let filename_template = if evaluate { + this.filename.clone() } else { - format!("{hash}{extension}").into() + let resolved_asset = asset.to_resolved().await?; + if ResolvedVc::try_downcast_type::(resolved_asset).is_some() + || ResolvedVc::try_downcast_type::(resolved_asset) + .is_some() + { + // TODO: distinguash initial or non-initial css chunk, the non-initial css + // chunk should use css_chunk_filename for template + this.css_filename.clone() + } else { + this.chunk_filename.clone() + } + }; + + match filename_template { + Some(filename) => { + let mut filename = filename.to_string(); + + if match_name_placeholder(&filename) { + filename = replace_name_placeholder(&filename, name); + } + + if match_content_hash_placeholder(&filename) { + let content = asset.content().await?; + if let AssetContent::File(file) = &*content { + let content_hash = hash_xxh3_hash64(&file.await?); + filename = replace_content_hash_placeholder( + &filename, + &format!("{content_hash:016x}"), + ); + } else { + bail!( + "chunk_path requires an asset with file content when content \ + hashing is enabled" + ); + } + }; + + filename + } + None => name.to_string(), } } + None => output_name.to_string(), }; - Ok(chunk_root_path.join(&name)?.cell()) + + if !filename.ends_with(extension.as_str()) { + filename.push_str(&extension); + } + + Ok(chunk_root_path.join(&filename)?.cell()) } #[turbo_tasks::function] async fn asset_url(&self, ident: FileSystemPath, tag: Option) -> Result> { let asset_path = ident.to_string(); - let client_root = tag - .as_ref() - .and_then(|tag| self.client_roots.get(tag)) - .unwrap_or(&self.client_root); + // let client_root = tag + // .as_ref() + // .and_then(|tag| self.client_roots.get(tag)) + // .unwrap_or(&self.client_root); let asset_base_path = tag .as_ref() @@ -612,8 +729,8 @@ impl ChunkingContext for BrowserChunkingContext { .or(self.asset_base_path.as_ref()); let asset_path = asset_path - .strip_prefix(&format!("{}/", client_root.path)) - .context("expected asset_path to contain client_root")?; + .strip_prefix(&format!("{}/", self.client_root.path)) + .unwrap_or(&asset_path); Ok(Vc::cell( format!( @@ -653,16 +770,40 @@ impl ChunkingContext for BrowserChunkingContext { let source_path = original_asset_ident.path().await?; let basename = source_path.file_name(); let content_hash = content_hash.await?; - let asset_path = match source_path.extension_ref() { - Some(ext) => format!( - "{basename}.{content_hash}.{ext}", - basename = &basename[..basename.len() - ext.len() - 1], - content_hash = &content_hash[..8] - ), - None => format!( - "{basename}.{content_hash}", - content_hash = &content_hash[..8] - ), + + let asset_path = match &self.asset_module_filename { + Some(filename_template) => { + let mut filename = filename_template.to_string(); + + let (_, name, ext) = source_path.split_file_stem_extension(); + + if match_name_placeholder(&filename) { + filename = replace_name_placeholder(&filename, name); + } + + if match_content_hash_placeholder(&filename) { + filename = replace_content_hash_placeholder(&filename, &content_hash); + }; + + if let Some(ext) = ext + && !filename.ends_with(ext) + { + filename = format!("{filename}.{ext}"); + } + + filename + } + None => match source_path.extension_ref() { + Some(ext) => format!( + "{basename}.{content_hash}.{ext}", + basename = &basename[..basename.len() - ext.len() - 1], + content_hash = &content_hash[..8] + ), + None => format!( + "{basename}.{content_hash}", + content_hash = &content_hash[..8] + ), + }, }; let asset_root_path = tag @@ -978,3 +1119,50 @@ struct ChunkPathInfo { chunk_root_path: FileSystemPath, content_hashing: Option, } + +static NAME_PLACEHOLDER_REGEX: LazyLock = LazyLock::new(|| Regex::new(r"\[name\]").unwrap()); + +pub fn match_name_placeholder(s: &str) -> bool { + NAME_PLACEHOLDER_REGEX.is_match(s) +} + +pub fn replace_name_placeholder(s: &str, name: &str) -> String { + NAME_PLACEHOLDER_REGEX + .replace_all(s, |caps: ®ex::Captures| { + let m = caps.get(0).unwrap(); + let after = &s[m.end()..]; + // If the name already ends with an extension (e.g. "foo.js") and the template + // text right after [name] starts with that same extension (e.g. ".js"), strip + // the extension from the name to avoid duplication like "foo.js.js". + if let Some(dot_pos) = name.rfind('.') { + let ext = &name[dot_pos..]; // e.g. ".js" + if after.starts_with(ext) { + return name[..dot_pos].to_string(); + } + } + name.to_string() + }) + .to_string() +} + +static CONTENT_HASH_PLACEHOLDER_REGEX: LazyLock = + LazyLock::new(|| Regex::new(r"\[contenthash(?::(?P\d+))?\]").unwrap()); + +pub fn match_content_hash_placeholder(s: &str) -> bool { + CONTENT_HASH_PLACEHOLDER_REGEX.is_match(s) +} + +pub fn replace_content_hash_placeholder(s: &str, hash: &str) -> String { + CONTENT_HASH_PLACEHOLDER_REGEX + .replace_all(s, |caps: ®ex::Captures| { + let len = caps.name("len").map(|m| m.as_str()).unwrap_or(""); + let len = if len.is_empty() { + hash.len() + } else { + len.parse().unwrap_or(hash.len()) + }; + let len = min(len, hash.len()); + hash[..len].to_string() + }) + .to_string() +} diff --git a/turbopack/crates/turbopack-browser/src/ecmascript/content.rs b/turbopack/crates/turbopack-browser/src/ecmascript/content.rs index 43f4114ca815..9f43594e9f99 100644 --- a/turbopack/crates/turbopack-browser/src/ecmascript/content.rs +++ b/turbopack/crates/turbopack-browser/src/ecmascript/content.rs @@ -21,7 +21,10 @@ use super::{ }; use crate::{ BrowserChunkingContext, - chunking_context::{CURRENT_CHUNK_METHOD_DOCUMENT_CURRENT_SCRIPT_EXPR, CurrentChunkMethod}, + chunking_context::{ + CURRENT_CHUNK_METHOD_DOCUMENT_CURRENT_SCRIPT_EXPR, GLOBAL_THIS_FALLBACK_EXPR, + CurrentChunkMethod, + }, }; #[turbo_tasks::value(serialization = "none")] @@ -109,7 +112,8 @@ impl EcmascriptBrowserChunkContent { code, // `||=` would be better but we need to be es2020 compatible //`x || (x = default)` is better than `x = x || default` simply because we avoid _writing_ the property in the common case. - r#"(globalThis[{chunk_loading_global}] || (globalThis[{chunk_loading_global}] = [])).push([{script_or_path},"#, + r#"(({global_this})[{chunk_loading_global}] || (({global_this})[{chunk_loading_global}] = [])).push([{script_or_path},"#, + global_this = GLOBAL_THIS_FALLBACK_EXPR, chunk_loading_global = StringifyJs(&chunk_loading_global), )?; diff --git a/turbopack/crates/turbopack-browser/src/ecmascript/evaluate/chunk.rs b/turbopack/crates/turbopack-browser/src/ecmascript/evaluate/chunk.rs index 1fd20e7e4d25..aace6b200d37 100644 --- a/turbopack/crates/turbopack-browser/src/ecmascript/evaluate/chunk.rs +++ b/turbopack/crates/turbopack-browser/src/ecmascript/evaluate/chunk.rs @@ -29,7 +29,10 @@ use turbopack_ecmascript_runtime::RuntimeType; use crate::{ BrowserChunkingContext, - chunking_context::{CURRENT_CHUNK_METHOD_DOCUMENT_CURRENT_SCRIPT_EXPR, CurrentChunkMethod}, + chunking_context::{ + CURRENT_CHUNK_METHOD_DOCUMENT_CURRENT_SCRIPT_EXPR, GLOBAL_THIS_FALLBACK_EXPR, + CurrentChunkMethod, + }, }; /// An Ecmascript chunk that: @@ -38,7 +41,7 @@ use crate::{ #[turbo_tasks::value(shared)] #[derive(ValueToString)] #[value_to_string("Ecmascript Browser Evaluate Chunk")] -pub(crate) struct EcmascriptBrowserEvaluateChunk { +pub struct EcmascriptBrowserEvaluateChunk { chunking_context: ResolvedVc, ident: ResolvedVc, other_chunks: ResolvedVc, @@ -70,13 +73,33 @@ impl EcmascriptBrowserEvaluateChunk { } #[turbo_tasks::function] - async fn chunks_data(&self) -> Result> { + pub async fn chunks_data(&self) -> Result> { Ok(ChunkData::from_assets( self.chunking_context.output_root().owned().await?, *self.other_chunks, )) } + #[turbo_tasks::function] + pub fn ident(&self) -> Vc { + *self.ident + } + + #[turbo_tasks::function] + pub fn evaluatable_assets(&self) -> Vc { + *self.evaluatable_assets + } + + #[turbo_tasks::function] + pub fn module_graph(&self) -> Vc { + *self.module_graph + } + + #[turbo_tasks::function] + pub fn chunking_context(&self) -> Vc> { + Vc::upcast(*self.chunking_context) + } + #[turbo_tasks::function] async fn code(self: Vc) -> Result> { let this = self.await?; @@ -162,11 +185,12 @@ impl EcmascriptBrowserEvaluateChunk { // `||=` would be better but we need to be es2020 compatible //`x || (x = default)` is better than `x = x || default` simply because we avoid _writing_ the property in the common case. r#" - (globalThis[{chunk_loading_global}] || (globalThis[{chunk_loading_global}] = [])).push([ + (({global_this})[{chunk_loading_global}] || (({global_this})[{chunk_loading_global}] = [])).push([ {script_or_path}, {params} ]); "#, + global_this = GLOBAL_THIS_FALLBACK_EXPR, chunk_loading_global = StringifyJs(&chunk_loading_global), params = StringifyJs(¶ms), )?; @@ -183,6 +207,7 @@ impl EcmascriptBrowserEvaluateChunk { output_root_to_root_path, source_maps, this.chunking_context.chunk_loading_global(), + this.chunking_context.entry_root_export(), ); code.push_code(&*runtime_code.await?); } diff --git a/turbopack/crates/turbopack-browser/src/ecmascript/list/asset.rs b/turbopack/crates/turbopack-browser/src/ecmascript/list/asset.rs index 1e5489073638..303a58a578c4 100644 --- a/turbopack/crates/turbopack-browser/src/ecmascript/list/asset.rs +++ b/turbopack/crates/turbopack-browser/src/ecmascript/list/asset.rs @@ -29,7 +29,7 @@ use crate::BrowserChunkingContext; #[turbo_tasks::value(shared)] #[derive(ValueToString)] #[value_to_string("Ecmascript Dev Chunk List")] -pub(crate) struct EcmascriptDevChunkList { +pub struct EcmascriptDevChunkList { pub(super) chunking_context: ResolvedVc, pub(super) ident: ResolvedVc, pub(super) evaluatable_assets: ResolvedVc, diff --git a/turbopack/crates/turbopack-browser/src/ecmascript/list/content.rs b/turbopack/crates/turbopack-browser/src/ecmascript/list/content.rs index 732fe122548e..a5c78820df21 100644 --- a/turbopack/crates/turbopack-browser/src/ecmascript/list/content.rs +++ b/turbopack/crates/turbopack-browser/src/ecmascript/list/content.rs @@ -27,7 +27,8 @@ use super::{ version::EcmascriptDevChunkListVersion, }; use crate::chunking_context::{ - CURRENT_CHUNK_METHOD_DOCUMENT_CURRENT_SCRIPT_EXPR, CurrentChunkMethod, + CURRENT_CHUNK_METHOD_DOCUMENT_CURRENT_SCRIPT_EXPR, GLOBAL_THIS_FALLBACK_EXPR, + CurrentChunkMethod, }; #[derive( @@ -158,7 +159,7 @@ impl EcmascriptDevChunkListContent { let mut code = CodeBuilder::default(); - // When loaded, JS chunks must register themselves with the `TURBOPACK` global + // When loaded, JS chunks must register themselves with the chunk loading global // variable. Similarly, we register the chunk list with the // `{chunk_loading_global}_CHUNK_LISTS` global variable. let chunk_lists_global = format!("{}_CHUNK_LISTS", this.chunk_loading_global); @@ -167,12 +168,13 @@ impl EcmascriptDevChunkListContent { // `||=` would be better but we need to be es2020 compatible //`x || (x = default)` is better than `x = x || default` simply because we avoid _writing_ the property in the common case. r#" - (globalThis[{chunk_lists_global}] || (globalThis[{chunk_lists_global}] = [])).push({{ + (({global_this})[{chunk_lists_global}] || (({global_this})[{chunk_lists_global}] = [])).push({{ script: {script_or_path}, chunks: {chunks}, source: {source} }}); "#, + global_this = GLOBAL_THIS_FALLBACK_EXPR, chunk_lists_global = StringifyJs(&chunk_lists_global), chunks = StringifyJs(&chunks), source = StringifyJs(&this.source), diff --git a/turbopack/crates/turbopack-browser/src/ecmascript/mod.rs b/turbopack/crates/turbopack-browser/src/ecmascript/mod.rs index d80b785e8ba6..3da0cc8b965f 100644 --- a/turbopack/crates/turbopack-browser/src/ecmascript/mod.rs +++ b/turbopack/crates/turbopack-browser/src/ecmascript/mod.rs @@ -10,4 +10,6 @@ pub(crate) mod worker; pub use chunk::EcmascriptBrowserChunk; pub use content::EcmascriptBrowserChunkContent; +pub use evaluate::chunk::EcmascriptBrowserEvaluateChunk; +pub use list::asset::EcmascriptDevChunkList; pub use worker::EcmascriptBrowserWorkerEntrypoint; diff --git a/turbopack/crates/turbopack-browser/src/lib.rs b/turbopack/crates/turbopack-browser/src/lib.rs index 0604839560af..e3f362cda84f 100644 --- a/turbopack/crates/turbopack-browser/src/lib.rs +++ b/turbopack/crates/turbopack-browser/src/lib.rs @@ -3,7 +3,7 @@ #![feature(arbitrary_self_types)] #![feature(arbitrary_self_types_pointers)] -pub(crate) mod chunking_context; +pub mod chunking_context; pub mod ecmascript; pub mod react_refresh; diff --git a/turbopack/crates/turbopack-cli-utils/Cargo.toml b/turbopack/crates/turbopack-cli-utils/Cargo.toml index 2cae973901ef..5e61eea0e61d 100644 --- a/turbopack/crates/turbopack-cli-utils/Cargo.toml +++ b/turbopack/crates/turbopack-cli-utils/Cargo.toml @@ -16,7 +16,7 @@ workspace = true [dependencies] anyhow = { workspace = true } -crossterm = "0.26.0" +clap = { workspace = true, features = ["derive"] } owo-colors = { workspace = true } rustc-hash = { workspace = true } serde = { workspace = true, features = ["derive"] } @@ -26,3 +26,5 @@ turbo-tasks-fs = { workspace = true } turbopack-core = { workspace = true } turbopack-resolve = { workspace = true } +[target.'cfg(not(all(target_family = "wasm", target_os = "unknown")))'.dependencies] +crossterm = "0.29.0" diff --git a/turbopack/crates/turbopack-cli-utils/src/issue.rs b/turbopack/crates/turbopack-cli-utils/src/issue.rs index 8614390ac3db..4d7c5cfcb1e4 100644 --- a/turbopack/crates/turbopack-cli-utils/src/issue.rs +++ b/turbopack/crates/turbopack-cli-utils/src/issue.rs @@ -8,6 +8,7 @@ use std::{ }; use anyhow::Result; +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] use crossterm::style::{StyledContent, Stylize}; use owo_colors::{OwoColorize as _, Style}; use rustc_hash::{FxHashMap, FxHashSet}; @@ -537,10 +538,18 @@ fn make_relative_to_cwd<'a>(path: &'a str, project_dir: &Path, cwd: &Path) -> Co } } +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] fn show_all_message(label: &str, size: usize) -> StyledContent { show_all_message_with_shown_count(label, size, DEFAULT_SHOW_COUNT) } +// FIXME: +#[cfg(all(target_family = "wasm", target_os = "unknown"))] +fn show_all_message(label: &str, size: usize) -> String { + format!("{label} {size}") +} + +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] fn show_all_message_with_shown_count( label: &str, size: usize, @@ -563,6 +572,12 @@ fn show_all_message_with_shown_count( } } +// FIXME: +#[cfg(all(target_family = "wasm", target_os = "unknown"))] +fn show_all_message_with_shown_count(label: &str, size: usize, shown: usize) -> String { + format!("{label} {size} {shown}") +} + fn render_styled_string_to_ansi(styled_string: &StyledString) -> String { match styled_string { StyledString::Line(parts) => { diff --git a/turbopack/crates/turbopack-core/src/ident.rs b/turbopack/crates/turbopack-core/src/ident.rs index a8e5e62eff79..aab60aee40af 100644 --- a/turbopack/crates/turbopack-core/src/ident.rs +++ b/turbopack/crates/turbopack-core/src/ident.rs @@ -268,6 +268,27 @@ impl AssetIdent { fragment.deterministic_hash(&mut hasher); has_hash = true; } + if !assets.is_empty() { + // Use XOR to combine asset hashes in an order-independent way + // This ensures chunks with the same modules but different order get the same hash + let mut asset_hashes = Vec::with_capacity(assets.len()); + for (key, ident) in assets.iter() { + let mut asset_hasher = Xxh3Hash64Hasher::new(); + key.deterministic_hash(&mut asset_hasher); + ident + .to_string() + .await? + .deterministic_hash(&mut asset_hasher); + asset_hashes.push(asset_hasher.finish()); + } + asset_hashes.sort_unstable(); + + 2_u8.deterministic_hash(&mut hasher); + for h in asset_hashes { + h.deterministic_hash(&mut hasher); + } + has_hash = true; + } for (key, ident) in assets.iter() { 2_u8.deterministic_hash(&mut hasher); key.deterministic_hash(&mut hasher); @@ -364,10 +385,10 @@ impl AssetIdent { // We need to make sure that `.json` and `.json.js` doesn't end up with the same // name. So when we add an extra extension when want to mark that with a "._" // suffix. - if !removed_extension { - name += "._"; - } - name += &expected_extension; + // if !removed_extension { + // name += "._"; + // } + // name += &expected_extension; Ok(Vc::cell(name.into())) } } @@ -433,8 +454,8 @@ impl ValueToString for AssetIdent { } } -fn escape_file_path(s: &str) -> String { - static SEPARATOR_REGEX: Lazy = Lazy::new(|| Regex::new(r"[/#?:]").unwrap()); +pub fn escape_file_path(s: &str) -> String { + static SEPARATOR_REGEX: Lazy = Lazy::new(|| Regex::new(r"[/#?:\[\]<>@\s()]").unwrap()); SEPARATOR_REGEX.replace_all(s, "_").to_string() } diff --git a/turbopack/crates/turbopack-core/src/resolve/mod.rs b/turbopack/crates/turbopack-core/src/resolve/mod.rs index ab3602fe81d6..289dca63f8e5 100644 --- a/turbopack/crates/turbopack-core/src/resolve/mod.rs +++ b/turbopack/crates/turbopack-core/src/resolve/mod.rs @@ -20,7 +20,7 @@ use turbo_tasks::{ FxIndexMap, FxIndexSet, NonLocalValue, ReadRef, ResolvedVc, TaskInput, TryFlatJoinIterExt, TryJoinIterExt, ValueToString, Vc, trace::TraceRawVcs, }; -use turbo_tasks_fs::{FileSystemEntryType, FileSystemPath}; +use turbo_tasks_fs::{DiskFileSystem, FileSystemEntryType, FileSystemPath}; use turbo_unix_path::normalize_request; use crate::{ @@ -479,6 +479,7 @@ pub enum ExternalType { EcmaScriptModule, Global, Script, + Umd, } impl Display for ExternalType { @@ -489,6 +490,7 @@ impl Display for ExternalType { ExternalType::Url => write!(f, "url"), ExternalType::Global => write!(f, "global"), ExternalType::Script => write!(f, "script"), + ExternalType::Umd => write!(f, "umd"), } } } @@ -2061,18 +2063,78 @@ async fn resolve_internal_inline( .await? } Request::Windows { - path: _, - query: _, - fragment: _, + path, + query, + fragment, } => { + if let Some(path_str) = path.as_constant_string() { + let sys_path = std::path::Path::new(path_str.as_str()); + + if let Some(disk_fs_vc) = + ResolvedVc::try_downcast_type::(lookup_path.fs) + { + let disk_fs = disk_fs_vc.await?; + let root_path = lookup_path.root().owned().await?; + + // Try to convert the Windows path to a FileSystemPath + if let Some(fs_path) = disk_fs.try_from_sys_path(disk_fs_vc, sys_path, None) + { + // Successfully converted - resolve as a raw path + let mut results = Vec::new(); + let unix_path = &fs_path.path; + let pattern = Pattern::Constant(unix_path.clone()); + let matches = read_matches( + root_path.clone(), + rcstr!(""), + false, + Pattern::new(pattern).resolve().await?, + ) + .await?; + + for m in matches.iter() { + match m { + PatternMatch::File(matched_pattern, path) => { + results.push( + resolved( + RequestKey::new(matched_pattern.clone()), + path.clone(), + lookup_path.clone(), + request, + options_value, + options, + query.clone(), + fragment.clone(), + ) + .await? + .into_cell(), + ); + } + PatternMatch::Directory(matched_pattern, path) => { + results.push( + resolve_into_folder(path.clone(), options) + .with_request(matched_pattern.clone()), + ); + } + } + } + + return Ok(merge_results(results)); + } + } + } + if !has_alias { ResolvingIssue { severity: resolve_error_severity(options).await?, - request_type: "windows import: not implemented yet".to_string(), + request_type: "windows import".to_string(), request: request.to_resolved().await?, file_path: lookup_path.clone(), resolve_options: options.to_resolved().await?, - error_message: Some("windows imports are not implemented yet".to_string()), + error_message: Some( + "Windows absolute path imports can only be resolved if the path is \ + within the project root. Please use a relative path instead." + .to_string(), + ), source: None, } .resolved_cell() @@ -2970,6 +3032,12 @@ async fn resolve_import_map_result( ImportMapResult::Result(result) => Some(ResolveResultOrCell::Cell(**result)), ImportMapResult::Alias(request, alias_lookup_path) => { let request = **request; + // Preserve the query string from the original request (e.g., ?modules for CSS modules) + let request = if !query.is_empty() { + request.with_query(query.clone()) + } else { + request + }; let lookup_path = match alias_lookup_path { Some(path) => path.clone(), None => lookup_path, @@ -3025,7 +3093,10 @@ async fn resolve_import_map_result( ExternalType::EcmaScriptModule => { node_esm_resolve_options(alias_lookup_path.root().owned().await?) } - ExternalType::Script | ExternalType::Url | ExternalType::Global => options, + ExternalType::Script + | ExternalType::Url + | ExternalType::Global + | ExternalType::Umd => options, }, ) .await? diff --git a/turbopack/crates/turbopack-core/src/resolve/parse.rs b/turbopack/crates/turbopack-core/src/resolve/parse.rs index 15f58172c845..4230c9f49961 100644 --- a/turbopack/crates/turbopack-core/src/resolve/parse.rs +++ b/turbopack/crates/turbopack-core/src/resolve/parse.rs @@ -162,6 +162,20 @@ impl Request { return Request::Empty; } + // Handle webpack-style tilde prefix (~) for node_modules resolution + // This is commonly used in CSS preprocessors like less-loader and sass-loader + // Only strip ~ if it's followed by a module path (not a relative path like ~/home) + // for utoopack issue: https://github.com/utooland/utoo/issues/2309 + let r = if let Some(remainder) = r + .strip_prefix('~') + .filter(|s| !s.is_empty() && !s.starts_with('/') && !s.starts_with('\\')) + .filter(|s| MODULE_PATH.is_match(s)) + { + remainder.into() + } else { + r + }; + if let Some(remainder) = r.strip_prefix("//") { return Request::Uri { protocol: rcstr!("//"), diff --git a/turbopack/crates/turbopack-core/src/source_map/utils.rs b/turbopack/crates/turbopack-core/src/source_map/utils.rs index 63df52bf29bd..8caa3e1ac27d 100644 --- a/turbopack/crates/turbopack-core/src/source_map/utils.rs +++ b/turbopack/crates/turbopack-core/src/source_map/utils.rs @@ -114,15 +114,24 @@ pub async fn resolve_source_map_sources( let fs_path = if let Ok(original_source_url_obj) = Url::parse(&maybe_file_url) { // We have an absolute URL, try to parse it as a `file://` URL - if let Ok(sys_path) = original_source_url_obj.to_file_path() { - if let Some((disk_fs_vc, disk_fs)) = disk_fs { - disk_fs.try_from_sys_path(*disk_fs_vc, &sys_path, Some(origin)) + #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] + { + if let Ok(sys_path) = original_source_url_obj.to_file_path() { + if let Some((disk_fs_vc, disk_fs)) = disk_fs { + disk_fs.try_from_sys_path(*disk_fs_vc, &sys_path, Some(origin)) + } else { + None + } } else { - None + // this is an absolute URL with a non-`file://` scheme, just assume it's valid + // and don't modify anything + return Ok(()); } - } else { - // this is an absolute URL with a non-`file://` scheme, just assume it's valid - // and don't modify anything + } + #[cfg(all(target_family = "wasm", target_os = "unknown"))] + { + // On WASM targets, to_file_path() is not available, + // just assume it's a valid absolute URL and don't modify anything return Ok(()); } } else { diff --git a/turbopack/crates/turbopack-css/src/process.rs b/turbopack/crates/turbopack-css/src/process.rs index d58750d7b298..047284822494 100644 --- a/turbopack/crates/turbopack-css/src/process.rs +++ b/turbopack/crates/turbopack-css/src/process.rs @@ -448,16 +448,20 @@ async fn process_content( let mut validator = CssValidator { errors: Vec::new() }; ss.visit(&mut validator).unwrap(); - for err in validator.errors { - err.report(source); - } + // TODO: remove pure selector + // for err in validator.errors { + // err.report(source); + // } } for err in warnings.read().unwrap().iter() { - match err.kind { + match &err.kind { + // Ignore all SelectorError errors + lightningcss::error::ParserError::SelectorError(..) => { + continue; + } lightningcss::error::ParserError::UnexpectedToken(_) | lightningcss::error::ParserError::UnexpectedImportRule - | lightningcss::error::ParserError::SelectorError(..) | lightningcss::error::ParserError::EndOfInput => { let source = match &err.loc { Some(loc) => IssueSource::from_single_line_col( @@ -482,7 +486,7 @@ async fn process_content( } _ => { - // Ignore + // Ignore other warnings } } } @@ -523,6 +527,11 @@ async fn process_content( stylesheet_into_static(&ss, without_warnings(config.clone())) } Err(e) => { + // Ignore all SelectorError errors + if matches!(e.kind, lightningcss::error::ParserError::SelectorError(..)) { + return Ok(ParseCssResult::Unparsable.cell()); + } + let source = match &e.loc { Some(loc) => IssueSource::from_single_line_col( source, @@ -580,6 +589,7 @@ enum CssError { } impl CssError { + #[allow(unused)] fn report(self, source: ResolvedVc>) { match self { CssError::CssSelectorInModuleNotPure { selector } => { diff --git a/turbopack/crates/turbopack-css/src/references/url.rs b/turbopack/crates/turbopack-css/src/references/url.rs index da2166521e11..c8f73d523b5e 100644 --- a/turbopack/crates/turbopack-css/src/references/url.rs +++ b/turbopack/crates/turbopack-css/src/references/url.rs @@ -82,7 +82,7 @@ impl ModuleReference for UrlAssetReference { *self.request, ReferenceType::Url(UrlReferenceSubType::CssUrl), Some(self.issue_source), - ResolveErrorMode::Error, + ResolveErrorMode::Warn, ) } diff --git a/turbopack/crates/turbopack-dev-server/Cargo.toml b/turbopack/crates/turbopack-dev-server/Cargo.toml index bbe1b58a04c9..685f19446e15 100644 --- a/turbopack/crates/turbopack-dev-server/Cargo.toml +++ b/turbopack/crates/turbopack-dev-server/Cargo.toml @@ -21,8 +21,6 @@ auto-hash-map = { workspace = true } bincode = { workspace = true } flate2 = { workspace = true } futures = { workspace = true } -hyper = { version = "0.14", features = ["full"] } -hyper-tungstenite = "0.9.0" mime = { workspace = true } mime_guess = "2.0.4" parking_lot = { workspace = true } @@ -31,10 +29,7 @@ rustc-hash = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } serde_qs = { workspace = true } -socket2 = "0.4.9" -tokio = { workspace = true } -tokio-stream = "0.1.9" -tokio-util = { workspace = true } + tracing = { workspace = true } turbo-bincode = { workspace = true } turbo-rcstr = { workspace = true } @@ -49,6 +44,14 @@ turbopack-ecmascript = { workspace = true } turbopack-ecmascript-hmr-protocol = { workspace = true } urlencoding = "2.1.2" +[target.'cfg(not(all(target_family = "wasm", target_os = "unknown")))'.dependencies] +hyper = { version = "0.14", features = ["full"] } +hyper-tungstenite = "0.9.0" +tokio = { workspace = true } +tokio-stream = "0.1.9" +tokio-util = { workspace = true } +socket2 = "0.4.9" + [dev-dependencies] turbo-tasks-backend = { workspace = true } diff --git a/turbopack/crates/turbopack-dev-server/src/lib.rs b/turbopack/crates/turbopack-dev-server/src/lib.rs index 4894eba6c967..2586c9286afe 100644 --- a/turbopack/crates/turbopack-dev-server/src/lib.rs +++ b/turbopack/crates/turbopack-dev-server/src/lib.rs @@ -6,12 +6,16 @@ #![feature(arbitrary_self_types_pointers)] pub mod html; +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] mod http; pub mod introspect; +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] mod invalidation; pub mod source; +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] pub mod update; +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] use std::{ collections::VecDeque, future::Future, @@ -21,27 +25,38 @@ use std::{ time::{Duration, Instant}, }; +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] use anyhow::{Context, Result}; +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] use hyper::{ Request, Response, Server, server::{Builder, conn::AddrIncoming}, service::{make_service_fn, service_fn}, }; +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] use parking_lot::Mutex; +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] use socket2::{Domain, Protocol, Socket, Type}; +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] use tokio::task::JoinHandle; +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] use tracing::{Instrument, Level, Span, event, info_span}; +use turbo_tasks::OperationVc; +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] use turbo_tasks::{ - Effects, NonLocalValue, OperationVc, PrettyPrintError, TurboTasksApi, Vc, get_effects, - run_once_with_reason, trace::TraceRawVcs, util::FormatDuration, + Effects, NonLocalValue, PrettyPrintError, TurboTasksApi, Vc, get_effects, run_once_with_reason, + trace::TraceRawVcs, util::FormatDuration, }; +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] use turbopack_core::issue::{IssueReporter, IssueSeverity, handle_issues}; -use self::{source::ContentSource, update::UpdateServer}; -use crate::{ - invalidation::{ServerRequest, ServerRequestSideEffects}, - source::ContentSourceSideEffect, -}; +use self::source::ContentSource; +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] +use self::update::UpdateServer; +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] +use crate::invalidation::{ServerRequest, ServerRequestSideEffects}; +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] +use crate::source::ContentSourceSideEffect; pub trait SourceProvider: Send + Clone + 'static { /// must call a turbo-tasks function internally @@ -80,6 +95,7 @@ pub struct DevServerBuilder { server: Builder, } +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] #[derive(TraceRawVcs, NonLocalValue)] pub struct DevServer { #[turbo_tasks(trace_ignore)] @@ -88,6 +104,7 @@ pub struct DevServer { pub future: Pin> + Send + 'static>>, } +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] impl DevServer { pub fn listen(addr: SocketAddr) -> Result { // This is annoying. The hyper::Server doesn't allow us to know which port was @@ -123,6 +140,7 @@ impl DevServer { } } +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] impl DevServerBuilder { pub fn serve( self, diff --git a/turbopack/crates/turbopack-dev-server/src/source/mod.rs b/turbopack/crates/turbopack-dev-server/src/source/mod.rs index 2d1bbf21a562..564135f8bd6a 100644 --- a/turbopack/crates/turbopack-dev-server/src/source/mod.rs +++ b/turbopack/crates/turbopack-dev-server/src/source/mod.rs @@ -5,7 +5,9 @@ pub mod headers; pub mod issue_context; pub mod lazy_instantiated; pub mod query; +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] pub mod request; +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] pub(crate) mod resolve; pub mod route_tree; pub mod router; diff --git a/turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/base/runtime-base.ts b/turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/base/runtime-base.ts index dc3fe78bdd75..3955a13cf8da 100644 --- a/turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/base/runtime-base.ts +++ b/turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/base/runtime-base.ts @@ -23,6 +23,14 @@ declare var CHUNK_BASE_PATH: string declare var ASSET_SUFFIX: string declare var WORKER_FORWARDED_GLOBALS: string[] +// Support runtime public path from window.publicPath +function getRuntimeChunkBasePath(): string { + if (CHUNK_BASE_PATH === '__RUNTIME_PUBLIC_PATH__') { + return contextPrototype.p() + } + return CHUNK_BASE_PATH +} + interface TurbopackBrowserBaseContext extends TurbopackBaseContext { R: ResolvePathFromModule } @@ -198,6 +206,36 @@ function loadChunkByUrl( } browserContextPrototype.L = loadChunkByUrl +const loadedScripts = new Map>() + +/** + * Load an external script by creating a