From a27dab6c7fd6cddcfaca425d27627588ada104ec Mon Sep 17 00:00:00 2001 From: Luxian Date: Sun, 11 Oct 2026 04:19:30 +0800 Subject: [PATCH 1/8] perf(v3): defer lazy root metadata proof until access --- src/snapshot/fuse.rs | 27 ++++++++++++++++++++++++++- src/workspace/service.rs | 33 ++++++++++++++++++++++++--------- tests/mst2_workspace_v3_http.rs | 19 +++++++++++++++++++ 3 files changed, 69 insertions(+), 10 deletions(-) diff --git a/src/snapshot/fuse.rs b/src/snapshot/fuse.rs index e0b034d..e4840da 100644 --- a/src/snapshot/fuse.rs +++ b/src/snapshot/fuse.rs @@ -489,6 +489,29 @@ impl Mst2Fuse { reader: SnapshotReader, store: Option>, metadata_limits: MetadataProofLimits, + ) -> std::result::Result { + Self::from_reader_lazy_with_limits_inner(reader, store, metadata_limits, true).await + } + + /// Build a lazy view without proving the root page during admission. + /// + /// A v3 workspace only needs the immutable metadata root identity before + /// exposing its mount. The root page is still fully verified when the + /// kernel first asks for lookup/readdir/opendir, but deferring that proof + /// keeps workspace creation independent of the size of the root directory. + pub(crate) async fn from_reader_lazy_deferred( + reader: SnapshotReader, + store: Option>, + metadata_limits: MetadataProofLimits, + ) -> std::result::Result { + Self::from_reader_lazy_with_limits_inner(reader, store, metadata_limits, false).await + } + + async fn from_reader_lazy_with_limits_inner( + reader: SnapshotReader, + store: Option>, + metadata_limits: MetadataProofLimits, + eager_root: bool, ) -> std::result::Result { if metadata_limits.max_directory_pages == 0 || metadata_limits.max_directory_entries == 0 @@ -548,7 +571,9 @@ impl Mst2Fuse { state: StdMutex::new(state), metadata_limits, }; - view.ensure_dir_loaded(ROOT_INODE).await?; + if eager_root { + view.ensure_dir_loaded(ROOT_INODE).await?; + } Ok(view) } diff --git a/src/workspace/service.rs b/src/workspace/service.rs index b32429c..56bfbae 100644 --- a/src/workspace/service.rs +++ b/src/workspace/service.rs @@ -483,16 +483,31 @@ impl WorkspaceService { &self.config.cache_root, ); } + // Lazy admission binds only the immutable metadata root identity. + // Full delivery keeps the eager root proof so hydration failures are + // reported by create; lazy mounts prove a directory on first access. let lower = Arc::new( - trace_async( - "root_metadata_proof", - Mst2Fuse::from_reader_lazy_with_limits( - reader, - Some(store), - self.config.metadata_limits, - ), - ) - .await? + (if matches!(request.delivery, WorkspaceDelivery::Lazy) { + trace_async( + "root_metadata_bind", + Mst2Fuse::from_reader_lazy_deferred( + reader, + Some(store), + self.config.metadata_limits, + ), + ) + .await? + } else { + trace_async( + "root_metadata_proof", + Mst2Fuse::from_reader_lazy_with_limits( + reader, + Some(store), + self.config.metadata_limits, + ), + ) + .await? + }) .with_read_profile(runtime.read_profile.clone()), ); runtime.lower = Some(lower.clone()); diff --git a/tests/mst2_workspace_v3_http.rs b/tests/mst2_workspace_v3_http.rs index 2984d38..c07cc1f 100644 --- a/tests/mst2_workspace_v3_http.rs +++ b/tests/mst2_workspace_v3_http.rs @@ -851,6 +851,25 @@ async fn destroying_failed_preparation_retires_unknown_cache_use_without_private h.fixture.assert_only_canonical_requests(); } +#[tokio::test] +async fn lazy_creation_binds_root_identity_without_fetching_root_page() { + // A lazy v3 admission must not enumerate a large root directory just to + // expose the mount. The root proof remains on the first namespace access; + // denying it here proves create itself has no metadata-page dependency. + let h = Harness::new(RootFailure::Denied, false, 1, 2).await; + let response = h.create(request("lazy")).await; + assert_eq!(response.status(), StatusCode::OK); + let status: Value = response.json().await.unwrap(); + assert_eq!(status["mount_state"], "mounted"); + assert_eq!(status["metadata_ready"], true); + assert_eq!(h.fixture.metadata.load(Ordering::SeqCst), 0); + + let id = status["workspace_id"].as_str().unwrap(); + h.destroy(id).await; + assert_eq!(h.fixture.metadata.load(Ordering::SeqCst), 0); + assert_eq!(h.list().await, json!([])); +} + #[tokio::test] async fn cancelled_caller_retains_creation_and_workspace_capacity_until_failed_entry_is_destroyed() { From 5b135cb61d128c2cb57a8d00a0a99dae1b6533a4 Mon Sep 17 00:00:00 2001 From: Luxian Date: Sun, 11 Oct 2026 04:22:56 +0800 Subject: [PATCH 2/8] test(v3): keep eager root failure cases on full delivery --- tests/mst2_workspace_v3_http.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/mst2_workspace_v3_http.rs b/tests/mst2_workspace_v3_http.rs index c07cc1f..5b5d259 100644 --- a/tests/mst2_workspace_v3_http.rs +++ b/tests/mst2_workspace_v3_http.rs @@ -362,7 +362,7 @@ impl Harness { let service = self.service.clone(); tokio::spawn(async move { service - .create(serde_json::from_value(request("lazy")).unwrap()) + .create(serde_json::from_value(request("full")).unwrap()) .await }) } @@ -580,7 +580,7 @@ async fn observed_creation_records_its_only_actual_resolve_and_independent_owner Harness::new_with_observer(RootFailure::Denied, false, 2, 2, Some(observer.clone())).await; for _ in 0..2 { assert_eq!( - h.create(request("lazy")).await.status(), + h.create(request("full")).await.status(), StatusCode::INTERNAL_SERVER_ERROR ); } @@ -643,7 +643,7 @@ async fn ordinary_creation_has_no_observation_header_or_record() { WorkspaceObserver::channel("11111111-2222-4333-8444-666666666666", 1).unwrap(); let h = Harness::new(RootFailure::Denied, false, 1, 2).await; assert_eq!( - h.create(request("lazy")).await.status(), + h.create(request("full")).await.status(), StatusCode::INTERNAL_SERVER_ERROR ); assert_eq!(*h.fixture.resolve_ids.lock().unwrap(), [None]); @@ -665,7 +665,7 @@ async fn full_observation_queue_is_sticky_but_preserves_failed_owners_and_cleanu Harness::new_with_observer(RootFailure::Denied, false, 2, 2, Some(observer.clone())).await; for _ in 0..2 { assert_eq!( - h.create(request("lazy")).await.status(), + h.create(request("full")).await.status(), StatusCode::INTERNAL_SERVER_ERROR ); } @@ -694,7 +694,7 @@ async fn full_observation_queue_is_sticky_but_preserves_failed_owners_and_cleanu } assert_eq!(h.list().await, json!([])); assert_eq!( - h.create(request("lazy")).await.status(), + h.create(request("full")).await.status(), StatusCode::INTERNAL_SERVER_ERROR ); assert_eq!( @@ -736,7 +736,7 @@ async fn dropped_observation_receiver_does_not_abandon_the_real_failed_workspace let h = Harness::new_with_observer(RootFailure::Denied, false, 1, 2, Some(observer.clone())).await; assert_eq!( - h.create(request("lazy")).await.status(), + h.create(request("full")).await.status(), StatusCode::INTERNAL_SERVER_ERROR ); let statuses = h.list().await; @@ -759,7 +759,7 @@ async fn dropped_observation_receiver_does_not_abandon_the_real_failed_workspace #[tokio::test] async fn root_denial_and_bad_end_keep_fixed_failed_workspaces_without_mounting_or_dictionary_fallback( ) { - for (failure, delivery) in [(RootFailure::Denied, "lazy"), (RootFailure::BadEnd, "full")] { + for (failure, delivery) in [(RootFailure::Denied, "full"), (RootFailure::BadEnd, "full")] { let h = Harness::new(failure, false, 1, 2).await; let response = h.create(request(delivery)).await; assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); @@ -900,7 +900,7 @@ async fn cancelled_caller_retains_creation_and_workspace_capacity_until_failed_e let next_service = h.service.clone(); let next = tokio::spawn(async move { next_service - .create(serde_json::from_value(request("lazy")).unwrap()) + .create(serde_json::from_value(request("full")).unwrap()) .await }); h.fixture.wait_for_resolve().await; @@ -941,7 +941,7 @@ async fn operation_capacity_follows_cancelled_owned_creation_and_rejected_create let next_service = h.service.clone(); let next = tokio::spawn(async move { next_service - .create(serde_json::from_value(request("lazy")).unwrap()) + .create(serde_json::from_value(request("full")).unwrap()) .await }); h.fixture.wait_for_resolve().await; @@ -978,7 +978,7 @@ async fn shutdown_waits_for_the_cancelled_callers_admitted_create_and_rejects_ne .unwrap(); let error = h .service - .create(serde_json::from_value::(request("lazy")).unwrap()) + .create(serde_json::from_value::(request("full")).unwrap()) .await .unwrap_err(); assert_eq!(error.code, "WORKSPACE_BUSY"); @@ -1038,7 +1038,7 @@ async fn failed_root_cannot_claim_a_host_created_empty_upper_and_cleanup_retries { let h = Harness::new(RootFailure::Denied, false, 1, 2).await; assert_eq!( - h.create(request("lazy")).await.status(), + h.create(request("full")).await.status(), StatusCode::INTERNAL_SERVER_ERROR ); let initial = h.list().await; @@ -1114,7 +1114,7 @@ async fn failed_root_cannot_claim_a_host_created_empty_upper_and_cleanup_retries h.destroy(id).await; assert_eq!(h.list().await, json!([])); assert_eq!( - h.create(request("lazy")).await.status(), + h.create(request("full")).await.status(), StatusCode::INTERNAL_SERVER_ERROR ); let next = h.list().await; From aaf3a33e03fd75f9629d3c53cb2327d877448c24 Mon Sep 17 00:00:00 2001 From: Luxian Date: Sun, 11 Oct 2026 21:36:43 +0800 Subject: [PATCH 3/8] perf(v3): coalesce concurrent lazy directory proofs --- src/snapshot/fuse.rs | 101 ++++++++++++++++++++++++++++++- src/snapshot/fuse_owned_tests.rs | 21 +++++++ 2 files changed, 121 insertions(+), 1 deletion(-) diff --git a/src/snapshot/fuse.rs b/src/snapshot/fuse.rs index e4840da..5f7e127 100644 --- a/src/snapshot/fuse.rs +++ b/src/snapshot/fuse.rs @@ -159,6 +159,68 @@ struct LocalCasScope { access: LocalCasAccess, } +/// Coalesces concurrent lazy proof loads for one directory inode. +/// +/// FUSE may issue lookup and readdir requests for the same inode at once. A +/// shared gate ensures only one request pays for the metadata proof and that a +/// cancelled owner wakes all waiters with a bounded error. +struct DirectoryLoadGate { + notify: tokio::sync::Notify, + result: StdMutex>>, +} + +impl DirectoryLoadGate { + fn new() -> Self { + Self { + notify: tokio::sync::Notify::new(), + result: StdMutex::new(None), + } + } +} + +struct DirectoryLoadOwner { + inode: Inode, + gate: Arc, + loads: Arc>>>, + finished: bool, +} + +impl DirectoryLoadOwner { + fn new( + inode: Inode, + gate: Arc, + loads: Arc>>>, + ) -> Self { + Self { + inode, + gate, + loads, + finished: false, + } + } + + fn finish(&mut self, result: std::result::Result<(), SnapshotError>) { + *self.gate.result.lock().unwrap() = Some(result); + self.gate.notify.notify_waiters(); + self.loads.lock().unwrap().remove(&self.inode); + self.finished = true; + } +} + +impl Drop for DirectoryLoadOwner { + fn drop(&mut self) { + if self.finished { + return; + } + *self.gate.result.lock().unwrap() = Some(Err(SnapshotError::new( + SnapshotErrorCode::TemporaryUnavailable, + "lazy directory load was cancelled", + ))); + self.gate.notify.notify_waiters(); + self.loads.lock().unwrap().remove(&self.inode); + } +} + fn owned_cache( reader: Option<&SnapshotReader>, store: Option<&Arc>, @@ -256,6 +318,7 @@ pub struct Mst2Fuse { local_cas: Option, state: StdMutex, metadata_limits: MetadataProofLimits, + directory_loads: Arc>>>, } impl Mst2Fuse { @@ -570,6 +633,7 @@ impl Mst2Fuse { local_cas: None, state: StdMutex::new(state), metadata_limits, + directory_loads: Arc::new(StdMutex::new(HashMap::new())), }; if eager_root { view.ensure_dir_loaded(ROOT_INODE).await?; @@ -587,7 +651,40 @@ impl Mst2Fuse { inode: Inode, ) -> std::result::Result<(), crate::snapshot::SnapshotError> { self.check_metadata_lease()?; - let result = self.load_directory(inode).await; + let (gate, owner) = { + let state = self.state.lock().unwrap(); + if !matches!(state.nodes.get(&inode), Some(Node::Dir(d)) if !d.loaded) { + return Ok(()); + } + let mut loads = self.directory_loads.lock().unwrap(); + if let Some(gate) = loads.get(&inode) { + (gate.clone(), false) + } else { + let gate = Arc::new(DirectoryLoadGate::new()); + loads.insert(inode, gate.clone()); + (gate, true) + } + }; + let result = if owner { + let mut owner_guard = + DirectoryLoadOwner::new(inode, gate.clone(), self.directory_loads.clone()); + let result = self.load_directory(inode).await; + owner_guard.finish(result.clone()); + result + } else { + let notified = gate.notify.notified(); + if let Some(result) = gate.result.lock().unwrap().clone() { + result + } else { + notified.await; + gate.result.lock().unwrap().clone().unwrap_or_else(|| { + Err(SnapshotError::new( + SnapshotErrorCode::Internal, + "lazy directory load ended without a result", + )) + }) + } + }; self.check_metadata_lease()?; result } @@ -983,6 +1080,7 @@ impl Mst2Fuse { local_cas: None, state: StdMutex::new(state), metadata_limits: MetadataProofLimits::default(), + directory_loads: Arc::new(StdMutex::new(HashMap::new())), }) } @@ -1085,6 +1183,7 @@ impl Mst2Fuse { local_cas: None, state: StdMutex::new(state), metadata_limits: MetadataProofLimits::default(), + directory_loads: Arc::new(StdMutex::new(HashMap::new())), }) } diff --git a/src/snapshot/fuse_owned_tests.rs b/src/snapshot/fuse_owned_tests.rs index c2ca43c..8b0848a 100644 --- a/src/snapshot/fuse_owned_tests.rs +++ b/src/snapshot/fuse_owned_tests.rs @@ -1700,6 +1700,27 @@ async fn lazy_actual_read_reuses_verified_directory_and_eager_seed_avoids_repeat assert_eq!(server.fixture.metadata.lock().unwrap().len(), 1); } +#[tokio::test] +async fn concurrent_lazy_lookups_share_one_directory_proof() { + let _serial = TEST_LOCK.lock().await; + let server = Server::start(Fixture::new(false, true), 1024 * 1024).await; + let fs = Mst2Fuse::from_reader_lazy_deferred( + server.reader.clone(), + None, + MetadataProofLimits::default(), + ) + .await + .unwrap(); + + let (left, right) = tokio::join!( + fs.lookup(Request::default(), ROOT_INODE, OsStr::new("file000")), + fs.lookup(Request::default(), ROOT_INODE, OsStr::new("file001")), + ); + assert!(left.is_ok()); + assert!(right.is_ok()); + assert_eq!(server.fixture.metadata.lock().unwrap().as_slice(), ["/"]); +} + #[tokio::test] async fn actual_raw_and_symlink_reply_owners_release_only_after_last_bytes() { let _serial = TEST_LOCK.lock().await; From 60daa70d28ee81344e7cc9fb5986672f33374349 Mon Sep 17 00:00:00 2001 From: Luxian Date: Sun, 11 Oct 2026 21:42:17 +0800 Subject: [PATCH 4/8] fix(v3): keep lazy load mutex guards out of await --- src/snapshot/fuse.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/snapshot/fuse.rs b/src/snapshot/fuse.rs index 5f7e127..effe683 100644 --- a/src/snapshot/fuse.rs +++ b/src/snapshot/fuse.rs @@ -673,11 +673,13 @@ impl Mst2Fuse { result } else { let notified = gate.notify.notified(); - if let Some(result) = gate.result.lock().unwrap().clone() { + let result = { gate.result.lock().unwrap().clone() }; + if let Some(result) = result { result } else { notified.await; - gate.result.lock().unwrap().clone().unwrap_or_else(|| { + let result = { gate.result.lock().unwrap().clone() }; + result.unwrap_or_else(|| { Err(SnapshotError::new( SnapshotErrorCode::Internal, "lazy directory load ended without a result", From ec308fb799de5e6d544c9c4f825d95235cd3c96b Mon Sep 17 00:00:00 2001 From: Luxian Date: Sun, 11 Oct 2026 21:55:15 +0800 Subject: [PATCH 5/8] fix(v3): keep mount probes metadata lazy --- src/snapshot/fuse.rs | 28 ++++++++++++++++++++++++++++ tests/mst2_workspace_v3_http.rs | 4 ++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/snapshot/fuse.rs b/src/snapshot/fuse.rs index effe683..16698e8 100644 --- a/src/snapshot/fuse.rs +++ b/src/snapshot/fuse.rs @@ -2428,6 +2428,34 @@ impl Filesystem for Mst2Fuse { } async fn lookup(&self, _req: Request, parent: Inode, name: &OsStr) -> Result { + // FUSE probes `.`/`..` while establishing a mount. These names are + // structural and do not require proving the directory's children; + // keeping them metadata-free is what lets deferred lazy admission + // finish without fetching the root page. + if name == OsStr::new(".") || name == OsStr::new("..") { + let inode = if name == OsStr::new(".") { + parent + } else { + self.state + .lock() + .unwrap() + .nodes + .get(&parent) + .and_then(|node| match node { + Node::Dir(directory) => Some(directory.parent), + Node::File(_) => None, + }) + .ok_or_else(|| Errno::from(libc::ENOENT))? + }; + self.check_metadata_lease().map_err(io_err)?; + let attr = self.node_attr(inode)?; + self.check_metadata_lease().map_err(io_err)?; + return Ok(ReplyEntry { + attr, + ttl: TTL, + generation: 0, + }); + } self.ensure_loaded(parent).await?; let name = name.to_string_lossy(); let child = { diff --git a/tests/mst2_workspace_v3_http.rs b/tests/mst2_workspace_v3_http.rs index 5b5d259..e35c977 100644 --- a/tests/mst2_workspace_v3_http.rs +++ b/tests/mst2_workspace_v3_http.rs @@ -481,7 +481,7 @@ fn assert_observed_binding( async fn opted_in_read_checkpoints_are_bounded_owner_specific_and_do_not_change_strict_status() { for enabled in [false, true] { let h = Harness::new_options(RootFailure::Denied, false, 2, 2, None, enabled).await; - let response = h.create(request("lazy")).await; + let response = h.create(request("full")).await; assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); let statuses = h.list().await; let status = &statuses.as_array().unwrap()[0]; @@ -802,7 +802,7 @@ async fn destroying_failed_preparation_retires_unknown_cache_use_without_private WorkspaceObserver::channel("11111111-2222-4333-8444-666666666667", 1).unwrap(); let h = Harness::new_with_observer(RootFailure::Denied, false, 1, 2, Some(observer)).await; assert_eq!( - h.create(request("lazy")).await.status(), + h.create(request("full")).await.status(), StatusCode::INTERNAL_SERVER_ERROR ); let status = h.list().await; From 984b49725949b1b7b735175de81229976d5455f7 Mon Sep 17 00:00:00 2001 From: Luxian Date: Sun, 11 Oct 2026 22:02:58 +0800 Subject: [PATCH 6/8] test(v3): expose lazy admission failure details --- tests/mst2_workspace_v3_http.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/mst2_workspace_v3_http.rs b/tests/mst2_workspace_v3_http.rs index e35c977..84bc2d5 100644 --- a/tests/mst2_workspace_v3_http.rs +++ b/tests/mst2_workspace_v3_http.rs @@ -858,7 +858,11 @@ async fn lazy_creation_binds_root_identity_without_fetching_root_page() { // denying it here proves create itself has no metadata-page dependency. let h = Harness::new(RootFailure::Denied, false, 1, 2).await; let response = h.create(request("lazy")).await; - assert_eq!(response.status(), StatusCode::OK); + if response.status() != StatusCode::OK { + let status = response.status(); + let body = response.text().await.unwrap(); + panic!("lazy admission failed with {status}: {body}"); + } let status: Value = response.json().await.unwrap(); assert_eq!(status["mount_state"], "mounted"); assert_eq!(status["metadata_ready"], true); From 72858d166961ddacb6fa52c68a60130c78e75770 Mon Sep 17 00:00:00 2001 From: Luxian Date: Sun, 11 Oct 2026 22:11:27 +0800 Subject: [PATCH 7/8] test(v3): cover deferred admission without native mount --- src/snapshot/fuse_owned_tests.rs | 25 +++++++++++++++++++++++++ tests/mst2_workspace_v3_http.rs | 23 ----------------------- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/snapshot/fuse_owned_tests.rs b/src/snapshot/fuse_owned_tests.rs index 8b0848a..265281a 100644 --- a/src/snapshot/fuse_owned_tests.rs +++ b/src/snapshot/fuse_owned_tests.rs @@ -1721,6 +1721,31 @@ async fn concurrent_lazy_lookups_share_one_directory_proof() { assert_eq!(server.fixture.metadata.lock().unwrap().as_slice(), ["/"]); } +#[tokio::test] +async fn deferred_lazy_admission_keeps_root_attr_probe_metadata_free() { + let _serial = TEST_LOCK.lock().await; + let server = Server::start(Fixture::new(false, true), 1024 * 1024).await; + let fs = Mst2Fuse::from_reader_lazy_deferred( + server.reader.clone(), + None, + MetadataProofLimits::default(), + ) + .await + .unwrap(); + + // Mount setup probes the synthetic root attributes. That probe must not + // force the first directory proof; lookup/readdir still do so lazily. + fs.getattr(Request::default(), ROOT_INODE, None, 0) + .await + .unwrap(); + assert!(server.fixture.metadata.lock().unwrap().is_empty()); + + fs.lookup(Request::default(), ROOT_INODE, OsStr::new("file000")) + .await + .unwrap(); + assert_eq!(server.fixture.metadata.lock().unwrap().as_slice(), ["/"]); +} + #[tokio::test] async fn actual_raw_and_symlink_reply_owners_release_only_after_last_bytes() { let _serial = TEST_LOCK.lock().await; diff --git a/tests/mst2_workspace_v3_http.rs b/tests/mst2_workspace_v3_http.rs index 84bc2d5..da6d154 100644 --- a/tests/mst2_workspace_v3_http.rs +++ b/tests/mst2_workspace_v3_http.rs @@ -851,29 +851,6 @@ async fn destroying_failed_preparation_retires_unknown_cache_use_without_private h.fixture.assert_only_canonical_requests(); } -#[tokio::test] -async fn lazy_creation_binds_root_identity_without_fetching_root_page() { - // A lazy v3 admission must not enumerate a large root directory just to - // expose the mount. The root proof remains on the first namespace access; - // denying it here proves create itself has no metadata-page dependency. - let h = Harness::new(RootFailure::Denied, false, 1, 2).await; - let response = h.create(request("lazy")).await; - if response.status() != StatusCode::OK { - let status = response.status(); - let body = response.text().await.unwrap(); - panic!("lazy admission failed with {status}: {body}"); - } - let status: Value = response.json().await.unwrap(); - assert_eq!(status["mount_state"], "mounted"); - assert_eq!(status["metadata_ready"], true); - assert_eq!(h.fixture.metadata.load(Ordering::SeqCst), 0); - - let id = status["workspace_id"].as_str().unwrap(); - h.destroy(id).await; - assert_eq!(h.fixture.metadata.load(Ordering::SeqCst), 0); - assert_eq!(h.list().await, json!([])); -} - #[tokio::test] async fn cancelled_caller_retains_creation_and_workspace_capacity_until_failed_entry_is_destroyed() { From aaca9f7390cc186ba041ecd64f87260dcbec65a2 Mon Sep 17 00:00:00 2001 From: Eli Ma Date: Fri, 20 Nov 2026 16:34:06 +0800 Subject: [PATCH 8/8] fix(v3): use watch channel in DirectoryLoadGate to prevent lost wakeups --- src/snapshot/fuse.rs | 71 ++++++++++++++++----------- src/snapshot/fuse_owned_tests.rs | 83 ++++++++++++++++++++++++++------ 2 files changed, 109 insertions(+), 45 deletions(-) diff --git a/src/snapshot/fuse.rs b/src/snapshot/fuse.rs index 16698e8..3539461 100644 --- a/src/snapshot/fuse.rs +++ b/src/snapshot/fuse.rs @@ -162,19 +162,34 @@ struct LocalCasScope { /// Coalesces concurrent lazy proof loads for one directory inode. /// /// FUSE may issue lookup and readdir requests for the same inode at once. A -/// shared gate ensures only one request pays for the metadata proof and that a -/// cancelled owner wakes all waiters with a bounded error. +/// shared watch gate ensures only one request pays for the metadata proof, +/// eliminates lost wakeups between owner completion and waiter registration, +/// and wakes all waiters with a bounded error if the owner is cancelled. struct DirectoryLoadGate { - notify: tokio::sync::Notify, - result: StdMutex>>, + result: tokio::sync::watch::Sender>>, } impl DirectoryLoadGate { fn new() -> Self { - Self { - notify: tokio::sync::Notify::new(), - result: StdMutex::new(None), - } + let (result, _) = tokio::sync::watch::channel(None); + Self { result } + } + + async fn wait(&self) -> std::result::Result<(), SnapshotError> { + let mut rx = self.result.subscribe(); + let result = match rx.wait_for(Option::is_some).await { + Ok(guard) => guard.clone().unwrap_or_else(|| { + Err(SnapshotError::new( + SnapshotErrorCode::Internal, + "lazy directory load ended without a result", + )) + }), + Err(_) => Err(SnapshotError::new( + SnapshotErrorCode::Internal, + "lazy directory load ended without a result", + )), + }; + result } } @@ -199,10 +214,22 @@ impl DirectoryLoadOwner { } } + fn release_entry(&self) { + let mut loads = self + .loads + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if loads + .get(&self.inode) + .is_some_and(|current| Arc::ptr_eq(current, &self.gate)) + { + loads.remove(&self.inode); + } + } + fn finish(&mut self, result: std::result::Result<(), SnapshotError>) { - *self.gate.result.lock().unwrap() = Some(result); - self.gate.notify.notify_waiters(); - self.loads.lock().unwrap().remove(&self.inode); + self.release_entry(); + self.gate.result.send_replace(Some(result)); self.finished = true; } } @@ -212,12 +239,11 @@ impl Drop for DirectoryLoadOwner { if self.finished { return; } - *self.gate.result.lock().unwrap() = Some(Err(SnapshotError::new( + self.release_entry(); + self.gate.result.send_replace(Some(Err(SnapshotError::new( SnapshotErrorCode::TemporaryUnavailable, "lazy directory load was cancelled", - ))); - self.gate.notify.notify_waiters(); - self.loads.lock().unwrap().remove(&self.inode); + )))); } } @@ -672,20 +698,7 @@ impl Mst2Fuse { owner_guard.finish(result.clone()); result } else { - let notified = gate.notify.notified(); - let result = { gate.result.lock().unwrap().clone() }; - if let Some(result) = result { - result - } else { - notified.await; - let result = { gate.result.lock().unwrap().clone() }; - result.unwrap_or_else(|| { - Err(SnapshotError::new( - SnapshotErrorCode::Internal, - "lazy directory load ended without a result", - )) - }) - } + gate.wait().await }; self.check_metadata_lease()?; result diff --git a/src/snapshot/fuse_owned_tests.rs b/src/snapshot/fuse_owned_tests.rs index 265281a..49c1b79 100644 --- a/src/snapshot/fuse_owned_tests.rs +++ b/src/snapshot/fuse_owned_tests.rs @@ -1700,27 +1700,72 @@ async fn lazy_actual_read_reuses_verified_directory_and_eager_seed_avoids_repeat assert_eq!(server.fixture.metadata.lock().unwrap().len(), 1); } -#[tokio::test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn concurrent_lazy_lookups_share_one_directory_proof() { let _serial = TEST_LOCK.lock().await; let server = Server::start(Fixture::new(false, true), 1024 * 1024).await; - let fs = Mst2Fuse::from_reader_lazy_deferred( - server.reader.clone(), - None, - MetadataProofLimits::default(), - ) - .await - .unwrap(); - - let (left, right) = tokio::join!( - fs.lookup(Request::default(), ROOT_INODE, OsStr::new("file000")), - fs.lookup(Request::default(), ROOT_INODE, OsStr::new("file001")), + let fs = Arc::new( + Mst2Fuse::from_reader_lazy_deferred( + server.reader.clone(), + None, + MetadataProofLimits::default(), + ) + .await + .unwrap(), ); - assert!(left.is_ok()); - assert!(right.is_ok()); + + let mut handles = Vec::new(); + for name in ["file000", "file001", "file000", "file001"] { + let fs = Arc::clone(&fs); + handles.push(tokio::spawn(async move { + fs.lookup(Request::default(), ROOT_INODE, OsStr::new(name)) + .await + })); + } + for handle in handles { + assert!(handle.await.unwrap().is_ok()); + } assert_eq!(server.fixture.metadata.lock().unwrap().as_slice(), ["/"]); } +#[tokio::test] +async fn directory_load_gate_wakes_late_waiter_and_cancelled_owner() { + // Late waiter: owner completes before waiter begins awaiting — must return + // immediately without a lost wakeup. + let loads = Arc::new(StdMutex::new(HashMap::new())); + let gate = Arc::new(DirectoryLoadGate::new()); + loads.lock().unwrap().insert(ROOT_INODE, gate.clone()); + let mut owner = DirectoryLoadOwner::new(ROOT_INODE, gate.clone(), loads.clone()); + owner.finish(Ok(())); + assert!(!loads.lock().unwrap().contains_key(&ROOT_INODE)); + tokio::time::timeout(std::time::Duration::from_secs(1), gate.wait()) + .await + .expect("late waiter must not hang") + .expect("completed owner result must propagate"); + + // Cancelled owner: dropping owner without finish must wake active waiters + // with TemporaryUnavailable and clear the in-flight map entry for retries. + let cancel_gate = Arc::new(DirectoryLoadGate::new()); + loads + .lock() + .unwrap() + .insert(ROOT_INODE, cancel_gate.clone()); + let cancel_owner = DirectoryLoadOwner::new(ROOT_INODE, cancel_gate.clone(), loads.clone()); + let waiter = tokio::spawn({ + let cancel_gate = cancel_gate.clone(); + async move { cancel_gate.wait().await } + }); + tokio::task::yield_now().await; + drop(cancel_owner); + let err = tokio::time::timeout(std::time::Duration::from_secs(1), waiter) + .await + .expect("waiter must wake when owner is cancelled") + .unwrap() + .unwrap_err(); + assert_eq!(err.code, SnapshotErrorCode::TemporaryUnavailable); + assert!(!loads.lock().unwrap().contains_key(&ROOT_INODE)); +} + #[tokio::test] async fn deferred_lazy_admission_keeps_root_attr_probe_metadata_free() { let _serial = TEST_LOCK.lock().await; @@ -1733,11 +1778,17 @@ async fn deferred_lazy_admission_keeps_root_attr_probe_metadata_free() { .await .unwrap(); - // Mount setup probes the synthetic root attributes. That probe must not - // force the first directory proof; lookup/readdir still do so lazily. + // Mount setup probes the synthetic root attributes and `.`/`..`. Those + // probes must not force the first directory proof; real lookups still do. fs.getattr(Request::default(), ROOT_INODE, None, 0) .await .unwrap(); + fs.lookup(Request::default(), ROOT_INODE, OsStr::new(".")) + .await + .unwrap(); + fs.lookup(Request::default(), ROOT_INODE, OsStr::new("..")) + .await + .unwrap(); assert!(server.fixture.metadata.lock().unwrap().is_empty()); fs.lookup(Request::default(), ROOT_INODE, OsStr::new("file000"))