Skip to content
171 changes: 169 additions & 2 deletions src/snapshot/fuse.rs
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,94 @@ struct LocalCasScope {
access: LocalCasAccess,
}

/// Coalesces concurrent lazy proof loads for one directory inode.
///
/// FUSE may issue lookup and readdir requests for the same inode at once. A
/// shared watch gate ensures only one request pays for the metadata proof,
/// eliminates lost wakeups between owner completion and waiter registration,
/// and wakes all waiters with a bounded error if the owner is cancelled.
struct DirectoryLoadGate {
result: tokio::sync::watch::Sender<Option<std::result::Result<(), SnapshotError>>>,
}

impl DirectoryLoadGate {
fn new() -> Self {
let (result, _) = tokio::sync::watch::channel(None);
Self { result }
}

async fn wait(&self) -> std::result::Result<(), SnapshotError> {
let mut rx = self.result.subscribe();
let result = match rx.wait_for(Option::is_some).await {
Ok(guard) => guard.clone().unwrap_or_else(|| {
Err(SnapshotError::new(
SnapshotErrorCode::Internal,
"lazy directory load ended without a result",
))
}),
Err(_) => Err(SnapshotError::new(
SnapshotErrorCode::Internal,
"lazy directory load ended without a result",
)),
};
result
}
}

struct DirectoryLoadOwner {
inode: Inode,
gate: Arc<DirectoryLoadGate>,
loads: Arc<StdMutex<HashMap<Inode, Arc<DirectoryLoadGate>>>>,
finished: bool,
}

impl DirectoryLoadOwner {
fn new(
inode: Inode,
gate: Arc<DirectoryLoadGate>,
loads: Arc<StdMutex<HashMap<Inode, Arc<DirectoryLoadGate>>>>,
) -> Self {
Self {
inode,
gate,
loads,
finished: false,
}
}

fn release_entry(&self) {
let mut loads = self
.loads
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
if loads
.get(&self.inode)
.is_some_and(|current| Arc::ptr_eq(current, &self.gate))
{
loads.remove(&self.inode);
}
}

fn finish(&mut self, result: std::result::Result<(), SnapshotError>) {
self.release_entry();
self.gate.result.send_replace(Some(result));
self.finished = true;
}
}

impl Drop for DirectoryLoadOwner {
fn drop(&mut self) {
if self.finished {
return;
}
self.release_entry();
self.gate.result.send_replace(Some(Err(SnapshotError::new(
SnapshotErrorCode::TemporaryUnavailable,
"lazy directory load was cancelled",
))));
}
}

fn owned_cache(
reader: Option<&SnapshotReader>,
store: Option<&Arc<DurableStore>>,
Expand Down Expand Up @@ -256,6 +344,7 @@ pub struct Mst2Fuse {
local_cas: Option<LocalCasScope>,
state: StdMutex<State>,
metadata_limits: MetadataProofLimits,
directory_loads: Arc<StdMutex<HashMap<Inode, Arc<DirectoryLoadGate>>>>,
}

impl Mst2Fuse {
Expand Down Expand Up @@ -489,6 +578,29 @@ impl Mst2Fuse {
reader: SnapshotReader,
store: Option<Arc<DurableStore>>,
metadata_limits: MetadataProofLimits,
) -> std::result::Result<Self, SnapshotError> {
Self::from_reader_lazy_with_limits_inner(reader, store, metadata_limits, true).await
}

/// Build a lazy view without proving the root page during admission.
///
/// A v3 workspace only needs the immutable metadata root identity before
/// exposing its mount. The root page is still fully verified when the
/// kernel first asks for lookup/readdir/opendir, but deferring that proof
/// keeps workspace creation independent of the size of the root directory.
pub(crate) async fn from_reader_lazy_deferred(
reader: SnapshotReader,
store: Option<Arc<DurableStore>>,
metadata_limits: MetadataProofLimits,
) -> std::result::Result<Self, SnapshotError> {
Self::from_reader_lazy_with_limits_inner(reader, store, metadata_limits, false).await
}

async fn from_reader_lazy_with_limits_inner(
reader: SnapshotReader,
store: Option<Arc<DurableStore>>,
metadata_limits: MetadataProofLimits,
eager_root: bool,
) -> std::result::Result<Self, SnapshotError> {
if metadata_limits.max_directory_pages == 0
|| metadata_limits.max_directory_entries == 0
Expand Down Expand Up @@ -547,8 +659,11 @@ impl Mst2Fuse {
local_cas: None,
state: StdMutex::new(state),
metadata_limits,
directory_loads: Arc::new(StdMutex::new(HashMap::new())),
};
view.ensure_dir_loaded(ROOT_INODE).await?;
if eager_root {
view.ensure_dir_loaded(ROOT_INODE).await?;
}
Ok(view)
}

Expand All @@ -562,7 +677,29 @@ impl Mst2Fuse {
inode: Inode,
) -> std::result::Result<(), crate::snapshot::SnapshotError> {
self.check_metadata_lease()?;
let result = self.load_directory(inode).await;
let (gate, owner) = {
let state = self.state.lock().unwrap();
if !matches!(state.nodes.get(&inode), Some(Node::Dir(d)) if !d.loaded) {
return Ok(());
}
let mut loads = self.directory_loads.lock().unwrap();
if let Some(gate) = loads.get(&inode) {
(gate.clone(), false)
} else {
let gate = Arc::new(DirectoryLoadGate::new());
loads.insert(inode, gate.clone());
(gate, true)
}
};
let result = if owner {
let mut owner_guard =
DirectoryLoadOwner::new(inode, gate.clone(), self.directory_loads.clone());
let result = self.load_directory(inode).await;
owner_guard.finish(result.clone());
result
} else {
gate.wait().await
};
self.check_metadata_lease()?;
result
}
Expand Down Expand Up @@ -958,6 +1095,7 @@ impl Mst2Fuse {
local_cas: None,
state: StdMutex::new(state),
metadata_limits: MetadataProofLimits::default(),
directory_loads: Arc::new(StdMutex::new(HashMap::new())),
})
}

Expand Down Expand Up @@ -1060,6 +1198,7 @@ impl Mst2Fuse {
local_cas: None,
state: StdMutex::new(state),
metadata_limits: MetadataProofLimits::default(),
directory_loads: Arc::new(StdMutex::new(HashMap::new())),
})
}

Expand Down Expand Up @@ -2302,6 +2441,34 @@ impl Filesystem for Mst2Fuse {
}

async fn lookup(&self, _req: Request, parent: Inode, name: &OsStr) -> Result<ReplyEntry> {
// FUSE probes `.`/`..` while establishing a mount. These names are
// structural and do not require proving the directory's children;
// keeping them metadata-free is what lets deferred lazy admission
// finish without fetching the root page.
if name == OsStr::new(".") || name == OsStr::new("..") {
let inode = if name == OsStr::new(".") {
parent
} else {
self.state
.lock()
.unwrap()
.nodes
.get(&parent)
.and_then(|node| match node {
Node::Dir(directory) => Some(directory.parent),
Node::File(_) => None,
})
.ok_or_else(|| Errno::from(libc::ENOENT))?
};
self.check_metadata_lease().map_err(io_err)?;
let attr = self.node_attr(inode)?;
self.check_metadata_lease().map_err(io_err)?;
return Ok(ReplyEntry {
attr,
ttl: TTL,
generation: 0,
});
}
self.ensure_loaded(parent).await?;
let name = name.to_string_lossy();
let child = {
Expand Down
97 changes: 97 additions & 0 deletions src/snapshot/fuse_owned_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1700,6 +1700,103 @@ async fn lazy_actual_read_reuses_verified_directory_and_eager_seed_avoids_repeat
assert_eq!(server.fixture.metadata.lock().unwrap().len(), 1);
}

#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn concurrent_lazy_lookups_share_one_directory_proof() {
let _serial = TEST_LOCK.lock().await;
let server = Server::start(Fixture::new(false, true), 1024 * 1024).await;
let fs = Arc::new(
Mst2Fuse::from_reader_lazy_deferred(
server.reader.clone(),
None,
MetadataProofLimits::default(),
)
.await
.unwrap(),
);

let mut handles = Vec::new();
for name in ["file000", "file001", "file000", "file001"] {
let fs = Arc::clone(&fs);
handles.push(tokio::spawn(async move {
fs.lookup(Request::default(), ROOT_INODE, OsStr::new(name))
.await
}));
}
for handle in handles {
assert!(handle.await.unwrap().is_ok());
}
assert_eq!(server.fixture.metadata.lock().unwrap().as_slice(), ["/"]);
}

#[tokio::test]
async fn directory_load_gate_wakes_late_waiter_and_cancelled_owner() {
// Late waiter: owner completes before waiter begins awaiting — must return
// immediately without a lost wakeup.
let loads = Arc::new(StdMutex::new(HashMap::new()));
let gate = Arc::new(DirectoryLoadGate::new());
loads.lock().unwrap().insert(ROOT_INODE, gate.clone());
let mut owner = DirectoryLoadOwner::new(ROOT_INODE, gate.clone(), loads.clone());
owner.finish(Ok(()));
assert!(!loads.lock().unwrap().contains_key(&ROOT_INODE));
tokio::time::timeout(std::time::Duration::from_secs(1), gate.wait())
.await
.expect("late waiter must not hang")
.expect("completed owner result must propagate");

// Cancelled owner: dropping owner without finish must wake active waiters
// with TemporaryUnavailable and clear the in-flight map entry for retries.
let cancel_gate = Arc::new(DirectoryLoadGate::new());
loads
.lock()
.unwrap()
.insert(ROOT_INODE, cancel_gate.clone());
let cancel_owner = DirectoryLoadOwner::new(ROOT_INODE, cancel_gate.clone(), loads.clone());
let waiter = tokio::spawn({
let cancel_gate = cancel_gate.clone();
async move { cancel_gate.wait().await }
});
tokio::task::yield_now().await;
drop(cancel_owner);
let err = tokio::time::timeout(std::time::Duration::from_secs(1), waiter)
.await
.expect("waiter must wake when owner is cancelled")
.unwrap()
.unwrap_err();
assert_eq!(err.code, SnapshotErrorCode::TemporaryUnavailable);
assert!(!loads.lock().unwrap().contains_key(&ROOT_INODE));
}

#[tokio::test]
async fn deferred_lazy_admission_keeps_root_attr_probe_metadata_free() {
let _serial = TEST_LOCK.lock().await;
let server = Server::start(Fixture::new(false, true), 1024 * 1024).await;
let fs = Mst2Fuse::from_reader_lazy_deferred(
server.reader.clone(),
None,
MetadataProofLimits::default(),
)
.await
.unwrap();

// Mount setup probes the synthetic root attributes and `.`/`..`. Those
// probes must not force the first directory proof; real lookups still do.
fs.getattr(Request::default(), ROOT_INODE, None, 0)
.await
.unwrap();
fs.lookup(Request::default(), ROOT_INODE, OsStr::new("."))
.await
.unwrap();
fs.lookup(Request::default(), ROOT_INODE, OsStr::new(".."))
.await
.unwrap();
assert!(server.fixture.metadata.lock().unwrap().is_empty());

fs.lookup(Request::default(), ROOT_INODE, OsStr::new("file000"))
.await
.unwrap();
assert_eq!(server.fixture.metadata.lock().unwrap().as_slice(), ["/"]);
}

#[tokio::test]
async fn actual_raw_and_symlink_reply_owners_release_only_after_last_bytes() {
let _serial = TEST_LOCK.lock().await;
Expand Down
33 changes: 24 additions & 9 deletions src/workspace/service.rs
Original file line number Diff line number Diff line change
Expand Up @@ -483,16 +483,31 @@ impl WorkspaceService {
&self.config.cache_root,
);
}
// Lazy admission binds only the immutable metadata root identity.
// Full delivery keeps the eager root proof so hydration failures are
// reported by create; lazy mounts prove a directory on first access.
let lower = Arc::new(
trace_async(
"root_metadata_proof",
Mst2Fuse::from_reader_lazy_with_limits(
reader,
Some(store),
self.config.metadata_limits,
),
)
.await?
(if matches!(request.delivery, WorkspaceDelivery::Lazy) {
trace_async(
"root_metadata_bind",
Mst2Fuse::from_reader_lazy_deferred(
Comment on lines +490 to +493

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Update the remaining eager-failure fixtures

For a denied root with delivery: lazy, this branch now skips the request that produced the expected error and successfully mounts the workspace, as the new regression test demonstrates. However, opted_in_read_checkpoints_are_bounded_owner_specific_and_do_not_change_strict_status (tests/mst2_workspace_v3_http.rs:483-488) and destroying_failed_preparation_retires_unknown_cache_use_without_private_directories (tests/mst2_workspace_v3_http.rs:803-816) still submit lazy requests and require a 500 response plus failed/no-mount state, so both tests will deterministically fail on Linux; make those remaining eager-failure fixtures use full delivery or rewrite their assertions for successful lazy admission.

Useful? React with 👍 / 👎.

reader,
Some(store),
self.config.metadata_limits,
),
)
.await?
} else {
trace_async(
"root_metadata_proof",
Mst2Fuse::from_reader_lazy_with_limits(
reader,
Some(store),
self.config.metadata_limits,
),
)
.await?
})
.with_read_profile(runtime.read_profile.clone()),
);
runtime.lower = Some(lower.clone());
Expand Down
Loading
Loading