From 7b71b0f86677899ccc7f18f20c83ba0ed91f037f Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 9 Nov 2026 22:09:45 +0800 Subject: [PATCH] fix(cloudhypervisor): balloon settle counts progress only past 1% of guest memory per stall window Any drop in memory_actual_size reset the 2 s stall window, so a guest whose anon sat above the balloon target trickled pages and held the pause for up to the one-minute cap. A hibernate of such a guest took 14.7-59.7 s against 3.8-3.9 s without the balloon, and every exec on that VM waited it out. A settling golden frees far more than 1% per window and still reaches its target. --- docs/snapshots.md | 2 +- hypervisor/cloudhypervisor/snapshot.go | 10 ++++++---- hypervisor/cloudhypervisor/snapshot_test.go | 3 ++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/snapshots.md b/docs/snapshots.md index 70667420..15475181 100644 --- a/docs/snapshots.md +++ b/docs/snapshots.md @@ -33,7 +33,7 @@ A snapshot contains the full VM state: `snapshot save` and `vm hibernate` are refused while a vhost-user-fs share, a VFIO device or a hot-attached disk is present; see [Devices](devices.md). -On Cloud Hypervisor, `snapshot save` and `vm hibernate` first let a running guest finish inflating its balloon, waiting up to a minute and moving on once inflation stalls for 2 s (the wait reads the live target, so it covers a target raised by `vm balloon`), so a `vm restore` or hibernate wake comes back with a settled balloon instead of inflating it again at a high CPU cost. A clone moves to the create-time balloon target after restore: it deflates from a raised snapshot target, so a golden saved at the default target keeps the settled balloon. +On Cloud Hypervisor, `snapshot save` and `vm hibernate` first let a running guest finish inflating its balloon, waiting up to a minute and moving on once inflation frees less than 1% of guest memory in 2 s (the wait reads the live target, so it covers a target raised by `vm balloon`), so a `vm restore` or hibernate wake comes back with a settled balloon instead of inflating it again at a high CPU cost. A clone moves to the create-time balloon target after restore: it deflates from a raised snapshot target, so a golden saved at the default target keeps the settled balloon. ### Clone Constraints diff --git a/hypervisor/cloudhypervisor/snapshot.go b/hypervisor/cloudhypervisor/snapshot.go index 0f54581a..947675b8 100644 --- a/hypervisor/cloudhypervisor/snapshot.go +++ b/hypervisor/cloudhypervisor/snapshot.go @@ -21,6 +21,7 @@ import ( const ( balloonPollInterval = 100 * time.Millisecond balloonStallWindow = 2 * time.Second + balloonStallPercent = 1 balloonSettleTimeout = time.Minute ejectRenotifyInterval = 5 * time.Second @@ -182,8 +183,9 @@ func waitBalloonSettled(ctx context.Context, hc *http.Client, vmID string, info return nil } target := info.Config.Memory.Size - balloon.Size + step := info.Config.Memory.Size * balloonStallPercent / 100 start := time.Now() - progressAt := start + progressAt, progressFrom := start, info.MemoryActualSize for actual := info.MemoryActualSize; actual > target; { if time.Since(progressAt) >= balloonStallWindow || time.Since(start) >= balloonSettleTimeout { log.WithFunc("cloudhypervisor.waitBalloonSettled").Warnf(ctx, "snapshot VM %s with balloon at %d of %d bytes", vmID, info.Config.Memory.Size-actual, balloon.Size) @@ -198,10 +200,10 @@ func waitBalloonSettled(ctx context.Context, hc *http.Client, vmID string, info if err != nil { return err } - if cur.MemoryActualSize < actual { - progressAt = time.Now() - } actual = cur.MemoryActualSize + if progressFrom-actual >= step { + progressAt, progressFrom = time.Now(), actual + } } return nil } diff --git a/hypervisor/cloudhypervisor/snapshot_test.go b/hypervisor/cloudhypervisor/snapshot_test.go index ab5e3587..f07f0062 100644 --- a/hypervisor/cloudhypervisor/snapshot_test.go +++ b/hypervisor/cloudhypervisor/snapshot_test.go @@ -246,7 +246,8 @@ func TestSnapshotPauseWaitsForBalloonToSettle(t *testing.T) { {"paused guest cannot inflate", chStatePaused, &chBalloon{Size: balloon}, func(int) int64 { return mem }, 0}, {"inflating reaches target", chStateRunning, &chBalloon{Size: balloon}, func(polls int) int64 { return mem - min(int64(polls)*(512<<20), balloon) }, 4 * balloonPollInterval}, {"stalled under guest memory pressure", chStateRunning, &chBalloon{Size: balloon}, func(int) int64 { return mem - (512 << 20) }, balloonStallWindow}, - {"slow inflation stops at the cap", chStateRunning, &chBalloon{Size: balloon}, func(polls int) int64 { return mem - int64(polls)<<12 }, balloonSettleTimeout}, + {"trickling inflation counts as stalled", chStateRunning, &chBalloon{Size: balloon}, func(polls int) int64 { return mem - int64(polls)<<12 }, balloonStallWindow}, + {"slow inflation stops at the cap", chStateRunning, &chBalloon{Size: mem * 3 / 4}, func(polls int) int64 { return mem - int64(polls)*(mem/1000) }, balloonSettleTimeout}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) {