From 103f8c6fb96534bed3f2999b3e5c55fad005fba6 Mon Sep 17 00:00:00 2001 From: gspark-etri Date: Mon, 2 Nov 2026 22:50:19 +0900 Subject: [PATCH 01/16] test(csi): exercise owner admission races and consumed-base recovery --- .github/workflows/ci.yml | 4 +- pkg/csi/owner_protocol_test.go | 257 +++++++++++++++++++++++++++++++++ 2 files changed, 260 insertions(+), 1 deletion(-) create mode 100644 pkg/csi/owner_protocol_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c220c3d..a2c3a1b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,9 @@ jobs: with: go-version-file: go.mod - name: gofmt - run: test -z "$(gofmt -l .)" || { gofmt -l .; exit 1; } + run: test -z "$(gofmt -l .)" || { gofmt -d .; exit 1; } + - name: Owner admission and post-consumption recovery witnesses + run: go test -race -count=1 -timeout=2m -run '^TestOwner' -v ./pkg/csi - run: go vet ./... - run: go test -race ./... - name: E11 payload accounting and validity regressions diff --git a/pkg/csi/owner_protocol_test.go b/pkg/csi/owner_protocol_test.go new file mode 100644 index 0000000..2f6afff --- /dev/null +++ b/pkg/csi/owner_protocol_test.go @@ -0,0 +1,257 @@ +package csi + +import ( + "context" + "errors" + "io/fs" + "os" + "path/filepath" + "testing" + "time" + + "github.com/ai-computing/onggi/pkg/digest" + "github.com/ai-computing/onggi/pkg/store" +) + +func ageOwnerDonor(t *testing.T, m *Materializer, d digest.Digest) { + t.Helper() + old := time.Unix(1, 0) + if err := os.Chtimes(m.usedFile(d), old, old); err != nil { + t.Fatal(err) + } +} + +// A stale offer cannot override a lease acquired before its actual claim. +func TestOwnerLeaseAfterOfferRefusesConsumption(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + m, c := newMaterializer(t, f) + m.RebaseIdle = time.Hour + if _, err := m.Ensure(ctx, f.a); err != nil { + t.Fatal(err) + } + ageOwnerDonor(t, m, f.a) + before, err := os.Stat(filepath.Join(m.Dir(f.a), "model.bin")) + if err != nil { + t.Fatal(err) + } + offer, claimed := m.offer(f.b) + if offer == nil || len(offer.Bases) != 1 || offer.Bases[0].Manifest != f.a { + t.Fatal("expected the idle A donor to be offered") + } + if err := m.Acquire("late-owner-lease", f.a, "/late-owner-target"); err != nil { + t.Fatal(err) + } + // Keep age eligible so refusal is due to the lease rather than recency. + ageOwnerDonor(t, m, f.a) + if dir, ok := offer.Claim(offer.Bases[0]); ok || dir != "" || len(*claimed) != 0 { + t.Fatalf("stale offer consumed a leased donor: dir=%q ok=%v", dir, ok) + } + if users, err := m.users(f.a); err != nil || users != 1 { + t.Fatalf("lease lost: users=%d err=%v", users, err) + } + dir, err := m.Ensure(ctx, f.b) + if err != nil { + t.Fatal(err) + } + sameTree(t, f.srcB, dir) + sameTree(t, f.srcA, m.Dir(f.a)) + after, err := os.Stat(filepath.Join(m.Dir(f.a), "model.bin")) + if err != nil || !os.SameFile(before, after) || c.pulls[1].Base != "" { + t.Fatalf("leased donor changed or was consumed: err=%v stats=%+v", err, c.pulls[1]) + } + t.Log("offer preceded lease; saved claim refused; lease/inode/base bytes retained; separate target verified") +} + +// Two distinct target offers contend on the real owner claim, not a stub callback. +func TestOwnerCompetingDonorClaims(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + m, _ := newMaterializer(t, f) + m.RebaseIdle = time.Hour + m.MaxPulls = 2 + if _, err := m.Ensure(ctx, f.a); err != nil { + t.Fatal(err) + } + ageOwnerDonor(t, m, f.a) + otherSource := t.TempDir() + shared, err := os.ReadFile(filepath.Join(f.srcA, "model.bin")) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(otherSource, "model.bin"), append(shared, random(3, 100_000)...), 0o644); err != nil { + t.Fatal(err) + } + other, _, err := f.s.Push(ctx, otherSource, store.PushOptions{}) + if err != nil { + t.Fatal(err) + } + first, claimedFirst := m.offer(f.b) + second, claimedSecond := m.offer(other) + for _, offer := range []*Offer{first, second} { + if offer == nil || len(offer.Bases) != 1 || offer.Bases[0].Manifest != f.a { + t.Fatal("both distinct target offers must name A") + } + } + before, err := os.Stat(filepath.Join(m.Dir(f.a), "model.bin")) + if err != nil { + t.Fatal(err) + } + type result struct { + dir string + ok bool + } + start := make(chan struct{}) + results := make(chan result, 2) + for _, offer := range []*Offer{first, second} { + go func(o *Offer) { + <-start + dir, ok := o.Claim(o.Bases[0]) + results <- result{dir: dir, ok: ok} + }(offer) + } + close(start) + winner := "" + wins := 0 + for range 2 { + select { + case r := <-results: + if r.ok { + wins++ + winner = r.dir + } else if r.dir != "" { + t.Fatalf("refused claim returned a directory: %q", r.dir) + } + case <-time.After(10 * time.Second): + t.Fatal("competing claims did not return") + } + } + if wins != 1 || len(*claimedFirst)+len(*claimedSecond) != 1 || isDir(m.Dir(f.a)) { + t.Fatalf("donor ownership moved incorrectly: wins=%d first=%v second=%v", wins, *claimedFirst, *claimedSecond) + } + sameTree(t, f.srcA, winner) + after, err := os.Stat(filepath.Join(winner, "model.bin")) + if err != nil || !os.SameFile(before, after) { + t.Fatalf("winner did not receive the original donor inode: %v", err) + } + for _, offer := range []*Offer{first, second} { + if _, ok := offer.Claim(offer.Bases[0]); ok { + t.Fatal("a later claim took the already-transferred donor") + } + } + sameTree(t, f.srcA, winner) + t.Log("two real claims released together; exactly one move; winner bytes/inode intact after repeated refusal") +} + +func ownerPackPaths(t *testing.T, root string) map[string]bool { + t.Helper() + paths := make(map[string]bool) + err := filepath.WalkDir(filepath.Join(root, "packs"), func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.Type().IsRegular() { + paths[path] = true + } + return nil + }) + if err != nil { + t.Fatal(err) + } + return paths +} + +// Actual residual corruption follows a successful real claim; retry has no donor. +func TestOwnerPostConsumptionFailureThenRetry(t *testing.T) { + ctx := context.Background() + storeDir := t.TempDir() + f := &fixture{s: openStore(t, storeDir), srcA: t.TempDir(), srcB: t.TempDir()} + shared := random(71, 3<<20) + if err := os.WriteFile(filepath.Join(f.srcA, "model.bin"), shared, 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(f.srcB, "model.bin"), append(shared, random(72, 100_000)...), 0o644); err != nil { + t.Fatal(err) + } + var err error + f.a, _, err = f.s.Push(ctx, f.srcA, store.PushOptions{}) + if err != nil { + t.Fatal(err) + } + basePacks := ownerPackPaths(t, storeDir) + f.b, _, err = f.s.Push(ctx, f.srcB, store.PushOptions{}) + if err != nil { + t.Fatal(err) + } + var residual []string + for path := range ownerPackPaths(t, storeDir) { + if !basePacks[path] { + residual = append(residual, path) + } + } + if len(residual) != 1 { + t.Fatalf("expected one target-only residual pack, got %d", len(residual)) + } + pack := residual[0] + original, err := os.ReadFile(pack) + if err != nil || len(original) == 0 { + t.Fatalf("read residual pack: %v", err) + } + m, _ := newMaterializer(t, f) + m.RebaseIdle = time.Hour + if _, err := m.Ensure(ctx, f.a); err != nil { + t.Fatal(err) + } + ageOwnerDonor(t, m, f.a) + // No cache/peer can mask the corrupted target-only store bytes. + realPull := StorePuller(f.s, nil, filepath.Join(m.Root, "tmp"), 2) + claimed := false + var injectionErr, pullErr error + var failedStats store.PullStats + m.Pull = func(ctx context.Context, target digest.Digest, dest string, offer *Offer) (store.PullStats, error) { + if offer != nil { + realClaim := offer.Claim + offer.Claim = func(base store.Base) (string, bool) { + dir, ok := realClaim(base) + if ok { + claimed = true + corrupt := append([]byte(nil), original...) + corrupt[len(corrupt)/2] ^= 0xff + injectionErr = os.Chmod(pack, 0o644) + if injectionErr == nil { + injectionErr = os.WriteFile(pack, corrupt, 0o644) + } + } + return dir, ok + } + } + failedStats, pullErr = realPull(ctx, target, dest, offer) + return failedStats, pullErr + } + _, err = m.Ensure(ctx, f.b) + if !claimed || injectionErr != nil || pullErr == nil || !errors.Is(err, pullErr) { + t.Fatalf("expected actual post-claim construction failure: claimed=%v injection=%v pull=%v ensure=%v", claimed, injectionErr, pullErr, err) + } + if failedStats.Base != f.a.String() || failedStats.Reused == 0 { + t.Fatalf("failure preceded genuine consumption/preservation: %+v", failedStats) + } + if isDir(m.Dir(f.a)) || isDir(m.Dir(f.b)) { + t.Fatal("consumed donor or incomplete target remained published") + } + if entries, err := os.ReadDir(filepath.Join(m.Root, "tmp")); err != nil || len(entries) != 0 { + t.Fatalf("private/consumed cleanup failed: entries=%v err=%v", entries, err) + } + if err := os.WriteFile(pack, original, 0o644); err != nil { + t.Fatal(err) + } + m.Pull = realPull + dir, err := m.Ensure(ctx, f.b) + if err != nil { + t.Fatal(err) + } + sameTree(t, f.srcB, dir) + if isDir(m.Dir(f.a)) { + t.Fatal("retry manufactured a replacement donor") + } + t.Logf("real consumed-base residual failure=%v reused=%d; no completed target; donor-free retry bytes verified", pullErr, failedStats.Reused) +} From 4b32c966908d85202f6ded07da590128604a6615 Mon Sep 17 00:00:00 2001 From: gspark-etri Date: Mon, 2 Nov 2026 22:55:28 +0900 Subject: [PATCH 02/16] Require the selected residual pack hash failure in recovery witness --- pkg/csi/owner_protocol_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/csi/owner_protocol_test.go b/pkg/csi/owner_protocol_test.go index 2f6afff..97cc2e5 100644 --- a/pkg/csi/owner_protocol_test.go +++ b/pkg/csi/owner_protocol_test.go @@ -6,6 +6,7 @@ import ( "io/fs" "os" "path/filepath" + "strings" "testing" "time" @@ -232,6 +233,9 @@ func TestOwnerPostConsumptionFailureThenRetry(t *testing.T) { if !claimed || injectionErr != nil || pullErr == nil || !errors.Is(err, pullErr) { t.Fatalf("expected actual post-claim construction failure: claimed=%v injection=%v pull=%v ensure=%v", claimed, injectionErr, pullErr, err) } + if !strings.Contains(pullErr.Error(), "fails its hash check") || !strings.Contains(pullErr.Error(), filepath.Base(pack)) { + t.Fatalf("expected the selected corrupted residual pack's hash failure: pack=%s err=%v", filepath.Base(pack), pullErr) + } if failedStats.Base != f.a.String() || failedStats.Reused == 0 { t.Fatalf("failure preceded genuine consumption/preservation: %+v", failedStats) } From 2d0ed885c391aa12ac92fb86dc1c038f5dce35cc Mon Sep 17 00:00:00 2001 From: gspark-etri Date: Wed, 11 Nov 2026 10:11:58 +0900 Subject: [PATCH 03/16] Take a base over only in place of an eviction, and check what is kept A pull that takes an idle artifact over changes its files in place. Four gaps around that are closed: - Claim refuses a base that has a file with a second hard link, which would otherwise change outside the artifact and outside every lease. - Materializer.MaxArtifacts ties consumption to eviction: below the limit nothing is consumed, at the limit the least recently used idle artifact is offered first, and Evict removes least recently used idle artifacts above the limit. A free slot is no longer filled by destroying a warm artifact. 0 keeps the previous behaviour. - PullOptions.VerifyKept reads back every chunk a taken-over base leaves in place and checks its hash. A chunk that fails is fetched like any other, so a base damaged after it was pulled is repaired, not carried into the new artifact. PullStats reports Verified and Repaired. - PullOptions.BaseCopy decides when a base that is not taken over is read where it is. BaseCopyRemote reads it only if the store is not local (Capabilities.Local): on 15-18 GB artifacts a local store took 32-35 s where reading the base took 44-47 s. The library default is unchanged (BaseCopyAlways); the CSI plugin defaults to -base-copy=remote. Base reads (and the new read-back) join chunks that follow one another in a file into single reads of up to MaxRange, instead of one read per chunk. Tests: go test -race ./pkg/... ./cmd/... passes on linux/amd64. No timing was measured for the joined reads or for VerifyKept. Co-Authored-By: Claude Fable 5.2 --- cmd/onggi/csi.go | 15 ++++ pkg/backend/backend.go | 2 + pkg/backend/fs/fs.go | 2 +- pkg/csi/capacity_test.go | 143 +++++++++++++++++++++++++++++++++++++ pkg/csi/evict.go | 75 ++++++++++++++----- pkg/csi/links_other.go | 6 ++ pkg/csi/links_unix.go | 35 +++++++++ pkg/csi/links_unix_test.go | 67 +++++++++++++++++ pkg/csi/materializer.go | 38 +++++++++- pkg/store/pull.go | 31 +++++++- pkg/store/rebase.go | 99 ++++++++++++------------- pkg/store/selector_test.go | 142 ++++++++++++++++++++++++++++++++++++ pkg/store/spans.go | 101 ++++++++++++++++++++++++++ 13 files changed, 686 insertions(+), 70 deletions(-) create mode 100644 pkg/csi/capacity_test.go create mode 100644 pkg/csi/links_other.go create mode 100644 pkg/csi/links_unix.go create mode 100644 pkg/csi/links_unix_test.go create mode 100644 pkg/store/selector_test.go create mode 100644 pkg/store/spans.go diff --git a/cmd/onggi/csi.go b/cmd/onggi/csi.go index 3c86d4e..341baef 100644 --- a/cmd/onggi/csi.go +++ b/cmd/onggi/csi.go @@ -38,6 +38,9 @@ func csiNode(args []string, stderr io.Writer) int { keepUsed := fs.Duration("keep-used", 7*24*time.Hour, "the node cache keeps what artifacts used within this period need") gcGrace := fs.Duration("gc-grace", time.Hour, "grace period of the node cache's GC") rebaseIdle := fs.Duration("rebase-idle", 5*time.Minute, "a pull may turn an artifact no volume used for this long into the new one, writing only what differs (0: never)") + maxArtifacts := fs.Int("max-artifacts", 0, "artifacts kept per node; a pull takes an idle one over only when the node holds this many, least recently used first (0: no limit)") + verifyKept := fs.Bool("verify-kept", false, "read back and hash-check every chunk a pull leaves in place when it takes an artifact over") + baseCopy := fs.String("base-copy", "remote", "read an idle artifact that is not taken over where it is: remote (only if the store is not local), always, never") peerListen := fs.String("peer-listen", "", "serve the node cache to other nodes' pulls on this address, e.g. :7070 (#37)") peers := fs.String("peers", "", "comma-separated peer URLs asked for chunks before the store") peersDNS := fs.String("peers-dns", "", "host:port whose name resolves to the peers, e.g. a headless Service; re-resolved every 30 s") @@ -106,6 +109,18 @@ func csiNode(args []string, stderr io.Writer) int { } mounter := csi.NewMounter() m.RebaseIdle, m.Unmount = *rebaseIdle, mounter.Unmount + m.MaxArtifacts, m.VerifyKept = *maxArtifacts, *verifyKept + switch *baseCopy { + case "remote": + m.BaseCopy = store.BaseCopyRemote + case "always": + m.BaseCopy = store.BaseCopyAlways + case "never": + m.BaseCopy = store.BaseCopyNever + default: + logger.Printf("-base-copy: %q is not remote, always or never", *baseCopy) + return 2 + } n := &csi.Node{ Version: version, NodeID: *nodeID, diff --git a/pkg/backend/backend.go b/pkg/backend/backend.go index eb03630..5defad2 100644 --- a/pkg/backend/backend.go +++ b/pkg/backend/backend.go @@ -49,6 +49,8 @@ type Capabilities struct { RangeReads bool AtomicCreate bool // a second Put of the same key can never expose partial or mixed bytes Durability Durability + // Local says the objects are files on this machine, so a range read costs no round trip. + Local bool } // Backend stores immutable objects. diff --git a/pkg/backend/fs/fs.go b/pkg/backend/fs/fs.go index 40d71d1..778767f 100644 --- a/pkg/backend/fs/fs.go +++ b/pkg/backend/fs/fs.go @@ -180,7 +180,7 @@ func (s *Store) Delete(ctx context.Context, k backend.Key) error { // Caps: range reads; atomic create through link or rename. func (s *Store) Caps() backend.Capabilities { - return backend.Capabilities{RangeReads: true, AtomicCreate: true, Durability: s.durability} + return backend.Capabilities{RangeReads: true, AtomicCreate: true, Durability: s.durability, Local: true} } // ---- refs: a lock file per ref makes SetRef a compare-and-swap on a single mount. diff --git a/pkg/csi/capacity_test.go b/pkg/csi/capacity_test.go new file mode 100644 index 0000000..9839204 --- /dev/null +++ b/pkg/csi/capacity_test.go @@ -0,0 +1,143 @@ +package csi + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/ai-computing/onggi/pkg/digest" + "github.com/ai-computing/onggi/pkg/store" +) + +// Below MaxArtifacts a pull consumes nothing, however idle the other artifact is: the node has +// room for both. At the limit the idle artifact is taken over, and the node still holds as many +// artifacts as before. +func TestOwnerFreeSlotRefusesConsumption(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + m, c := newMaterializer(t, f) + m.RebaseIdle = time.Hour + m.MaxArtifacts = 2 + if _, err := m.Ensure(ctx, f.a); err != nil { + t.Fatal(err) + } + ageOwnerDonor(t, m, f.a) + dir, err := m.Ensure(ctx, f.b) + if err != nil { + t.Fatal(err) + } + sameTree(t, f.srcB, dir) + sameTree(t, f.srcA, m.Dir(f.a)) + if last := c.pulls[len(c.pulls)-1]; last.Base != "" { + t.Fatalf("a pull consumed an artifact although a slot was free: %+v", last) + } + if ds, err := m.Materialized(); err != nil || len(ds) != 2 { + t.Fatalf("materialized %v, %v; want both", ds, err) + } + + // The node is full now. Drop B and pull it again with the limit at one: A is consumed. + if err := os.RemoveAll(m.Dir(f.b)); err != nil { + t.Fatal(err) + } + m.MaxArtifacts = 1 + ageOwnerDonor(t, m, f.a) + if dir, err = m.Ensure(ctx, f.b); err != nil { + t.Fatal(err) + } + sameTree(t, f.srcB, dir) + if last := c.pulls[len(c.pulls)-1]; last.Base != f.a.String() { + t.Fatalf("a full node did not take the idle artifact over: %+v", last) + } + if ds, err := m.Materialized(); err != nil || len(ds) != 1 || ds[0] != f.b { + t.Fatalf("materialized %v, %v; want only B", ds, err) + } +} + +// Evict brings a node back under MaxArtifacts by removing the least recently used unused +// artifact, and leaves an artifact a volume uses alone even if that keeps the node over. +func TestEvictAboveMaxArtifacts(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + m, _ := newMaterializer(t, f) + if _, err := m.Ensure(ctx, f.a); err != nil { + t.Fatal(err) + } + if _, err := m.Ensure(ctx, f.b); err != nil { + t.Fatal(err) + } + now := time.Now() + older, newer := now.Add(-2*time.Minute), now.Add(-time.Minute) + if err := os.Chtimes(m.usedFile(f.a), older, older); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(m.usedFile(f.b), newer, newer); err != nil { + t.Fatal(err) + } + + // no limit: nothing is idle for an hour, nothing goes + if gone, err := m.Evict(now, time.Hour, nil); err != nil || len(gone) != 0 { + t.Fatalf("evicted %v, %v without a limit", gone, err) + } + // limit one, but the older artifact is in use: the newer, unused one goes instead + m.MaxArtifacts = 1 + if err := m.Acquire("vol-a", f.a, "/target-a"); err != nil { + t.Fatal(err) + } + gone, err := m.Evict(now, time.Hour, nil) + if err != nil || len(gone) != 1 || gone[0] != f.b { + t.Fatalf("evicted %v, %v; want the unused B", gone, err) + } + if _, err := os.Stat(filepath.Join(m.Dir(f.a), "model.bin")); err != nil { + t.Fatalf("the artifact in use was removed: %v", err) + } + + // both unused: the least recently used goes + if _, _, _, err := m.Release("vol-a"); err != nil { + t.Fatal(err) + } + if _, err := m.Ensure(ctx, f.b); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(m.usedFile(f.a), older, older); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(m.usedFile(f.b), newer, newer); err != nil { + t.Fatal(err) + } + gone, err = m.Evict(now, time.Hour, nil) + if err != nil || len(gone) != 1 || gone[0] != f.a { + t.Fatalf("evicted %v, %v; want the least recently used A", gone, err) + } +} + +// The materializer hands its policies to the pull. +func TestOfferCarriesPolicies(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + m, _ := newMaterializer(t, f) + m.RebaseIdle = time.Hour + m.VerifyKept, m.BaseCopy = true, store.BaseCopyRemote + if _, err := m.Ensure(ctx, f.a); err != nil { + t.Fatal(err) + } + ageOwnerDonor(t, m, f.a) + offer, _ := m.offer(f.b) + if offer == nil || !offer.VerifyKept || offer.BaseCopy != store.BaseCopyRemote { + t.Fatalf("offer %+v does not carry the policies", offer) + } + var seen store.PullStats + inner := m.Pull + m.Pull = func(ctx context.Context, d digest.Digest, dest string, o *Offer) (store.PullStats, error) { + st, err := inner(ctx, d, dest, o) + seen = st + return st, err + } + if _, err := m.Ensure(ctx, f.b); err != nil { + t.Fatal(err) + } + if seen.Base != f.a.String() || seen.Verified <= 0 || seen.Verified != seen.Reused { + t.Fatalf("the pull did not verify what it kept: %+v", seen) + } +} diff --git a/pkg/csi/evict.go b/pkg/csi/evict.go index 446502a..006366d 100644 --- a/pkg/csi/evict.go +++ b/pkg/csi/evict.go @@ -1,6 +1,7 @@ package csi import ( + "sort" "os" "path/filepath" "strings" @@ -61,7 +62,10 @@ func (m *Materializer) materialized() ([]digest.Digest, error) { return out, nil } -// Evict removes the artifacts that no volume uses and that nothing used for at least idle. +// Evict removes the artifacts that no volume uses and that nothing used for at least idle. With +// MaxArtifacts set it then removes the least recently used of the remaining unused artifacts +// until the node holds at most that many; artifacts a volume or a pull uses always stay, so the +// node can be over the limit while they are all in use. // unmount is called on the directory first: on Linux it holds the read-only self-bind that // publishing made. The directory leaves artifacts/ by a rename under the materializer's lock, // so Ensure either finds it complete or pulls it again; deleting the files happens after. @@ -74,38 +78,73 @@ func (m *Materializer) Evict(now time.Time, idle time.Duration, unmount func(dir } var gone []digest.Digest var trash []string - for _, d := range ds { - if _, busy := m.inflight[d]; busy { - continue - } - if n, err := m.users(d); err != nil || n > 0 { - continue - } - if now.Sub(m.lastUsed(d)) < idle { - continue - } + over := map[digest.Digest]bool{} + // remove takes d out of artifacts/. ok false: it stays (logged); err: stop. + remove := func(d digest.Digest) (bool, error) { dir := m.Dir(d) if unmount != nil { if err := unmount(dir); err != nil { m.logf("evict %s: %v", d, err) - continue + return false, nil } } t, err := os.MkdirTemp(filepath.Join(m.Root, "tmp"), "evict-") if err != nil { - m.mu.Unlock() - return gone, err + return false, err } dst := filepath.Join(t, d.Hex()) if err := os.Rename(dir, dst); err != nil { os.Remove(t) m.logf("evict %s: %v", d, err) - continue + return false, nil } os.Remove(m.usedFile(d)) os.Remove(filepath.Join(m.Root, "leases", d.Hex())) trash = append(trash, t) gone = append(gone, d) + return true, nil + } + left := 0 + var spare []digest.Digest // unused, but used too recently for idle + for _, d := range ds { + if _, busy := m.inflight[d]; busy { + left++ + continue + } + if n, err := m.users(d); err != nil || n > 0 { + left++ + continue + } + if now.Sub(m.lastUsed(d)) < idle { + left++ + spare = append(spare, d) + continue + } + ok, err := remove(d) + if err != nil { + m.mu.Unlock() + return gone, err + } + if !ok { + left++ + } + } + if m.MaxArtifacts > 0 && left > m.MaxArtifacts { + sort.Slice(spare, func(i, j int) bool { return m.lastUsed(spare[i]).Before(m.lastUsed(spare[j])) }) + for _, d := range spare { + if left <= m.MaxArtifacts { + break + } + ok, err := remove(d) + if err != nil { + m.mu.Unlock() + return gone, err + } + if ok { + left-- + over[d] = true + } + } } m.mu.Unlock() for _, t := range trash { @@ -114,7 +153,11 @@ func (m *Materializer) Evict(now time.Time, idle time.Duration, unmount func(dir } } for _, d := range gone { - m.logf("evicted %s (no volume, idle %s or more)", d, idle) + if over[d] { + m.logf("evicted %s (no volume, least recently used above the limit of %d)", d, m.MaxArtifacts) + } else { + m.logf("evicted %s (no volume, idle %s or more)", d, idle) + } } return gone, nil } diff --git a/pkg/csi/links_other.go b/pkg/csi/links_other.go new file mode 100644 index 0000000..7698814 --- /dev/null +++ b/pkg/csi/links_other.go @@ -0,0 +1,6 @@ +//go:build !unix + +package csi + +// aliased reports no hard links where the platform does not expose link counts. +func aliased(string) (string, error) { return "", nil } diff --git a/pkg/csi/links_unix.go b/pkg/csi/links_unix.go new file mode 100644 index 0000000..3db22e8 --- /dev/null +++ b/pkg/csi/links_unix.go @@ -0,0 +1,35 @@ +//go:build unix + +package csi + +import ( + "io/fs" + "path/filepath" + "syscall" +) + +// aliased returns a regular file under root that has more than one hard link, or "" if there is +// none. A pull that takes a base over changes its files in place; a second link to one of them +// (a snapshot, a copy made with cp -l) would change with it, outside the artifact and outside +// every lease. +func aliased(root string) (string, error) { + found := "" + err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.Type().IsRegular() { + return nil + } + fi, err := d.Info() + if err != nil { + return err + } + if st, ok := fi.Sys().(*syscall.Stat_t); ok && st.Nlink > 1 { + found = p + return fs.SkipAll + } + return nil + }) + return found, err +} diff --git a/pkg/csi/links_unix_test.go b/pkg/csi/links_unix_test.go new file mode 100644 index 0000000..0d83a35 --- /dev/null +++ b/pkg/csi/links_unix_test.go @@ -0,0 +1,67 @@ +//go:build unix + +package csi + +import ( + "bytes" + "context" + "os" + "path/filepath" + "testing" + "time" +) + +// A base whose file has a second hard link is not taken over: changing it in place would change +// the other name too. The pull then builds the target without the base, and both stay intact. +func TestOwnerHardLinkedDonorRefusesConsumption(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + m, c := newMaterializer(t, f) + m.RebaseIdle = time.Hour + if _, err := m.Ensure(ctx, f.a); err != nil { + t.Fatal(err) + } + ageOwnerDonor(t, m, f.a) + inside := filepath.Join(m.Dir(f.a), "model.bin") + want, err := os.ReadFile(inside) + if err != nil { + t.Fatal(err) + } + // The second name lives under the materializer root so that both are on one filesystem. + outside := filepath.Join(m.Root, "snapshot-model.bin") + if err := os.Link(inside, outside); err != nil { + t.Skipf("hard links unavailable here: %v", err) + } + if p, err := aliased(m.Dir(f.a)); err != nil || p != inside { + t.Fatalf("aliased = %q, %v; want %q", p, err, inside) + } + + offer, claimed := m.offer(f.b) + if offer == nil || len(offer.Bases) != 1 || offer.Bases[0].Manifest != f.a { + t.Fatal("expected the idle A donor to be offered") + } + if dir, ok := offer.Claim(offer.Bases[0]); ok || dir != "" || len(*claimed) != 0 { + t.Fatalf("a hard-linked donor was consumed: dir=%q ok=%v", dir, ok) + } + + dir, err := m.Ensure(ctx, f.b) + if err != nil { + t.Fatal(err) + } + sameTree(t, f.srcB, dir) + sameTree(t, f.srcA, m.Dir(f.a)) + if got, err := os.ReadFile(outside); err != nil || !bytes.Equal(got, want) { + t.Fatalf("the second link changed: err=%v", err) + } + if last := c.pulls[len(c.pulls)-1]; last.Base != "" { + t.Fatalf("the pull took over a hard-linked base: %+v", last) + } + + // Without the second link the same donor is consumed as before. + if err := os.Remove(outside); err != nil { + t.Fatal(err) + } + if p, err := aliased(m.Dir(f.a)); err != nil || p != "" { + t.Fatalf("aliased after unlink = %q, %v", p, err) + } +} diff --git a/pkg/csi/materializer.go b/pkg/csi/materializer.go index 59b86e9..3bb4ee5 100644 --- a/pkg/csi/materializer.go +++ b/pkg/csi/materializer.go @@ -29,6 +29,9 @@ type Puller func(ctx context.Context, m digest.Digest, dest string, offer *Offer type Offer struct { Bases []store.Base Claim func(store.Base) (dir string, ok bool) + // VerifyKept and BaseCopy are passed to the pull (store.PullOptions). + VerifyKept bool + BaseCopy store.BaseCopy } // StorePuller pulls from s through the node cache (nil: no cache), assembling cache packs in tmp. @@ -57,6 +60,7 @@ func StorePullerPeers(s *store.Store, openCache func(context.Context) (*store.St } if offer != nil { o.Bases, o.Claim = offer.Bases, offer.Claim + o.VerifyKept, o.BaseCopy = offer.VerifyKept, offer.BaseCopy } return s.Pull(ctx, m, dest, o) } @@ -84,6 +88,18 @@ type Materializer struct { // MaxPulls bounds distinct artifacts materialized concurrently (0: one). // Set before the first Ensure; retries for one digest still share one pull. MaxPulls int + // MaxArtifacts is how many artifacts the node keeps (0: no limit). With a limit, a pull takes + // an idle artifact over only when the node already holds that many, least recently used + // first: the artifact it consumes is the one that would be evicted to make room. Below the + // limit nothing is consumed, so a free slot is never filled by destroying a warm artifact. + // Evict removes the least recently used idle artifacts above the limit. + MaxArtifacts int + // VerifyKept makes a pull that takes an artifact over read back and hash-check every chunk it + // leaves in place (store.PullOptions.VerifyKept). + VerifyKept bool + // BaseCopy says when an idle artifact that is not taken over is read where it is + // (store.PullOptions.BaseCopy). + BaseCopy store.BaseCopy mu sync.Mutex inflight map[digest.Digest]*flight @@ -390,7 +406,15 @@ func (m *Materializer) offer(d digest.Digest) (*Offer, *[]string) { } return nil, claimed } - sort.Slice(bases, func(i, j int) bool { return m.lastUsed(bases[i].Manifest).After(m.lastUsed(bases[j].Manifest)) }) + // Most recently used first; with a limit, least recently used first, so that among equally + // good bases the pull consumes the one eviction would remove. + sort.Slice(bases, func(i, j int) bool { + a, b := m.lastUsed(bases[i].Manifest), m.lastUsed(bases[j].Manifest) + if m.MaxArtifacts > 0 { + return a.Before(b) + } + return a.After(b) + }) if len(bases) > 3 { bases = bases[:3] } @@ -400,7 +424,17 @@ func (m *Materializer) offer(d digest.Digest) (*Offer, *[]string) { if !m.idle(b.Manifest, time.Now()) { return "", false } + if m.MaxArtifacts > 0 { + if ds, err := m.materialized(); err != nil || len(ds) < m.MaxArtifacts { + return "", false // a free slot: consume nothing + } + } dir := m.Dir(b.Manifest) + // A file with a second hard link would change outside the artifact too; leave it alone. + if p, err := aliased(dir); err != nil || p != "" { + m.logf("rebase from %s refused: %q has another hard link (err %v)", b.Manifest, p, err) + return "", false + } if m.Unmount != nil { if err := m.Unmount(dir); err != nil { m.logf("rebase from %s: %v", b.Manifest, err) @@ -422,7 +456,7 @@ func (m *Materializer) offer(d digest.Digest) (*Offer, *[]string) { *claimed = append(*claimed, t) return dst, true } - return &Offer{Bases: bases, Claim: claim}, claimed + return &Offer{Bases: bases, Claim: claim, VerifyKept: m.VerifyKept, BaseCopy: m.BaseCopy}, claimed } // idle says whether artifact d may be consumed: materialized, no volume, no pull, unused for diff --git a/pkg/store/pull.go b/pkg/store/pull.go index a90931d..cc26427 100644 --- a/pkg/store/pull.go +++ b/pkg/store/pull.go @@ -37,8 +37,28 @@ type PullOptions struct { Bases []Base Claim func(Base) (dir string, ok bool) MinSame float64 + // VerifyKept makes a pull that takes a base over read back every chunk it leaves in place and + // check its hash, so nothing reaches the new artifact unchecked. A chunk that fails (the base + // was damaged after it was pulled) is fetched like any other. + VerifyKept bool + // BaseCopy says when a base that is not taken over is read where it is (#58). + BaseCopy BaseCopy } +// BaseCopy is the rule for reading a base that stays intact. +type BaseCopy int + +const ( + // BaseCopyAlways reads the base whenever it holds MinSame of the artifact anywhere. + BaseCopyAlways BaseCopy = iota + // BaseCopyRemote reads the base only if the store is not local. Packs on this machine are + // read in large ranges and decoded faster than a base is read and hashed chunk by chunk: + // on 15-18 GB artifacts a local store took 32-35 s where the base took 44-47 s. + BaseCopyRemote + // BaseCopyNever never reads a base that is not taken over. + BaseCopyNever +) + // PullStats reports a pull. type PullStats struct { Files int `json:"files"` @@ -50,6 +70,8 @@ type PullStats struct { Cached int64 `json:"cached"` // stored bytes added to the node cache Base string `json:"base,omitempty"` // the artifact this pull turned into the new one Reused int64 `json:"reused"` // bytes left in place in the base's files + Verified int64 `json:"verified"` // of those, bytes read back and hash-checked (VerifyKept) + Repaired int64 `json:"repaired"` // bytes of the base that failed that check and were fetched instead BaseBest int64 `json:"base_best"` // bytes the best offered base keeps at the same offsets, taken or not (#49) LocalBase string `json:"local_base,omitempty"` // a base read where it is, not consumed (#58) LocalBytes int64 `json:"local_bytes"` // bytes read from that base's files @@ -158,12 +180,17 @@ func (s *Store) Pull(ctx context.Context, manifest digest.Digest, dest string, o if st.Reused, err = rebase(p, dir, m, nodes, handles, done); err != nil { return st, err } + if o.VerifyKept { + st.Verified, st.Repaired = verifyKept(ctx, nodes, handles, done, o.Workers, o.MaxRange) + st.Reused -= st.Repaired + } } } // too little in place, but the base holds the bytes elsewhere: read them from it (#58) - if !claimed && okMost { + copyBase := o.BaseCopy == BaseCopyAlways || o.BaseCopy == BaseCopyRemote && !s.B.Caps().Local + if !claimed && okMost && copyBase { st.LocalBase = most.base.Manifest.String() - if st.LocalBytes, err = fromBase(ctx, most, nodes, done, handles, o.Workers); err != nil { + if st.LocalBytes, err = fromBase(ctx, most, nodes, done, handles, o.Workers, o.MaxRange); err != nil { return st, err } } diff --git a/pkg/store/rebase.go b/pkg/store/rebase.go index 76ab229..1b945d8 100644 --- a/pkg/store/rebase.go +++ b/pkg/store/rebase.go @@ -110,32 +110,22 @@ func (s *Store) chooseBase(ctx context.Context, m *Manifest, newExt [][]Extent, } // fromBase reads, from the base's files where they are, the stored chunks a pull still needs -// that the base holds anywhere (#58). The base is not consumed. Each chunk is hash-checked; one -// that cannot be read or does not match is left for the other sources. It returns the bytes read. -func fromBase(ctx context.Context, p rebasePlan, nodes []node, done []bool, handles []*os.File, workers int) (int64, error) { - type job struct { - node int32 - file int - off int64 - } - var jobs []job +// that the base holds anywhere (#58). The base is not consumed. Chunks that follow one another in +// a base file are read together. Each chunk is hash-checked; one that cannot be read or does not +// match is left for the other sources. It returns the bytes read. +func fromBase(ctx context.Context, p rebasePlan, nodes []node, done []bool, handles []*os.File, workers int, maxRange int64) (int64, error) { + var jobs []rjob for i := range nodes { if !nodes[i].stored || done[i] { continue } if f, e, ok := p.find(nodes[i].chunk); ok && e.Len == nodes[i].loc.Length { - jobs = append(jobs, job{int32(i), f, e.Off}) + jobs = append(jobs, rjob{int32(i), f, e.Off, int(e.Len)}) } } if len(jobs) == 0 { return 0, nil } - slices.SortFunc(jobs, func(a, b job) int { // read each base file front to back - if a.file != b.file { - return a.file - b.file - } - return cmp.Compare(a.off, b.off) - }) files := make([]*os.File, len(p.files)) defer func() { for _, f := range files { @@ -156,42 +146,53 @@ func fromBase(ctx context.Context, p rebasePlan, nodes []node, done []bool, hand var n atomic.Int64 var mu sync.Mutex var firstErr error - var wg sync.WaitGroup - w := max(workers, 1) - per := (len(jobs) + w - 1) / w - for a := 0; a < len(jobs); a += per { - part := jobs[a:min(a+per, len(jobs))] - wg.Add(1) - go func() { - defer wg.Done() - var buf []byte - for _, j := range part { - if ctx.Err() != nil { - return - } - nd := &nodes[j.node] - if cap(buf) < int(nd.loc.Length) { - buf = make([]byte, nd.loc.Length) - } - b := buf[:nd.loc.Length] - if _, err := files[j.file].ReadAt(b, j.off); err != nil || digest.Of(b) != nd.chunk { - continue - } - if err := emit(nodes, j.node, b, 0, handles); err != nil { - mu.Lock() - firstErr = cmp.Or(firstErr, err) - mu.Unlock() - return - } - done[j.node] = true - n.Add(int64(len(b))) - } - }() - } - wg.Wait() + readSpans(ctx, files, jobs, workers, maxRange, func(j rjob, b []byte) { + if b == nil || digest.Of(b) != nodes[j.node].chunk { + return + } + if err := emit(nodes, j.node, b, 0, handles); err != nil { + mu.Lock() + firstErr = cmp.Or(firstErr, err) + mu.Unlock() + return + } + done[j.node] = true + n.Add(int64(len(b))) + }) return n.Load(), firstErr } +// verifyKept reads back, from the files of the new artifact, every chunk a taken-over base left +// in place, and checks its hash. A chunk that fails in any of its places is no longer done, so +// the pull fetches it and writes it everywhere it goes. It returns the bytes that passed and the +// bytes that did not. +func verifyKept(ctx context.Context, nodes []node, handles []*os.File, done []bool, workers int, maxRange int64) (ok, bad int64) { + var jobs []rjob + for i := range nodes { + if !nodes[i].stored || !done[i] { + continue + } + places(nodes, int32(i), 0, func(t target) { + jobs = append(jobs, rjob{int32(i), int(t.file), t.off, int(nodes[i].loc.Length)}) + }) + } + failed := make([]atomic.Bool, len(nodes)) + readSpans(ctx, handles, jobs, workers, maxRange, func(j rjob, b []byte) { + if b == nil || digest.Of(b) != nodes[j.node].chunk { + failed[j.node].Store(true) + } + }) + for _, j := range jobs { + if failed[j.node].Load() { + done[j.node] = false + bad += int64(j.n) + } else { + ok += int64(j.n) + } + } + return ok, bad +} + // sameAt says whether base holds e's chunk at e's offset. func sameAt(base []Extent, e Extent) bool { k, ok := slices.BinarySearchFunc(base, e.Off, func(x Extent, off int64) int { diff --git a/pkg/store/selector_test.go b/pkg/store/selector_test.go new file mode 100644 index 0000000..c17401e --- /dev/null +++ b/pkg/store/selector_test.go @@ -0,0 +1,142 @@ +package store + +import ( + "context" + "os" + "path/filepath" + "testing" +) + +// A base damaged after it was pulled is not carried into the new artifact: with VerifyKept the +// chunk that no longer matches is fetched, and only the rest is left in place. +func TestRebaseVerifiesWhatItKeeps(t *testing.T) { + ctx := context.Background() + s := openTwoLevel(t, t.TempDir()) + w := random(41, 8<<20) + v := append([]byte(nil), w...) + copy(v[3<<20:], random(42, 200<<10)) + b, nm, src := rebaseFixture(t, s, + map[string][]byte{"decode_B2.rbln": w}, + map[string][]byte{"decode_B8.rbln": v}) + + // flip bytes of the base in a region both artifacts share + f, err := os.OpenFile(filepath.Join(b.Dir, "decode_B2.rbln"), os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + if _, err := f.WriteAt([]byte("damaged after the pull"), 6<<20); err != nil { + t.Fatal(err) + } + f.Close() + + claim := func(x Base) (string, bool) { return x.Dir, true } + out := t.TempDir() + st, err := s.Pull(ctx, nm, out, PullOptions{Bases: []Base{b}, Claim: claim, VerifyKept: true}) + if err != nil { + t.Fatal(err) + } + sameTree(t, src, out) + t.Logf("%+v", st) + if st.Base != b.Manifest.String() || st.Repaired <= 0 || st.Repaired > 1<<20 || st.Verified < int64(len(v))*8/10 { + t.Fatalf("damaged chunk not repaired, or too much distrusted: %+v", st) + } + if st.Reused+st.Repaired > int64(len(v)) || st.Reused != st.Verified { + t.Fatalf("kept bytes and verified bytes disagree: %+v", st) + } +} + +// Without VerifyKept the pull reads nothing back, as before. +func TestRebaseDoesNotVerifyByDefault(t *testing.T) { + ctx := context.Background() + s := openTwoLevel(t, t.TempDir()) + w := random(43, 4<<20) + v := append([]byte(nil), w...) + copy(v[1<<20:], random(44, 100<<10)) + b, nm, src := rebaseFixture(t, s, map[string][]byte{"a.rbln": w}, map[string][]byte{"a.rbln": v}) + out := t.TempDir() + st, err := s.Pull(ctx, nm, out, PullOptions{Bases: []Base{b}, Claim: func(x Base) (string, bool) { return x.Dir, true }}) + if err != nil { + t.Fatal(err) + } + sameTree(t, src, out) + if st.Verified != 0 || st.Repaired != 0 || st.Reused < int64(len(v))*8/10 { + t.Fatalf("%+v", st) + } +} + +// With a local store, BaseCopyRemote leaves a shifted base alone and reads the packs; the same +// pull under BaseCopyAlways reads the base. Both produce the artifact. +func TestBaseCopyRemoteSkipsALocalStore(t *testing.T) { + ctx := context.Background() + s := openTwoLevel(t, t.TempDir()) + if !s.B.Caps().Local { + t.Fatal("the fs backend must report itself local") + } + w := random(93, 8<<20) + v := append(random(94, 17952), w...) + b, nm, src := rebaseFixture(t, s, map[string][]byte{"rank0.engine": w}, map[string][]byte{"rank0.engine": v}) + never := func(Base) (string, bool) { + t.Fatal("consumed a base that keeps nothing in place") + return "", false + } + for _, c := range []struct { + name string + rule BaseCopy + base bool + }{{"remote", BaseCopyRemote, false}, {"never", BaseCopyNever, false}, {"always", BaseCopyAlways, true}} { + out := t.TempDir() + st, err := s.Pull(ctx, nm, out, PullOptions{Bases: []Base{b}, Claim: never, BaseCopy: c.rule}) + if err != nil { + t.Fatal(c.name, err) + } + sameTree(t, src, out) + if got := st.LocalBase != ""; got != c.base { + t.Fatalf("%s: read the base = %v, want %v: %+v", c.name, got, c.base, st) + } + if !c.base && st.ReadBytes == 0 { + t.Fatalf("%s: nothing read from the store: %+v", c.name, st) + } + } + if _, err := os.Stat(filepath.Join(b.Dir, "rank0.engine")); err != nil { + t.Fatalf("the base was touched: %v", err) + } +} + +// Chunks that follow one another in a file are read together, and each still reaches fn alone. +func TestReadSpansJoinsNeighbours(t *testing.T) { + data := random(95, 1<<20) + p := filepath.Join(t.TempDir(), "f") + write(t, p, data) + f, err := os.Open(p) + if err != nil { + t.Fatal(err) + } + defer f.Close() + var jobs []rjob + for off := 0; off < len(data); off += 4096 { // contiguous, then a hole after each 64 KiB + if off/65536%2 == 1 { + continue + } + jobs = append(jobs, rjob{node: int32(off / 4096), off: int64(off), n: 4096}) + } + jobs = append(jobs, rjob{node: -1, off: int64(len(data)) - 100, n: 4096}) // runs past the end + got := make(map[int32]bool) + bad := 0 + done := make(chan struct{}, 1) + done <- struct{}{} + readSpans(context.Background(), []*os.File{f}, jobs, 4, 32<<10, func(j rjob, b []byte) { + <-done + defer func() { done <- struct{}{} }() + if b == nil { + bad++ + return + } + if string(b) != string(data[j.off:j.off+int64(j.n)]) { + t.Errorf("chunk at %d has the wrong bytes", j.off) + } + got[j.node] = true + }) + if bad != 1 || len(got) != len(jobs)-1 { + t.Fatalf("read %d of %d chunks, %d failed", len(got), len(jobs), bad) + } +} diff --git a/pkg/store/spans.go b/pkg/store/spans.go new file mode 100644 index 0000000..d187281 --- /dev/null +++ b/pkg/store/spans.go @@ -0,0 +1,101 @@ +package store + +import ( + "cmp" + "context" + "os" + "slices" + "sync" +) + +// rjob is one chunk to read from a file: n bytes at off. +type rjob struct { + node int32 + file int + off int64 + n int +} + +// readSpans reads the jobs from files, joining chunks that follow one another in a file into +// single reads of at most maxRange bytes, and calls fn with each chunk's bytes. A read of 15 GB +// in 64 KiB chunks is 230 000 system calls one by one and under a thousand joined. fn gets nil +// for a chunk whose read failed. It is called from several goroutines; b is only valid during +// the call. +func readSpans(ctx context.Context, files []*os.File, jobs []rjob, workers int, maxRange int64, fn func(j rjob, b []byte)) { + if len(jobs) == 0 { + return + } + if maxRange <= 0 { + maxRange = 16 << 20 + } + slices.SortFunc(jobs, func(a, b rjob) int { // each file front to back + if a.file != b.file { + return a.file - b.file + } + return cmp.Compare(a.off, b.off) + }) + type span struct{ a, b int } // jobs[a:b] + var spans []span + for k, j := range jobs { + if n := len(spans); n > 0 { + s := &spans[n-1] + first, prev := jobs[s.a], jobs[s.b-1] + if prev.file == j.file && prev.off+int64(prev.n) == j.off && j.off+int64(j.n)-first.off <= maxRange { + s.b = k + 1 + continue + } + } + spans = append(spans, span{k, k + 1}) + } + ch := make(chan span) + var wg sync.WaitGroup + for w := 0; w < max(workers, 1); w++ { + wg.Add(1) + go func() { + defer wg.Done() + var buf []byte + for s := range ch { + if ctx.Err() != nil { + continue + } + first, last := jobs[s.a], jobs[s.b-1] + size := int(last.off + int64(last.n) - first.off) + if cap(buf) < size { + buf = make([]byte, size) + } + b := buf[:size] + _, err := files[first.file].ReadAt(b, first.off) + for _, j := range jobs[s.a:s.b] { + if err != nil { + fn(j, nil) + continue + } + o := int(j.off - first.off) + fn(j, b[o:o+j.n]) + } + } + }() + } + for _, s := range spans { + ch <- s + } + close(ch) + wg.Wait() +} + +// places calls fn for every place chunk id goes: a file and the offset in it. +func places(nodes []node, id int32, base int64, fn func(target)) { + nd := &nodes[id] + if nd.hasT { + fn(target{nd.firstT.file, nd.firstT.off + base}) + for _, t := range nd.moreT { + fn(target{t.file, t.off + base}) + } + } + if nd.hasP { + places(nodes, nd.firstP.node, base+int64(nd.firstP.off), fn) + for _, e := range nd.moreP { + places(nodes, e.node, base+int64(e.off), fn) + } + } +} From 00456d4705ea5fde4ab5f0fe5ce1a0c880920d13 Mon Sep 17 00:00:00 2001 From: gspark-etri Date: Wed, 11 Nov 2026 10:21:19 +0900 Subject: [PATCH 04/16] cli: expose base policies on onggi pull; add a base read size onggi pull gains --verify-kept, --base-copy (always, remote, never), --base-intact (read --base where it is, never move its files away) and --base-range (largest single read from the base; 1 reads chunk by chunk). PullOptions.BaseRange carries the last one; 0 means MaxRange as before. The CSI plugin and the pull command share parseBaseCopy. These make the paths of the previous commit measurable from the command line: a base read chunk by chunk against the same read in joined ranges, and a taken-over base with and without the read-back. Tests: go test -race ./pkg/store/ ./pkg/csi/ ./cmd/... passes. Co-Authored-By: Claude Fable 5.2 --- cmd/onggi/csi.go | 10 ++-------- cmd/onggi/store.go | 25 ++++++++++++++++++++++++- pkg/store/pull.go | 10 ++++++++-- pkg/store/selector_test.go | 23 +++++++++++++++++++++++ 4 files changed, 57 insertions(+), 11 deletions(-) diff --git a/cmd/onggi/csi.go b/cmd/onggi/csi.go index 341baef..b77268c 100644 --- a/cmd/onggi/csi.go +++ b/cmd/onggi/csi.go @@ -110,14 +110,8 @@ func csiNode(args []string, stderr io.Writer) int { mounter := csi.NewMounter() m.RebaseIdle, m.Unmount = *rebaseIdle, mounter.Unmount m.MaxArtifacts, m.VerifyKept = *maxArtifacts, *verifyKept - switch *baseCopy { - case "remote": - m.BaseCopy = store.BaseCopyRemote - case "always": - m.BaseCopy = store.BaseCopyAlways - case "never": - m.BaseCopy = store.BaseCopyNever - default: + var ok bool + if m.BaseCopy, ok = parseBaseCopy(*baseCopy); !ok { logger.Printf("-base-copy: %q is not remote, always or never", *baseCopy) return 2 } diff --git a/cmd/onggi/store.go b/cmd/onggi/store.go index 21ba66c..6a61b9f 100644 --- a/cmd/onggi/store.go +++ b/cmd/onggi/store.go @@ -192,6 +192,10 @@ func pull(args []string, stdout, stderr io.Writer) int { peers := fs.String("peers", "", "comma-separated peer URLs (onggi peer serve) asked for chunks before the store") baseRef := fs.String("base", "", "an artifact already restored in --base-dir that the pull may turn into this one, writing only what differs; its files are moved away (#41)") baseDir := fs.String("base-dir", "", "the directory holding --base") + baseIntact := fs.Bool("base-intact", false, "never move --base's files away: read from them where they are") + baseCopy := fs.String("base-copy", "always", "read a base that is not taken over where it is: always, remote (only if the store is not local), never") + baseRange := fs.Int64("base-range", 0, "largest single read from the base's files in bytes (0: 16 MiB; 1: chunk by chunk)") + verifyKept := fs.Bool("verify-kept", false, "read back and hash-check every chunk left in place when --base is taken over") asJSON := fs.Bool("json", false, "print JSON") if err := fs.Parse(args); err != nil || fs.NArg() != 2 { fmt.Fprintln(stderr, "usage: onggi pull --store S [--cache DIR] [--peers URLS] [--base REF --base-dir DIR] ") @@ -220,8 +224,14 @@ func pull(args []string, stdout, stderr io.Writer) int { return 1 } o.Bases = []store.Base{{Manifest: bd, Dir: *baseDir}} - o.Claim = func(b store.Base) (string, bool) { return b.Dir, true } + o.Claim = func(b store.Base) (string, bool) { return b.Dir, !*baseIntact } } + var ok bool + if o.BaseCopy, ok = parseBaseCopy(*baseCopy); !ok { + fmt.Fprintf(stderr, "onggi: --base-copy: %q is not always, remote or never\n", *baseCopy) + return 2 + } + o.VerifyKept, o.BaseRange = *verifyKept, *baseRange if *cacheDir != "" { cb, err := bfs.Open(*cacheDir, backend.DurabilityNone) if err == nil { @@ -430,3 +440,16 @@ func diff(args []string, stdout, stderr io.Writer) int { } return 0 } + +// parseBaseCopy maps a --base-copy value to the rule. +func parseBaseCopy(v string) (store.BaseCopy, bool) { + switch v { + case "always": + return store.BaseCopyAlways, true + case "remote": + return store.BaseCopyRemote, true + case "never": + return store.BaseCopyNever, true + } + return store.BaseCopyAlways, false +} diff --git a/pkg/store/pull.go b/pkg/store/pull.go index cc26427..0b63e0f 100644 --- a/pkg/store/pull.go +++ b/pkg/store/pull.go @@ -43,6 +43,9 @@ type PullOptions struct { VerifyKept bool // BaseCopy says when a base that is not taken over is read where it is (#58). BaseCopy BaseCopy + // BaseRange is the largest single read from a base's files (0: MaxRange). Chunks that follow + // one another are read together up to this; 1 reads chunk by chunk. + BaseRange int64 } // BaseCopy is the rule for reading a base that stays intact. @@ -126,6 +129,9 @@ func (s *Store) Pull(ctx context.Context, manifest digest.Digest, dest string, o if o.MaxRange <= 0 { o.MaxRange = 16 << 20 } + if o.BaseRange <= 0 { + o.BaseRange = o.MaxRange + } m, err := s.ReadManifest(ctx, manifest) if err != nil { return PullStats{}, err @@ -181,7 +187,7 @@ func (s *Store) Pull(ctx context.Context, manifest digest.Digest, dest string, o return st, err } if o.VerifyKept { - st.Verified, st.Repaired = verifyKept(ctx, nodes, handles, done, o.Workers, o.MaxRange) + st.Verified, st.Repaired = verifyKept(ctx, nodes, handles, done, o.Workers, o.BaseRange) st.Reused -= st.Repaired } } @@ -190,7 +196,7 @@ func (s *Store) Pull(ctx context.Context, manifest digest.Digest, dest string, o copyBase := o.BaseCopy == BaseCopyAlways || o.BaseCopy == BaseCopyRemote && !s.B.Caps().Local if !claimed && okMost && copyBase { st.LocalBase = most.base.Manifest.String() - if st.LocalBytes, err = fromBase(ctx, most, nodes, done, handles, o.Workers, o.MaxRange); err != nil { + if st.LocalBytes, err = fromBase(ctx, most, nodes, done, handles, o.Workers, o.BaseRange); err != nil { return st, err } } diff --git a/pkg/store/selector_test.go b/pkg/store/selector_test.go index c17401e..44f177c 100644 --- a/pkg/store/selector_test.go +++ b/pkg/store/selector_test.go @@ -140,3 +140,26 @@ func TestReadSpansJoinsNeighbours(t *testing.T) { t.Fatalf("read %d of %d chunks, %d failed", len(got), len(jobs), bad) } } + +// Reading a base chunk by chunk and in joined ranges gives the same artifact and the same bytes. +func TestBaseRangeDoesNotChangeTheResult(t *testing.T) { + ctx := context.Background() + s := openTwoLevel(t, t.TempDir()) + w := random(96, 8<<20) + v := append(random(97, 17952), w...) + b, nm, src := rebaseFixture(t, s, map[string][]byte{"rank0.engine": w}, map[string][]byte{"rank0.engine": v}) + never := func(Base) (string, bool) { return "", false } + var local [2]int64 + for i, r := range []int64{1, 0} { + out := t.TempDir() + st, err := s.Pull(ctx, nm, out, PullOptions{Bases: []Base{b}, Claim: never, BaseRange: r}) + if err != nil { + t.Fatal(err) + } + sameTree(t, src, out) + local[i] = st.LocalBytes + } + if local[0] != local[1] || local[0] < int64(len(w))*9/10 { + t.Fatalf("bytes read from the base: chunk by chunk %d, joined %d", local[0], local[1]) + } +} From 6d78193c45e8a63f32d65a77049c710777c1aee1 Mon Sep 17 00:00:00 2001 From: gspark-etri Date: Wed, 11 Nov 2026 10:53:40 +0900 Subject: [PATCH 05/16] Separate what a pull may consume from what it may read The cache policy now picks what a pull consumes, and a pull may read an artifact that is in use. - With MaxArtifacts, only the eviction victim (the least recently used idle artifact) is offered for consumption. Before, the pull chose the best aligned of up to three idle artifacts, which could destroy an artifact the cache would have kept. - Offer.Readable lists up to three published artifacts, in use or not, that a pull may read where they are (PullOptions.ReadBases). Offer.Hold (PullOptions.Hold) keeps the one being read from being taken over or evicted until the read ends: Claim and Evict treat it as busy. - The CSI plugin now reads back and checks what a taken-over artifact leaves in place by default (-verify-kept=true). On a 15 GB artifact this added 1.7 s to a 4.9 s pull. - A one-level store can frame its chunks (Create.FrameFlat, --frame-flat), to compare flat chunking with two levels under the same compression. Tests added: a leased artifact is read and not consumed; a read hold blocks Claim and Evict; only the victim is offered under a limit; leases survive a new materializer on the same root; read-only bases are held and never claimed; a framed one-level store round-trips and is smaller. go test -race ./pkg/store/ ./pkg/csi/ ./cmd/... passes on linux/amd64. Co-Authored-By: Claude Fable 5.2 --- cmd/onggi/csi.go | 2 +- cmd/onggi/store.go | 9 ++ pkg/csi/candidates_test.go | 172 +++++++++++++++++++++++++++++++++++++ pkg/csi/evict.go | 2 +- pkg/csi/materializer.go | 58 ++++++++++--- pkg/store/pull.go | 58 +++++++++---- pkg/store/push.go | 10 ++- pkg/store/selector_test.go | 90 +++++++++++++++++++ pkg/store/store.go | 5 ++ 9 files changed, 375 insertions(+), 31 deletions(-) create mode 100644 pkg/csi/candidates_test.go diff --git a/cmd/onggi/csi.go b/cmd/onggi/csi.go index b77268c..974dcaa 100644 --- a/cmd/onggi/csi.go +++ b/cmd/onggi/csi.go @@ -39,7 +39,7 @@ func csiNode(args []string, stderr io.Writer) int { gcGrace := fs.Duration("gc-grace", time.Hour, "grace period of the node cache's GC") rebaseIdle := fs.Duration("rebase-idle", 5*time.Minute, "a pull may turn an artifact no volume used for this long into the new one, writing only what differs (0: never)") maxArtifacts := fs.Int("max-artifacts", 0, "artifacts kept per node; a pull takes an idle one over only when the node holds this many, least recently used first (0: no limit)") - verifyKept := fs.Bool("verify-kept", false, "read back and hash-check every chunk a pull leaves in place when it takes an artifact over") + verifyKept := fs.Bool("verify-kept", true, "read back and hash-check every chunk a pull leaves in place when it takes an artifact over") baseCopy := fs.String("base-copy", "remote", "read an idle artifact that is not taken over where it is: remote (only if the store is not local), always, never") peerListen := fs.String("peer-listen", "", "serve the node cache to other nodes' pulls on this address, e.g. :7070 (#37)") peers := fs.String("peers", "", "comma-separated peer URLs asked for chunks before the store") diff --git a/cmd/onggi/store.go b/cmd/onggi/store.go index 6a61b9f..d327dfe 100644 --- a/cmd/onggi/store.go +++ b/cmd/onggi/store.go @@ -26,6 +26,7 @@ import ( type storeFlags struct { url, durability, min, avg, max, sub, subKinds, packSize, frame *string + frameFlat *bool } func addStoreFlags(fs *flag.FlagSet) storeFlags { @@ -38,6 +39,7 @@ func addStoreFlags(fs *flag.FlagSet) storeFlags { sub: fs.String("sub", "4KiB", "average sub-chunk size for two-level chunking, 0 for one level (only when creating a store)"), subKinds: fs.String("sub-kinds", "atom-prefill,atom-decode,atom-other", "file kinds that get sub-chunks, or all (only when creating a store)"), packSize: fs.String("pack-size", "64MiB", "pack object size (only when creating a store)"), + frameFlat: fs.Bool("frame-flat", false, "with --sub 0, compress the new chunks of a one-level store in frames of --frame too (only when creating a store)"), frame: fs.String("frame", "256KiB", "compress new chunks of sub-chunked files together up to this size, 0 for one by one (only when creating a store)"), } } @@ -101,6 +103,13 @@ func (f storeFlags) open(ctx context.Context) (*store.Store, backend.Refs, error } } } + if c.Sub == nil && *f.frameFlat && *f.frame != "0" { + fr, err := size(*f.frame) + if err != nil { + return nil, nil, err + } + c.Frame, c.FrameFlat = int64(fr), true + } s, err := store.OpenWith(ctx, b, c) return s, b, err } diff --git a/pkg/csi/candidates_test.go b/pkg/csi/candidates_test.go new file mode 100644 index 0000000..6cba439 --- /dev/null +++ b/pkg/csi/candidates_test.go @@ -0,0 +1,172 @@ +package csi + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/ai-computing/onggi/pkg/store" +) + +// An artifact a volume uses is never offered for consumption, but a pull may read it where it is. +func TestLeasedArtifactIsReadNotConsumed(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + m, c := newMaterializer(t, f) + m.RebaseIdle = time.Hour + if _, err := m.Ensure(ctx, f.a); err != nil { + t.Fatal(err) + } + if err := m.Acquire("vol-a", f.a, "/target-a"); err != nil { + t.Fatal(err) + } + ageOwnerDonor(t, m, f.a) + before, err := os.Stat(filepath.Join(m.Dir(f.a), "model.bin")) + if err != nil { + t.Fatal(err) + } + offer, _ := m.offer(f.b) + if offer == nil || len(offer.Bases) != 0 || len(offer.Readable) != 1 || offer.Readable[0].Manifest != f.a { + t.Fatalf("offer %+v: the leased artifact must be readable and not consumable", offer) + } + dir, err := m.Ensure(ctx, f.b) + if err != nil { + t.Fatal(err) + } + sameTree(t, f.srcB, dir) + sameTree(t, f.srcA, m.Dir(f.a)) + last := c.pulls[len(c.pulls)-1] + if last.Base != "" || last.LocalBase != f.a.String() || last.LocalBytes == 0 { + t.Fatalf("the pull did not read the leased artifact where it is: %+v", last) + } + after, err := os.Stat(filepath.Join(m.Dir(f.a), "model.bin")) + if err != nil || !os.SameFile(before, after) { + t.Fatalf("the leased artifact changed: %v", err) + } + if n, err := m.users(f.a); err != nil || n != 1 { + t.Fatalf("lease lost: %d, %v", n, err) + } +} + +// While a pull reads an artifact, another pull cannot take it over and eviction leaves it alone. +func TestReadHoldBlocksClaimAndEviction(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + m, _ := newMaterializer(t, f) + m.RebaseIdle = time.Hour + if _, err := m.Ensure(ctx, f.a); err != nil { + t.Fatal(err) + } + ageOwnerDonor(t, m, f.a) + offer, claimed := m.offer(f.b) + if offer == nil || len(offer.Bases) != 1 || offer.Hold == nil { + t.Fatalf("offer %+v", offer) + } + release, ok := offer.Hold(offer.Readable[0]) + if !ok { + t.Fatal("could not hold a published artifact") + } + if dir, ok := offer.Claim(offer.Bases[0]); ok || dir != "" || len(*claimed) != 0 { + t.Fatal("an artifact that a pull is reading was taken over") + } + if gone, err := m.Evict(time.Now(), 0, nil); err != nil || len(gone) != 0 { + t.Fatalf("an artifact that a pull is reading was evicted: %v, %v", gone, err) + } + release() + ageOwnerDonor(t, m, f.a) + if dir, ok := offer.Claim(offer.Bases[0]); !ok || dir == "" { + t.Fatal("the artifact could not be taken over after the read ended") + } +} + +// With a limit, the cache policy picks the victim: only the least recently used idle artifact is +// offered for consumption, whatever the others would keep in place. All of them may be read. +func TestOnlyTheEvictionVictimIsOffered(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + m, _ := newMaterializer(t, f) + m.RebaseIdle = time.Hour + other := t.TempDir() + if err := os.WriteFile(filepath.Join(other, "model.bin"), random(7, 2<<20), 0o644); err != nil { + t.Fatal(err) + } + o, _, err := f.s.Push(ctx, other, store.PushOptions{}) + if err != nil { + t.Fatal(err) + } + for _, d := range []struct { + name string + age time.Duration + }{{"a", 3 * time.Hour}, {"o", 2 * time.Hour}} { + x := f.a + if d.name == "o" { + x = o + } + if _, err := m.Ensure(ctx, x); err != nil { + t.Fatal(err) + } + when := time.Now().Add(-d.age) + if err := os.Chtimes(m.usedFile(x), when, when); err != nil { + t.Fatal(err) + } + } + offer, _ := m.offer(f.b) + if offer == nil || len(offer.Bases) != 2 || len(offer.Readable) != 2 { + t.Fatalf("without a limit both idle artifacts are offered: %+v", offer) + } + m.MaxArtifacts = 2 + offer, _ = m.offer(f.b) + if offer == nil || len(offer.Bases) != 1 || offer.Bases[0].Manifest != f.a || len(offer.Readable) != 2 { + t.Fatalf("with a limit only the least recently used idle artifact is offered: %+v", offer) + } + // make the other one the least recently used: it becomes the victim, though it shares nothing with B + when := time.Now().Add(-4 * time.Hour) + if err := os.Chtimes(m.usedFile(o), when, when); err != nil { + t.Fatal(err) + } + offer, _ = m.offer(f.b) + if offer == nil || len(offer.Bases) != 1 || offer.Bases[0].Manifest != o { + t.Fatalf("the victim is the least recently used artifact, not the best aligned one: %+v", offer) + } +} + +// Leases are files: a new materializer on the same root (a restarted plugin) still refuses to +// take over an artifact a volume uses. +func TestLeasesSurviveARestart(t *testing.T) { + ctx := context.Background() + f := newFixture(t) + m, _ := newMaterializer(t, f) + if _, err := m.Ensure(ctx, f.a); err != nil { + t.Fatal(err) + } + if err := m.Acquire("vol-a", f.a, "/target-a"); err != nil { + t.Fatal(err) + } + c := &counting{Puller: StorePuller(f.s, f.cache, filepath.Join(m.Root, "tmp"), 4)} + again := &Materializer{Root: m.Root, Pull: c.pull, Logf: t.Logf, RebaseIdle: time.Hour} + if err := again.Init(); err != nil { + t.Fatal(err) + } + ageOwnerDonor(t, again, f.a) + if n, err := again.Users(f.a); err != nil || n != 1 { + t.Fatalf("the restarted materializer sees %d leases, %v", n, err) + } + if offer, _ := again.offer(f.b); offer == nil || len(offer.Bases) != 0 { + t.Fatalf("a leased artifact was offered for consumption after a restart: %+v", offer) + } + dir, err := again.Ensure(ctx, f.b) + if err != nil { + t.Fatal(err) + } + sameTree(t, f.srcB, dir) + sameTree(t, f.srcA, again.Dir(f.a)) + if _, _, ok, err := again.Release("vol-a"); err != nil || !ok { + t.Fatalf("the lease could not be released after the restart: %v", err) + } + ageOwnerDonor(t, again, f.a) + if offer, _ := again.offer(f.b); offer == nil || len(offer.Bases) != 1 { + t.Fatalf("the released artifact is not offered: %+v", offer) + } +} diff --git a/pkg/csi/evict.go b/pkg/csi/evict.go index 006366d..a3ff98e 100644 --- a/pkg/csi/evict.go +++ b/pkg/csi/evict.go @@ -107,7 +107,7 @@ func (m *Materializer) Evict(now time.Time, idle time.Duration, unmount func(dir left := 0 var spare []digest.Digest // unused, but used too recently for idle for _, d := range ds { - if _, busy := m.inflight[d]; busy { + if _, busy := m.inflight[d]; busy || m.reading[d] > 0 { left++ continue } diff --git a/pkg/csi/materializer.go b/pkg/csi/materializer.go index 3bb4ee5..e6c6fc0 100644 --- a/pkg/csi/materializer.go +++ b/pkg/csi/materializer.go @@ -32,6 +32,10 @@ type Offer struct { // VerifyKept and BaseCopy are passed to the pull (store.PullOptions). VerifyKept bool BaseCopy store.BaseCopy + // Readable are artifacts the pull may read where they are, in use or not; Hold keeps one + // from being taken over or evicted while it is read. + Readable []store.Base + Hold func(store.Base) (release func(), ok bool) } // StorePuller pulls from s through the node cache (nil: no cache), assembling cache packs in tmp. @@ -61,6 +65,7 @@ func StorePullerPeers(s *store.Store, openCache func(context.Context) (*store.St if offer != nil { o.Bases, o.Claim = offer.Bases, offer.Claim o.VerifyKept, o.BaseCopy = offer.VerifyKept, offer.BaseCopy + o.ReadBases, o.Hold = offer.Readable, offer.Hold } return s.Pull(ctx, m, dest, o) } @@ -103,6 +108,7 @@ type Materializer struct { mu sync.Mutex inflight map[digest.Digest]*flight + reading map[digest.Digest]int // pulls reading an artifact where it is slots chan struct{} } @@ -386,11 +392,14 @@ func (m *Materializer) offer(d digest.Digest) (*Offer, *[]string) { } m.mu.Lock() ds, _ := m.materialized() - var bases []store.Base + var bases, readable []store.Base inUse, recent := 0, 0 for _, b := range ds { + if b == d { + continue + } + readable = append(readable, store.Base{Manifest: b, Dir: m.Dir(b)}) switch { - case b == d: case m.idle(b, time.Now()): bases = append(bases, store.Base{Manifest: b, Dir: m.Dir(b)}) case m.busy(b): @@ -400,14 +409,14 @@ func (m *Materializer) offer(d digest.Digest) (*Offer, *[]string) { } } m.mu.Unlock() - if len(bases) == 0 { - if len(ds) > 0 { - m.logf("pull of %s: no artifact to take over (%d in use, %d used within %s)", d, inUse, recent, m.RebaseIdle) - } + if len(readable) == 0 { return nil, claimed } - // Most recently used first; with a limit, least recently used first, so that among equally - // good bases the pull consumes the one eviction would remove. + if len(bases) == 0 { + m.logf("pull of %s: no artifact to take over (%d in use, %d used within %s)", d, inUse, recent, m.RebaseIdle) + } + // To consume: most recently used first. With a limit, only the least recently used idle + // artifact, the one eviction would remove: the cache policy picks the victim, not the pull. sort.Slice(bases, func(i, j int) bool { a, b := m.lastUsed(bases[i].Manifest), m.lastUsed(bases[j].Manifest) if m.MaxArtifacts > 0 { @@ -415,9 +424,37 @@ func (m *Materializer) offer(d digest.Digest) (*Offer, *[]string) { } return a.After(b) }) + if m.MaxArtifacts > 0 && len(bases) > 1 { + bases = bases[:1] + } if len(bases) > 3 { bases = bases[:3] } + // To read: most recently used first, in use or not. + sort.Slice(readable, func(i, j int) bool { + return m.lastUsed(readable[i].Manifest).After(m.lastUsed(readable[j].Manifest)) + }) + if len(readable) > 3 { + readable = readable[:3] + } + hold := func(b store.Base) (func(), bool) { + m.mu.Lock() + defer m.mu.Unlock() + if !isDir(m.Dir(b.Manifest)) { + return nil, false + } + if m.reading == nil { + m.reading = map[digest.Digest]int{} + } + m.reading[b.Manifest]++ + return func() { + m.mu.Lock() + if m.reading[b.Manifest]--; m.reading[b.Manifest] <= 0 { + delete(m.reading, b.Manifest) + } + m.mu.Unlock() + }, true + } claim := func(b store.Base) (string, bool) { m.mu.Lock() defer m.mu.Unlock() @@ -456,7 +493,8 @@ func (m *Materializer) offer(d digest.Digest) (*Offer, *[]string) { *claimed = append(*claimed, t) return dst, true } - return &Offer{Bases: bases, Claim: claim, VerifyKept: m.VerifyKept, BaseCopy: m.BaseCopy}, claimed + return &Offer{Bases: bases, Claim: claim, VerifyKept: m.VerifyKept, BaseCopy: m.BaseCopy, + Readable: readable, Hold: hold}, claimed } // idle says whether artifact d may be consumed: materialized, no volume, no pull, unused for @@ -470,7 +508,7 @@ func (m *Materializer) idle(d digest.Digest, now time.Time) bool { // busy says whether a volume or a pull uses artifact d. The caller holds m.mu. func (m *Materializer) busy(d digest.Digest) bool { - if _, ok := m.inflight[d]; ok { + if _, ok := m.inflight[d]; ok || m.reading[d] > 0 { return true } n, err := m.users(d) diff --git a/pkg/store/pull.go b/pkg/store/pull.go index 0b63e0f..44bba01 100644 --- a/pkg/store/pull.go +++ b/pkg/store/pull.go @@ -43,6 +43,11 @@ type PullOptions struct { VerifyKept bool // BaseCopy says when a base that is not taken over is read where it is (#58). BaseCopy BaseCopy + // ReadBases are artifacts on the node the pull may read where they are but never consumes + // (nil: Bases). They may be in use. Hold, if set, is called before one is read and keeps it + // from being taken over or evicted until the returned release is called (ok false: gone). + ReadBases []Base + Hold func(Base) (release func(), ok bool) // BaseRange is the largest single read from a base's files (0: MaxRange). Chunks that follow // one another are read together up to this; 1 reads chunk by chunk. BaseRange int64 @@ -171,33 +176,50 @@ func (s *Store) Pull(ctx context.Context, manifest digest.Digest, dest string, o return st, err } done := make([]bool, len(nodes)) // in place from a base, or written from peers - if len(o.Bases) > 0 && o.Claim != nil { + readBases := o.ReadBases + if readBases == nil { + readBases = o.Bases + } + if len(o.Bases) > 0 && o.Claim != nil || len(readBases) > 0 { minSame := o.MinSame if minSame <= 0 { minSame = DefaultMinSame } - p, ok, most, okMost := s.chooseBase(ctx, m, extentsOf(nodes, len(m.Files), false), o.Bases, minSame) - st.BaseBest = p.same + newExt := extentsOf(nodes, len(m.Files), false) claimed := false - if ok { - if dir, ok := o.Claim(p.base); ok { - claimed = true - st.Base = p.base.Manifest.String() - if st.Reused, err = rebase(p, dir, m, nodes, handles, done); err != nil { - return st, err - } - if o.VerifyKept { - st.Verified, st.Repaired = verifyKept(ctx, nodes, handles, done, o.Workers, o.BaseRange) - st.Reused -= st.Repaired + if len(o.Bases) > 0 && o.Claim != nil { + p, ok, _, _ := s.chooseBase(ctx, m, newExt, o.Bases, minSame) + st.BaseBest = p.same + if ok { + if dir, ok := o.Claim(p.base); ok { + claimed = true + st.Base = p.base.Manifest.String() + if st.Reused, err = rebase(p, dir, m, nodes, handles, done); err != nil { + return st, err + } + if o.VerifyKept { + st.Verified, st.Repaired = verifyKept(ctx, nodes, handles, done, o.Workers, o.BaseRange) + st.Reused -= st.Repaired + } } } } - // too little in place, but the base holds the bytes elsewhere: read them from it (#58) + // nothing taken over, but an artifact on the node holds the bytes: read them from it (#58) copyBase := o.BaseCopy == BaseCopyAlways || o.BaseCopy == BaseCopyRemote && !s.B.Caps().Local - if !claimed && okMost && copyBase { - st.LocalBase = most.base.Manifest.String() - if st.LocalBytes, err = fromBase(ctx, most, nodes, done, handles, o.Workers, o.BaseRange); err != nil { - return st, err + if !claimed && copyBase && len(readBases) > 0 { + if _, _, most, okMost := s.chooseBase(ctx, m, newExt, readBases, minSame); okMost { + release, held := func() {}, true + if o.Hold != nil { + release, held = o.Hold(most.base) + } + if held { + st.LocalBase = most.base.Manifest.String() + st.LocalBytes, err = fromBase(ctx, most, nodes, done, handles, o.Workers, o.BaseRange) + release() + if err != nil { + return st, err + } + } } } } diff --git a/pkg/store/push.go b/pkg/store/push.go index 4620e11..c564a2a 100644 --- a/pkg/store/push.go +++ b/pkg/store/push.go @@ -408,7 +408,7 @@ func (s *Store) chunkFile(ctx context.Context, path string, twoLevel bool, mode var list []ChunkRef var nb int64 var fb *frameBuilder // new chunks of a sub-chunked file go out in compression frames (#52) - if twoLevel && s.Config.Frame > 0 { + if s.Config.Frame > 0 && (twoLevel || s.Config.Sub == nil) { fb = &frameBuilder{limit: int(s.Config.Frame), mode: mode} } for { @@ -430,6 +430,14 @@ func (s *Store) chunkFile(ctx context.Context, path string, twoLevel bool, mode continue } if !twoLevel { + if fb != nil { // a framed one-level store (Create.FrameFlat) + n, err := fb.add(ctx, ch, d, out) + if err != nil { + return nil, 0, err + } + nb += n + continue + } n, err := send(ctx, ch, d, mode, out) if err != nil { return nil, 0, err diff --git a/pkg/store/selector_test.go b/pkg/store/selector_test.go index 44f177c..c51ea5f 100644 --- a/pkg/store/selector_test.go +++ b/pkg/store/selector_test.go @@ -5,6 +5,11 @@ import ( "os" "path/filepath" "testing" + + "github.com/ai-computing/onggi/pkg/backend" + bfs "github.com/ai-computing/onggi/pkg/backend/fs" + "github.com/ai-computing/onggi/pkg/chunker" + "github.com/ai-computing/onggi/pkg/codec" ) // A base damaged after it was pulled is not carried into the new artifact: with VerifyKept the @@ -163,3 +168,88 @@ func TestBaseRangeDoesNotChangeTheResult(t *testing.T) { t.Fatalf("bytes read from the base: chunk by chunk %d, joined %d", local[0], local[1]) } } + +// A base given only to read is never claimed; it is held while it is read and released after. +func TestReadBasesAreHeldAndNeverClaimed(t *testing.T) { + ctx := context.Background() + s := openTwoLevel(t, t.TempDir()) + w := random(51, 8<<20) + v := append([]byte(nil), w...) + copy(v[3<<20:], random(52, 200<<10)) + b, nm, src := rebaseFixture(t, s, map[string][]byte{"decode_B2.rbln": w}, map[string][]byte{"decode_B8.rbln": v}) + held, released := 0, 0 + out := t.TempDir() + st, err := s.Pull(ctx, nm, out, PullOptions{ReadBases: []Base{b}, + Claim: func(Base) (string, bool) { t.Fatal("claimed a base given only to read"); return "", false }, + Hold: func(x Base) (func(), bool) { + if x.Manifest != b.Manifest { + t.Fatalf("held %s", x.Manifest) + } + held++ + return func() { released++ }, true + }}) + if err != nil { + t.Fatal(err) + } + sameTree(t, src, out) + if held != 1 || released != 1 || st.LocalBase != b.Manifest.String() || st.Base != "" || st.LocalBytes < int64(len(v))*8/10 { + t.Fatalf("held %d, released %d, %+v", held, released, st) + } + if _, err := os.Stat(filepath.Join(b.Dir, "decode_B2.rbln")); err != nil { + t.Fatalf("the base was touched: %v", err) + } + + // a base that cannot be held is not read + out = t.TempDir() + st, err = s.Pull(ctx, nm, out, PullOptions{ReadBases: []Base{b}, Hold: func(Base) (func(), bool) { return nil, false }}) + if err != nil { + t.Fatal(err) + } + sameTree(t, src, out) + if st.LocalBase != "" || st.LocalBytes != 0 { + t.Fatalf("read a base it could not hold: %+v", st) + } +} + +// A one-level store with FrameFlat stores its chunks in frames and gives the same files back; +// compressible data takes less room than without frames. +func TestFlatStoreCanFrameItsChunks(t *testing.T) { + ctx := context.Background() + data := make([]byte, 6<<20) + for i := range data { // compressible, but not chunk by chunk identical + data[i] = byte(i>>9) ^ byte(i%7) + } + src := t.TempDir() + write(t, filepath.Join(src, "decode_B2.rbln"), data) + sizes := map[bool]int64{} + for _, framed := range []bool{false, true} { + b, err := bfs.Open(t.TempDir(), backend.DurabilityNone) + if err != nil { + t.Fatal(err) + } + s, err := OpenWith(ctx, b, Create{Chunker: chunker.Params{Min: 1 << 10, Avg: 4 << 10, Max: 16 << 10}, + PackSize: 1 << 20, Frame: 256 << 10, FrameFlat: framed}) + if err != nil { + t.Fatal(err) + } + if (s.Config.Frame > 0) != framed || s.Config.Sub != nil { + t.Fatalf("framed %v: config %+v", framed, s.Config) + } + m, _, err := s.Push(ctx, src, PushOptions{Compress: codec.ModeAuto}) + if err != nil { + t.Fatal(err) + } + out := t.TempDir() + if _, err := s.Pull(ctx, m, out, PullOptions{}); err != nil { + t.Fatal(err) + } + sameTree(t, src, out) + err = b.List(ctx, backend.Pack, func(_ string, n int64) error { sizes[framed] += n; return nil }) + if err != nil { + t.Fatal(err) + } + } + if sizes[true] <= 0 || sizes[true] >= sizes[false] { + t.Fatalf("packs: %d B framed, %d B unframed", sizes[true], sizes[false]) + } +} diff --git a/pkg/store/store.go b/pkg/store/store.go index 124161f..3cc274e 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -52,6 +52,9 @@ type Create struct { SubKinds []layout.Kind // file kinds that get sub-chunks; empty: all PackSize int64 Frame int64 // compression frames for sub-chunked files (#52); used only with Sub + // FrameFlat frames the new chunks of a one-level store too (Sub nil): Frame is then the frame + // size. It exists to compare flat chunking with two levels under the same compression. + FrameFlat bool } // DefaultSubKinds are the kinds whose compile variants rearrange the same bytes (#15). @@ -85,6 +88,8 @@ func OpenWith(ctx context.Context, b backend.Backend, c Create) (*Store, error) cfg = Config{Schema: "onggi.store.v1", Chunker: p, PackSize: packSize, Sub: c.Sub, Created: time.Now().UTC()} if c.Sub != nil { cfg.SubKinds, cfg.Frame = c.SubKinds, c.Frame + } else if c.FrameFlat { + cfg.Frame = c.Frame } data, _ := json.MarshalIndent(cfg, "", " ") if err := b.Put(ctx, configKey, bytes.NewReader(data), int64(len(data))); err != nil { From 57bb2b8999c8cd0522b6ad27da8a30288141fa4b Mon Sep 17 00:00:00 2001 From: gspark-etri Date: Wed, 11 Nov 2026 17:14:54 +0900 Subject: [PATCH 06/16] casync engine: desync chunk store with state-aware in-place reuse pkg/casync materializes artifacts from a casync-format (desync) chunk store and implements csi.Puller. Keep pairs files by path, finds the chunks the base holds at the same offset from the two indexes alone, and writes only the others from the store; it reads kept chunks back only when the base is not attested unchanged (inode, size, mtime recorded when this engine wrote it). Copy seeds desync's assembler with a held base; Reconstruct assembles from the store. Written files are synced unless --no-sync. onggi casync publish|pull|verify|attest. go.mod is updated on the build host (go get desync v1.1.5). Co-Authored-By: Claude Opus 5.5 --- cmd/onggi/casync.go | 139 +++++++++ cmd/onggi/main.go | 3 + pkg/casync/casync.go | 636 ++++++++++++++++++++++++++++++++++++++ pkg/casync/casync_test.go | 154 +++++++++ 4 files changed, 932 insertions(+) create mode 100644 cmd/onggi/casync.go create mode 100644 pkg/casync/casync.go create mode 100644 pkg/casync/casync_test.go diff --git a/cmd/onggi/casync.go b/cmd/onggi/casync.go new file mode 100644 index 0000000..9faa280 --- /dev/null +++ b/cmd/onggi/casync.go @@ -0,0 +1,139 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "os" + "time" + + "github.com/ai-computing/onggi/pkg/casync" + "github.com/ai-computing/onggi/pkg/csi" + "github.com/ai-computing/onggi/pkg/digest" + "github.com/ai-computing/onggi/pkg/store" +) + +const casyncUsage = `onggi casync — artifacts in casync format (desync chunk store), with state-aware in-place reuse + + onggi casync publish --store S --index I chunk dir into S; print the manifest digest + onggi casync pull --store S --index I [--attest A] + [--base M --base-dir D] take base M over (Keep): write only differing chunks; kept chunks are + read back unless A attests the base unchanged since it was written + [--copy M --copy-dir D] read base M where it is (Copy); it stays intact + [--verify-kept] [--no-sync] [--workers N] [--log] + onggi casync verify --store S --index I hash-check every chunk of dir + onggi casync attest --store S --index I --attest A verify dir, then record it as unchanged +` + +func casyncCmd(args []string, stdout, stderr io.Writer) int { + if len(args) < 1 { + fmt.Fprint(stderr, casyncUsage) + return 2 + } + fs := flag.NewFlagSet("casync "+args[0], flag.ContinueOnError) + fs.SetOutput(stderr) + storeDir := fs.String("store", "", "desync local chunk store") + indexDir := fs.String("index", "", "directory of artifact indexes") + attestDir := fs.String("attest", "", "directory of attestations (none: kept chunks are always read back)") + workers := fs.Int("workers", 12, "parallel chunk reads and writes") + base := fs.String("base", "", "Keep: manifest of the artifact to take over") + baseDir := fs.String("base-dir", "", "Keep: directory holding the base's files") + cp := fs.String("copy", "", "Copy: manifest of an artifact to read where it is") + copyDir := fs.String("copy-dir", "", "Copy: directory holding that artifact's files") + verifyKept := fs.Bool("verify-kept", false, "read back kept chunks even if the base is attested unchanged") + noSync := fs.Bool("no-sync", false, "do not sync written files (desync's default behaviour)") + logPhases := fs.Bool("log", false, "log phase durations to stderr") + if err := fs.Parse(args[1:]); err != nil || *storeDir == "" || *indexDir == "" { + fmt.Fprint(stderr, casyncUsage) + return 2 + } + e, err := casync.New(*storeDir, *indexDir, *attestDir, *workers) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + e.Durable = !*noSync + e.VerifyKept = *verifyKept + if *logPhases { + l := log.New(stderr, "", log.Lmicroseconds) + e.Logf = l.Printf + } + ctx := context.Background() + switch args[0] { + case "publish": + if fs.NArg() != 1 { + fmt.Fprint(stderr, casyncUsage) + return 2 + } + start := time.Now() + d, err := e.Publish(ctx, fs.Arg(0)) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + fmt.Fprintf(stdout, "%s\t%.2fs\n", d, time.Since(start).Seconds()) + return 0 + case "pull", "verify", "attest": + if fs.NArg() != 2 { + fmt.Fprint(stderr, casyncUsage) + return 2 + } + m, err := digest.Parse(fs.Arg(0)) + if err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + dest := fs.Arg(1) + if args[0] != "pull" { + if err := e.Verify(ctx, m, dest); err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + if args[0] == "attest" { + if err := e.Attest(m, dest); err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + } + return 0 + } + if err := os.MkdirAll(dest, 0o755); err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + var offer *csi.Offer + if *base != "" { + b, err := digest.Parse(*base) + if err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + offer = &csi.Offer{Bases: []store.Base{{Manifest: b, Dir: *baseDir}}, + Claim: func(sb store.Base) (string, bool) { return sb.Dir, true }} + } + if *cp != "" { + c, err := digest.Parse(*cp) + if err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + if offer == nil { + offer = &csi.Offer{} + } + offer.Readable, offer.BaseCopy = []store.Base{{Manifest: c, Dir: *copyDir}}, store.BaseCopyAlways + } + st, err := e.Pull(ctx, m, dest, offer) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + js, _ := json.Marshal(st) + fmt.Fprintln(stdout, string(js)) + return 0 + } + fmt.Fprint(stderr, casyncUsage) + return 2 +} diff --git a/cmd/onggi/main.go b/cmd/onggi/main.go index ae9f0bc..e63bf1e 100644 --- a/cmd/onggi/main.go +++ b/cmd/onggi/main.go @@ -34,6 +34,7 @@ usage: evict from a node cache what recently used artifacts do not need onggi dedup-report [flags] ... measure how much the files would shrink in onggi + onggi casync publish|pull|verify|attest ... casync-format artifacts with state-aware in-place reuse onggi csi --store S [flags] CSI node plugin: serve artifacts as read-only volumes (Linux) onggi peer serve --cache DIR [--listen :7070] serve a node cache to other nodes' pulls (--peers) onggi version @@ -61,6 +62,8 @@ func main() { os.Exit(inspect(os.Args[2:], os.Stdout, os.Stderr)) case "diff": os.Exit(diff(os.Args[2:], os.Stdout, os.Stderr)) + case "casync": + os.Exit(casyncCmd(os.Args[2:], os.Stdout, os.Stderr)) case "csi": os.Exit(csiNode(os.Args[2:], os.Stderr)) case "rm-ref": diff --git a/pkg/casync/casync.go b/pkg/casync/casync.go new file mode 100644 index 0000000..504c51e --- /dev/null +++ b/pkg/casync/casync.go @@ -0,0 +1,636 @@ +// Package casync materializes artifacts from a casync-format chunk store (desync), and adds what a +// stateless tool cannot do: it knows which sibling on the node is idle, aligned and unchanged since +// it was written, and turns that sibling into the requested artifact by writing only the chunks that +// differ, without reading the ones it keeps. +// +// Layout under Engine.IndexDir: +// +// /artifact.json files of the artifact: path, size, mode, index file +// /.caibx casync index of file i +// +// The manifest digest is the SHA-256 of artifact.json. Chunks live in a desync local store +// (one zstd-compressed file per chunk, named by its SHA512/256). Engine.AttestDir records, for every +// file this engine wrote, the inode, size and modification time it had when the write completed; +// a later Keep trusts the chunks it leaves in place only if the file still has them (state-aware +// in-place), and otherwise reads them back and checks each digest. +package casync + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/folbricht/desync" + + "github.com/ai-computing/onggi/pkg/csi" + "github.com/ai-computing/onggi/pkg/digest" + "github.com/ai-computing/onggi/pkg/store" +) + +// Chunk sizes of desync's defaults (16/64/256 KiB). +const ( + MinChunk = 16 << 10 + AvgChunk = 64 << 10 + MaxChunk = 256 << 10 +) + +// Engine reads and writes artifacts in casync format. +type Engine struct { + Store desync.LocalStore + IndexDir string + AttestDir string // "" : never trust kept chunks, always read them back + Workers int + MinSame float64 // share of bytes at the same offset needed to take a base (0: store.DefaultMinSame) + // VerifyKept forces reading back kept chunks even when the base is attested unchanged. + VerifyKept bool + // Durable syncs every written file before the pull returns (default true via New). + Durable bool + // Logf, if set, receives one line per phase with its duration. + Logf func(format string, args ...any) +} + +// New opens the chunk store at storeDir and the index and attestation directories. +func New(storeDir, indexDir, attestDir string, workers int) (*Engine, error) { + for _, d := range []string{storeDir, indexDir} { + if err := os.MkdirAll(d, 0o755); err != nil { + return nil, err + } + } + if attestDir != "" { + if err := os.MkdirAll(attestDir, 0o755); err != nil { + return nil, err + } + } + s, err := desync.NewLocalStore(storeDir, desync.StoreOptions{N: max(workers, 1)}) + if err != nil { + return nil, err + } + return &Engine{Store: s, IndexDir: indexDir, AttestDir: attestDir, Workers: max(workers, 1), Durable: true}, nil +} + +// FileEntry is one file of an artifact. +type FileEntry struct { + Path string `json:"path"` + Size int64 `json:"size"` + Mode uint32 `json:"mode"` + Index string `json:"index"` +} + +// Artifact is the content of artifact.json. +type Artifact struct { + Files []FileEntry `json:"files"` +} + +func (e *Engine) logf(format string, args ...any) { + if e.Logf != nil { + e.Logf(format, args...) + } +} + +// Publish chunks every regular file under dir into the store and writes the artifact's indexes. +// It returns the manifest digest. +func (e *Engine) Publish(ctx context.Context, dir string) (digest.Digest, error) { + var paths []string + err := filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.Type().IsRegular() { + rel, _ := filepath.Rel(dir, p) + paths = append(paths, filepath.ToSlash(rel)) + } + return nil + }) + if err != nil { + return digest.Digest{}, err + } + sort.Strings(paths) + tmp, err := os.MkdirTemp(e.IndexDir, ".publish-") + if err != nil { + return digest.Digest{}, err + } + defer os.RemoveAll(tmp) + var a Artifact + for i, rel := range paths { + src := filepath.Join(dir, filepath.FromSlash(rel)) + fi, err := os.Stat(src) + if err != nil { + return digest.Digest{}, err + } + idx, _, err := desync.IndexFromFile(ctx, src, e.Workers, MinChunk, AvgChunk, MaxChunk, desync.NullProgressBar{}) + if err != nil { + return digest.Digest{}, fmt.Errorf("index %s: %w", rel, err) + } + if err := desync.ChopFile(ctx, src, idx.Chunks, e.Store, e.Workers, desync.NullProgressBar{}); err != nil { + return digest.Digest{}, fmt.Errorf("chop %s: %w", rel, err) + } + name := fmt.Sprintf("%d.caibx", i) + f, err := os.Create(filepath.Join(tmp, name)) + if err != nil { + return digest.Digest{}, err + } + if _, err := idx.WriteTo(f); err != nil { + f.Close() + return digest.Digest{}, err + } + if err := f.Close(); err != nil { + return digest.Digest{}, err + } + a.Files = append(a.Files, FileEntry{Path: rel, Size: fi.Size(), Mode: uint32(fi.Mode().Perm()), Index: name}) + } + data, _ := json.Marshal(a) + sum := sha256.Sum256(data) + d, err := digest.Parse("sha256:" + hex.EncodeToString(sum[:])) + if err != nil { + return digest.Digest{}, err + } + if err := os.WriteFile(filepath.Join(tmp, "artifact.json"), data, 0o644); err != nil { + return digest.Digest{}, err + } + final := filepath.Join(e.IndexDir, d.Hex()) + if _, err := os.Stat(final); err == nil { + return d, nil // published before: same content + } + if err := os.Rename(tmp, final); err != nil { + return digest.Digest{}, err + } + return d, nil +} + +// Load returns the artifact's file list and the casync index of each file. +func (e *Engine) Load(d digest.Digest) (*Artifact, []desync.Index, error) { + dir := filepath.Join(e.IndexDir, d.Hex()) + data, err := os.ReadFile(filepath.Join(dir, "artifact.json")) + if err != nil { + return nil, nil, err + } + sum := sha256.Sum256(data) + if hex.EncodeToString(sum[:]) != d.Hex() { + return nil, nil, fmt.Errorf("artifact %s: artifact.json does not match its digest", d) + } + var a Artifact + if err := json.Unmarshal(data, &a); err != nil { + return nil, nil, err + } + idxs := make([]desync.Index, len(a.Files)) + for i, f := range a.Files { + r, err := os.Open(filepath.Join(dir, f.Index)) + if err != nil { + return nil, nil, err + } + idxs[i], err = desync.IndexFromReader(r) + r.Close() + if err != nil { + return nil, nil, fmt.Errorf("%s: %w", f.Path, err) + } + } + return &a, idxs, nil +} + +// sameOffset marks the target chunks that the base file holds at the same offset. Both chunk +// lists are sorted by offset, so one pass suffices; no file is read. +func sameOffset(base, target []desync.IndexChunk) ([]bool, int64) { + kept := make([]bool, len(target)) + var same int64 + j := 0 + for i, c := range target { + for j < len(base) && base[j].Start < c.Start { + j++ + } + if j < len(base) && base[j].Start == c.Start && base[j].ID == c.ID && base[j].Size == c.Size { + kept[i] = true + same += int64(c.Size) + } + } + return kept, same +} + +// keepPlan pairs the target's files with a base's files by path. +type keepPlan struct { + base store.Base + files *Artifact + idxs []desync.Index + pair []int // base file per target file, -1: none + kept [][]bool // per target file, chunks the paired base file holds at the same offset + same int64 +} + +func (e *Engine) plan(target *Artifact, tidx []desync.Index, b store.Base) (keepPlan, error) { + ba, bidx, err := e.Load(b.Manifest) + if err != nil { + return keepPlan{}, err + } + byPath := map[string]int{} + for j, f := range ba.Files { + byPath[f.Path] = j + } + p := keepPlan{base: b, files: ba, idxs: bidx, pair: make([]int, len(target.Files)), kept: make([][]bool, len(target.Files))} + for i, f := range target.Files { + j, ok := byPath[f.Path] + if !ok { + p.pair[i] = -1 + continue + } + p.pair[i] = j + var s int64 + p.kept[i], s = sameOffset(bidx[j].Chunks, tidx[i].Chunks) + p.same += s + } + return p, nil +} + +// attestation is what a file looked like when this engine finished writing it. +type attestation struct { + Ino uint64 `json:"ino"` + Size int64 `json:"size"` + MtimeNs int64 `json:"mtime_ns"` +} + +func stat(p string) (attestation, error) { + fi, err := os.Stat(p) + if err != nil { + return attestation{}, err + } + a := attestation{Size: fi.Size(), MtimeNs: fi.ModTime().UnixNano()} + if st, ok := fi.Sys().(*syscall.Stat_t); ok { + a.Ino = uint64(st.Ino) + } + return a, nil +} + +func (e *Engine) attestFile(d digest.Digest) string { + return filepath.Join(e.AttestDir, d.Hex()+".json") +} + +func (e *Engine) recordAttestation(d digest.Digest, dir string, a *Artifact) error { + if e.AttestDir == "" { + return nil + } + rec := map[string]attestation{} + for _, f := range a.Files { + s, err := stat(filepath.Join(dir, filepath.FromSlash(f.Path))) + if err != nil { + return err + } + rec[f.Path] = s + } + data, _ := json.Marshal(rec) + tmp := e.attestFile(d) + ".tmp" + if err := os.WriteFile(tmp, data, 0o644); err != nil { + return err + } + return os.Rename(tmp, e.attestFile(d)) +} + +// Attest records the files of manifest d under dir as unchanged; call it only after Verify. +func (e *Engine) Attest(d digest.Digest, dir string) error { + a, _, err := e.Load(d) + if err != nil { + return err + } + return e.recordAttestation(d, dir, a) +} + +// unchanged reports whether the base file at p is the one this engine wrote for manifest d. +func (e *Engine) unchanged(d digest.Digest, path, p string) bool { + if e.AttestDir == "" { + return false + } + data, err := os.ReadFile(e.attestFile(d)) + if err != nil { + return false + } + var rec map[string]attestation + if json.Unmarshal(data, &rec) != nil { + return false + } + want, ok := rec[path] + if !ok { + return false + } + got, err := stat(p) + return err == nil && got == want +} + +// Pull writes artifact m into the empty directory dest. With an offer it takes an idle aligned +// base over (Keep), or reads a base that stays intact (Copy); otherwise it assembles from the +// store (Reconstruct). It satisfies csi.Puller. +func (e *Engine) Pull(ctx context.Context, m digest.Digest, dest string, offer *csi.Offer) (store.PullStats, error) { + start := time.Now() + a, idxs, err := e.Load(m) + if err != nil { + return store.PullStats{}, err + } + st := store.PullStats{Files: len(a.Files)} + for _, f := range a.Files { + st.Bytes += f.Size + } + finals := make([]string, len(a.Files)) + for i, f := range a.Files { + p := filepath.Join(dest, filepath.FromSlash(f.Path)) + if !filepath.IsLocal(filepath.FromSlash(f.Path)) { + return st, fmt.Errorf("file path %q leaves the artifact", f.Path) + } + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + return st, err + } + finals[i] = p + } + done := make([]bool, len(a.Files)) + minSame := e.MinSame + if minSame <= 0 { + minSame = store.DefaultMinSame + } + + // Keep: the idle base that holds the most bytes at the same offsets + if offer != nil && offer.Claim != nil && len(offer.Bases) > 0 { + var best keepPlan + for _, b := range offer.Bases { + p, err := e.plan(a, idxs, b) + if err == nil && p.same > best.same { + best = p + } + } + st.BaseBest = best.same + if best.same > 0 && float64(best.same) >= minSame*float64(st.Bytes) { + if dir, ok := offer.Claim(best.base); ok { + st.Base = best.base.Manifest.String() + if err := e.keep(ctx, a, idxs, best, dir, finals, done, offer.VerifyKept || e.VerifyKept, &st); err != nil { + return st, err + } + } + } + } + // Copy: a base that stays where it is seeds the files Keep did not produce + var release func() + var seedBase *keepPlan + if offer != nil && len(offer.Readable) > 0 && offer.BaseCopy != store.BaseCopyNever { + var most keepPlan + for _, b := range offer.Readable { + if p, err := e.plan(a, idxs, b); err == nil && p.same > most.same { + most = p + } + } + if most.same > 0 { + ok := true + release = func() {} + if offer.Hold != nil { + release, ok = offer.Hold(most.base) + } + if ok { + seedBase = &most + st.LocalBase = most.base.Manifest.String() + } + } + } + if release != nil { + defer release() + } + // whatever is left: assemble from the store, seeded by the held base if there is one + for i := range a.Files { + if done[i] { + continue + } + partial := finals[i] + ".casync-partial" + os.Remove(partial) + var seeds []desync.Seed + if seedBase != nil && seedBase.pair[i] >= 0 { + j := seedBase.pair[i] + dir, _ := baseDir(offer, seedBase.base) + src := filepath.Join(dir, filepath.FromSlash(seedBase.files.Files[j].Path)) + s, err := desync.NewIndexSeed(partial, src, seedBase.idxs[j]) + if err == nil { + seeds = append(seeds, s) + st.LocalBytes += countKept(seedBase.kept[i], idxs[i].Chunks) + } + } + t0 := time.Now() + if _, err := desync.AssembleFile(ctx, partial, idxs[i], e.Store, seeds, desync.AssembleOptions{N: e.Workers, InvalidSeedAction: desync.InvalidSeedActionSkip}); err != nil { + os.Remove(partial) + return st, fmt.Errorf("%s: %w", a.Files[i].Path, err) + } + e.logf("assemble %s (%d seeds): %s", a.Files[i].Path, len(seeds), time.Since(t0).Round(time.Millisecond)) + if err := e.finish(partial, finals[i], a.Files[i]); err != nil { + return st, err + } + done[i] = true + } + if err := e.recordAttestation(m, dest, a); err != nil { + return st, err + } + st.Duration = time.Since(start) + return st, nil +} + +// baseDir is where a readable base's files are: the materializer keeps them under its root, and +// the offer's Base.Dir says where. +func baseDir(_ *csi.Offer, b store.Base) (string, bool) { return b.Dir, b.Dir != "" } + +func countKept(kept []bool, chunks []desync.IndexChunk) int64 { + var n int64 + for i, k := range kept { + if k { + n += int64(chunks[i].Size) + } + } + return n +} + +// keep turns the claimed base's files into the target's: each paired file is moved into place, +// resized, and only its differing chunks are written from the store. Kept chunks are read back and +// hash-checked unless the file is attested unchanged since this engine wrote it. +func (e *Engine) keep(ctx context.Context, a *Artifact, idxs []desync.Index, p keepPlan, dir string, finals []string, done []bool, verify bool, st *store.PullStats) error { + for i, j := range p.pair { + if j < 0 { + continue + } + src := filepath.Join(dir, filepath.FromSlash(p.files.Files[j].Path)) + trusted := !verify && e.unchanged(p.base.Manifest, p.files.Files[j].Path, src) + partial := finals[i] + ".casync-partial" + if err := os.Rename(src, partial); err != nil { + continue // left for the store + } + f, err := os.OpenFile(partial, os.O_RDWR, 0) + if err != nil { + return err + } + if err := f.Chmod(0o644); err != nil { + f.Close() + return err + } + if err := f.Truncate(a.Files[i].Size); err != nil { + f.Close() + return err + } + kept := append([]bool(nil), p.kept[i]...) + t0 := time.Now() + if !trusted { + ok, bad := e.verifyKept(ctx, f, idxs[i].Chunks, kept) + st.Verified += ok + st.Repaired += bad + } + t1 := time.Now() + written, err := e.writeChunks(ctx, f, idxs[i].Chunks, kept) + if err != nil { + f.Close() + return err + } + t2 := time.Now() + if e.Durable { + if err := f.Sync(); err != nil { + f.Close() + return err + } + } + if err := f.Close(); err != nil { + return err + } + st.Reused += a.Files[i].Size - written + st.ReadBytes += written + e.logf("keep %s: trusted=%v verify %s, write %d B %s, sync %s", a.Files[i].Path, trusted, + t1.Sub(t0).Round(time.Millisecond), written, t2.Sub(t1).Round(time.Millisecond), time.Since(t2).Round(time.Millisecond)) + if err := e.finish(partial, finals[i], a.Files[i]); err != nil { + return err + } + done[i] = true + } + return nil +} + +// verifyKept reads every kept chunk and checks its digest; a chunk that does not match is no +// longer kept, so the write step fetches it. It returns the bytes that matched and that did not. +func (e *Engine) verifyKept(ctx context.Context, f *os.File, chunks []desync.IndexChunk, kept []bool) (ok, bad int64) { + var okN, badN atomic.Int64 + work := make(chan int) + var wg sync.WaitGroup + for w := 0; w < e.Workers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + var buf []byte + for i := range work { + c := chunks[i] + if cap(buf) < int(c.Size) { + buf = make([]byte, c.Size) + } + b := buf[:c.Size] + if _, err := f.ReadAt(b, int64(c.Start)); err != nil || desync.ChunkID(desync.Digest.Sum(b)) != c.ID { + kept[i] = false + badN.Add(int64(c.Size)) + continue + } + okN.Add(int64(c.Size)) + } + }() + } + for i := range chunks { + if kept[i] { + select { + case work <- i: + case <-ctx.Done(): + } + } + } + close(work) + wg.Wait() + return okN.Load(), badN.Load() +} + +// writeChunks writes every chunk that is not kept, from the store (each chunk's digest is checked +// when it is read from the store). It returns the bytes written. +func (e *Engine) writeChunks(ctx context.Context, f *os.File, chunks []desync.IndexChunk, kept []bool) (int64, error) { + var written atomic.Int64 + var firstErr error + var once sync.Once + work := make(chan int) + var wg sync.WaitGroup + for w := 0; w < e.Workers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := range work { + c := chunks[i] + ch, err := e.Store.GetChunk(c.ID) + if err == nil { + var b []byte + if b, err = ch.Data(); err == nil { + if uint64(len(b)) != c.Size { + err = fmt.Errorf("chunk %s: %d bytes, index says %d", c.ID, len(b), c.Size) + } else if _, err = f.WriteAt(b, int64(c.Start)); err == nil { + written.Add(int64(len(b))) + } + } + } + if err != nil { + once.Do(func() { firstErr = err }) + } + } + }() + } + for i := range chunks { + if !kept[i] { + select { + case work <- i: + case <-ctx.Done(): + once.Do(func() { firstErr = ctx.Err() }) + } + } + } + close(work) + wg.Wait() + return written.Load(), firstErr +} + +// finish syncs a partial file (when the writer did not), sets its mode, and renames it into place. +func (e *Engine) finish(partial, final string, f FileEntry) error { + if e.Durable { + h, err := os.OpenFile(partial, os.O_RDWR, 0) + if err != nil { + return err + } + if err := h.Sync(); err != nil { + h.Close() + return err + } + h.Close() + } + if err := os.Chmod(partial, os.FileMode(f.Mode)|0o400); err != nil { + return err + } + return os.Rename(partial, final) +} + +// Verify hash-checks every chunk of the artifact's files under dir against its indexes. +func (e *Engine) Verify(ctx context.Context, m digest.Digest, dir string) error { + a, idxs, err := e.Load(m) + if err != nil { + return err + } + for i, f := range a.Files { + h, err := os.Open(filepath.Join(dir, filepath.FromSlash(f.Path))) + if err != nil { + return err + } + kept := make([]bool, len(idxs[i].Chunks)) + for k := range kept { + kept[k] = true + } + _, bad := e.verifyKept(ctx, h, idxs[i].Chunks, kept) + fi, _ := h.Stat() + h.Close() + if bad > 0 || fi.Size() != f.Size { + return fmt.Errorf("%s: %d bytes do not match the index", f.Path, bad) + } + } + return nil +} + diff --git a/pkg/casync/casync_test.go b/pkg/casync/casync_test.go new file mode 100644 index 0000000..b7856bc --- /dev/null +++ b/pkg/casync/casync_test.go @@ -0,0 +1,154 @@ +package casync + +import ( + "bytes" + "context" + "math/rand" + "os" + "path/filepath" + "testing" + "time" + + "github.com/ai-computing/onggi/pkg/csi" + "github.com/ai-computing/onggi/pkg/store" +) + +// variants returns a base and an aligned sibling (a middle region rewritten in place) and a shifted +// sibling (a header inserted at the front, so every later byte moves). +func variants() (base, aligned, shifted []byte) { + r := rand.New(rand.NewSource(7)) + base = make([]byte, 6<<20) + r.Read(base) + aligned = append([]byte(nil), base...) + r.Read(aligned[3<<20 : 3<<20+300<<10]) + shifted = append(append(make([]byte, 0, len(base)+4096), bytes.Repeat([]byte{9}, 4096)...), base...) + return +} + +func publish(t *testing.T, e *Engine, root, name string, data []byte) (dir string) { + t.Helper() + dir = filepath.Join(root, "src-"+name) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "model.bin"), data, 0o644); err != nil { + t.Fatal(err) + } + return dir +} + +func setup(t *testing.T) (*Engine, string) { + t.Helper() + root := t.TempDir() + e, err := New(filepath.Join(root, "store"), filepath.Join(root, "index"), filepath.Join(root, "attest"), 4) + if err != nil { + t.Fatal(err) + } + return e, root +} + +func readFile(t *testing.T, p string) []byte { + t.Helper() + b, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + return b +} + +func TestPaths(t *testing.T) { + ctx := context.Background() + e, root := setup(t) + base, aligned, shifted := variants() + mA, err := e.Publish(ctx, publish(t, e, root, "a", base)) + if err != nil { + t.Fatal(err) + } + mB, _ := e.Publish(ctx, publish(t, e, root, "b", aligned)) + mC, _ := e.Publish(ctx, publish(t, e, root, "c", shifted)) + + // Reconstruct: no offer + dA := filepath.Join(root, "art", "A") + os.MkdirAll(dA, 0o755) + st, err := e.Pull(ctx, mA, dA, nil) + if err != nil || !bytes.Equal(readFile(t, filepath.Join(dA, "model.bin")), base) || st.Base != "" { + t.Fatalf("reconstruct: %v %+v", err, st) + } + + // Keep, trusted: A is attested unchanged, so kept chunks are not read back + claimed := false + offer := &csi.Offer{Bases: []store.Base{{Manifest: mA, Dir: dA}}, + Claim: func(b store.Base) (string, bool) { claimed = true; return b.Dir, true }} + dB := filepath.Join(root, "art", "B") + os.MkdirAll(dB, 0o755) + st, err = e.Pull(ctx, mB, dB, offer) + if err != nil || !claimed || st.Base != mA.String() || st.Verified != 0 || st.Reused < int64(len(aligned))*8/10 { + t.Fatalf("keep trusted: %v claimed=%v %+v", err, claimed, st) + } + if !bytes.Equal(readFile(t, filepath.Join(dB, "model.bin")), aligned) { + t.Fatal("keep trusted: output differs") + } + if _, err := os.Stat(filepath.Join(dA, "model.bin")); !os.IsNotExist(err) { + t.Fatal("keep must consume the base file") + } + + // Keep, tampered: B is changed after it was written; the attestation no longer matches, so kept + // chunks are read back and the corrupted one is fetched again + p := filepath.Join(dB, "model.bin") + os.Chmod(p, 0o644) + f, _ := os.OpenFile(p, os.O_RDWR, 0) + f.WriteAt([]byte("corrupt"), 100) + f.Close() + future := time.Now().Add(time.Second) + os.Chtimes(p, future, future) + dA2 := filepath.Join(root, "art", "A2") + os.MkdirAll(dA2, 0o755) + st, err = e.Pull(ctx, mA, dA2, &csi.Offer{Bases: []store.Base{{Manifest: mB, Dir: dB}}, + Claim: func(b store.Base) (string, bool) { return b.Dir, true }}) + if err != nil || st.Verified == 0 || st.Repaired == 0 { + t.Fatalf("keep tampered: %v %+v", err, st) + } + if !bytes.Equal(readFile(t, filepath.Join(dA2, "model.bin")), base) { + t.Fatal("keep tampered: output differs") + } + + // a shifted sibling keeps nothing at the same offset: no claim, assembled from the store + claimed = false + dC := filepath.Join(root, "art", "C") + os.MkdirAll(dC, 0o755) + st, err = e.Pull(ctx, mC, dC, &csi.Offer{Bases: []store.Base{{Manifest: mA, Dir: dA2}}, + Claim: func(b store.Base) (string, bool) { claimed = true; return b.Dir, true }}) + if err != nil || claimed || st.BaseBest != 0 || !bytes.Equal(readFile(t, filepath.Join(dC, "model.bin")), shifted) { + t.Fatalf("shifted: %v claimed=%v %+v", err, claimed, st) + } + + // Copy: A2 stays intact and seeds B + held := false + dB2 := filepath.Join(root, "art", "B2") + os.MkdirAll(dB2, 0o755) + st, err = e.Pull(ctx, mB, dB2, &csi.Offer{Readable: []store.Base{{Manifest: mA, Dir: dA2}}, BaseCopy: store.BaseCopyAlways, + Hold: func(store.Base) (func(), bool) { held = true; return func() {}, true }}) + if err != nil || !held || st.LocalBase != mA.String() || !bytes.Equal(readFile(t, filepath.Join(dB2, "model.bin")), aligned) { + t.Fatalf("copy: %v held=%v %+v", err, held, st) + } + if !bytes.Equal(readFile(t, filepath.Join(dA2, "model.bin")), base) { + t.Fatal("copy must leave the base intact") + } + if err := e.Verify(ctx, mB, dB2); err != nil { + t.Fatal(err) + } +} + +func TestSameOffset(t *testing.T) { + e, root := setup(t) + base, aligned, _ := variants() + ctx := context.Background() + mA, _ := e.Publish(ctx, publish(t, e, root, "a", base)) + mB, _ := e.Publish(ctx, publish(t, e, root, "b", aligned)) + _, ia, _ := e.Load(mA) + _, ib, _ := e.Load(mB) + _, same := sameOffset(ia[0].Chunks, ib[0].Chunks) + if same < int64(len(aligned))*8/10 || same == int64(len(aligned)) { + t.Fatalf("same=%d of %d", same, len(aligned)) + } +} From b329e9fe63a7c7992d03341d8cfdbfa03dfd58b3 Mon Sep 17 00:00:00 2001 From: gspark-etri Date: Wed, 11 Nov 2026 17:16:17 +0900 Subject: [PATCH 07/16] casync: manifest covers index content; add desync dependency The manifest digest hashed only paths, sizes and modes, so two variants of equal size shared one manifest (caught by the new tests). Each file entry now carries the SHA-256 of its index, checked on load. go.mod/go.sum from go get github.com/folbricht/desync@v1.1.5. Co-Authored-By: Claude Opus 5.5 --- go.mod | 91 ++++++++++++---- go.sum | 249 ++++++++++++++++++++++++++++++++++--------- pkg/casync/casync.go | 25 +++-- 3 files changed, 288 insertions(+), 77 deletions(-) diff --git a/go.mod b/go.mod index 4fe7267..c55c69e 100644 --- a/go.mod +++ b/go.mod @@ -1,32 +1,87 @@ module github.com/ai-computing/onggi -go 1.24 +go 1.25.0 require ( github.com/container-storage-interface/spec v1.12.0 - github.com/klauspost/compress v1.18.0 - github.com/minio/minio-go/v7 v7.0.95 - golang.org/x/sys v0.33.0 - google.golang.org/grpc v1.75.1 + github.com/folbricht/desync v1.1.5 + github.com/klauspost/compress v1.19.2 + github.com/minio/minio-go/v7 v7.3.0 + golang.org/x/sys v0.47.0 + google.golang.org/grpc v1.83.2 ) require ( - github.com/davecgh/go-spew v1.1.1 // indirect + cel.dev/expr v0.25.2 // indirect + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.18.2 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.5.3 // indirect + cloud.google.com/go/monitoring v1.24.3 // indirect + cloud.google.com/go/storage v1.60.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect + github.com/boljen/go-bitmap v0.0.0-20151001105940-23cd2fb0ce7d // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect + github.com/dchest/siphash v1.2.3 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/go-ini/ini v1.67.0 // indirect - github.com/goccy/go-json v0.10.5 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/folbricht/tempfile v0.0.1 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/s2a-go v0.1.9 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/klauspost/cpuid/v2 v2.2.11 // indirect - github.com/minio/crc64nvme v1.0.2 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect + github.com/googleapis/gax-go/v2 v2.17.0 // indirect + github.com/hanwen/go-fuse/v2 v2.11.0 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/kr/fs v0.1.0 // indirect + github.com/mattn/go-runewidth v0.0.23 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect github.com/minio/md5-simd v1.1.2 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect github.com/philhofer/fwd v1.2.0 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pkg/sftp v1.13.5 // indirect + github.com/pkg/xattr v0.4.9 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/rs/xid v1.6.0 // indirect - github.com/tinylib/msgp v1.3.0 // indirect - golang.org/x/crypto v0.39.0 // indirect - golang.org/x/net v0.41.0 // indirect - golang.org/x/text v0.26.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect - google.golang.org/protobuf v1.36.6 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + github.com/sirupsen/logrus v1.9.4 // indirect + github.com/spiffe/go-spiffe/v2 v2.7.0 // indirect + github.com/tinylib/msgp v1.6.4 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect + go.opentelemetry.io/otel v1.45.0 // indirect + go.opentelemetry.io/otel/metric v1.45.0 // indirect + go.opentelemetry.io/otel/sdk v1.45.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect + go.opentelemetry.io/otel/trace v1.45.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/term v0.45.0 // indirect + golang.org/x/text v0.41.0 // indirect + golang.org/x/time v0.14.0 // indirect + google.golang.org/api v0.267.0 // indirect + google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect + gopkg.in/ini.v1 v1.67.3 // indirect + oras.land/oras-go/v2 v2.6.2 // indirect ) diff --git a/go.sum b/go.sum index 85baef5..36902fa 100644 --- a/go.sum +++ b/go.sum @@ -1,72 +1,225 @@ +cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= +cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.18.2 h1:+Nbt5Ev0xEqxlNjd6c+yYUeosQ5TtEUaNcN/3FozlaM= +cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= +cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= +cloud.google.com/go/logging v1.13.1 h1:O7LvmO0kGLaHY/gq8cV7T0dyp6zJhYAOtZPX4TF3QtY= +cloud.google.com/go/logging v1.13.1/go.mod h1:XAQkfkMBxQRjQek96WLPNze7vsOmay9H5PqfsNYDqvw= +cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8= +cloud.google.com/go/longrunning v0.8.0/go.mod h1:UmErU2Onzi+fKDg2gR7dusz11Pe26aknR4kHmJJqIfk= +cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE= +cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI= +cloud.google.com/go/storage v1.60.0 h1:oBfZrSOCimggVNz9Y/bXY35uUcts7OViubeddTTVzQ8= +cloud.google.com/go/storage v1.60.0/go.mod h1:q+5196hXfejkctrnx+VYU8RKQr/L3c0cBIlrjmiAKE0= +cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U= +cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 h1:l7+6kwRMJNwdCvYdDl7Eax+wzEYHSnNY7zrrfbhDdTA= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0/go.mod h1:pJTkW8hEUIIi3Pf65lPZOnn4Y81yCllX6IWk2jNXdkM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0 h1:7t/qx5Ost0s0wbA/VDrByOooURhp+ikYwv20i9Y07TQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= +github.com/boljen/go-bitmap v0.0.0-20151001105940-23cd2fb0ce7d h1:zsO4lp+bjv5XvPTF58Vq+qgmZEYZttJK+CWtSZhKenI= +github.com/boljen/go-bitmap v0.0.0-20151001105940-23cd2fb0ce7d/go.mod h1:f1iKL6ZhUWvbk7PdWVmOaak10o86cqMUYEmn1CZNGEI= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= github.com/container-storage-interface/spec v1.12.0 h1:zrFOEqpR5AghNaaDG4qyedwPBqU2fU0dWjLQMP/azK0= github.com/container-storage-interface/spec v1.12.0/go.mod h1:txsm+MA2B2WDa5kW69jNbqPnvTtfvZma7T/zsAZ9qX8= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dchest/siphash v1.2.3 h1:QXwFc8cFOR2dSa/gE6o/HokBMWtLUaNDVd+22aKHeEA= +github.com/dchest/siphash v1.2.3/go.mod h1:0NvQU092bT0ipiFN++/rXm69QG9tVxLAlQHIXMPAkHc= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= -github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= +github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= +github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= +github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= +github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/folbricht/desync v1.1.5 h1:FxWghh6PzTglDW6zh6Gx235H33EffX+yxgJu2bBtc+A= +github.com/folbricht/desync v1.1.5/go.mod h1:AvU6wBGs+PrGrInuNo1FX9k+AGNPohdeB6/O4mgke7c= +github.com/folbricht/tempfile v0.0.1 h1:kB3DubP2Fm5e3W7TrWFNZBfzFEHBoKL7Pjn0HvqKxSQ= +github.com/folbricht/tempfile v0.0.1/go.mod h1:/Flpxx/6U+clQJ61jQ3y6Z7L2l6j1/ZSiU4B9EDPgWw= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= -github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao= +github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8= +github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc= +github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY= +github.com/hanwen/go-fuse/v2 v2.11.0 h1:CGVkJh9gRz0pTRMADNcqdFl3ec/5QbE/Vx1Gl7ESozM= +github.com/hanwen/go-fuse/v2 v2.11.0/go.mod h1:aU7NkGYZUmuJrZapoI3mEcNve7PZTySUOLBuch/vR6U= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= -github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= -github.com/minio/crc64nvme v1.0.2 h1:6uO1UxGAD+kwqWWp7mBFsi5gAse66C4NXO8cmcVculg= -github.com/minio/crc64nvme v1.0.2/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= +github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw= +github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU= -github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo= +github.com/minio/minio-go/v7 v7.3.0 h1:HM4pFCSQq/TK+j0/zmorSh5ddh81iDgRgU0BG0Vz/YU= +github.com/minio/minio-go/v7 v7.3.0/go.mod h1:KUPWdecEO1LWyUz+sTGXAuf2jZHrPh5fCsRH86QbPfk= +github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= +github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.13.5 h1:a3RLUqkyjYRtBTZJZ1VRrKbN3zhuPLlUc3sphVz81go= +github.com/pkg/sftp v1.13.5/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg= +github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE= +github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= -github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= -go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= -go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= -go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= -go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= -go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= -go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= -go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= -go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= -go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= -golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= -golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= -golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= -golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= -golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI= -google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/spiffe/go-spiffe/v2 v2.7.0 h1:uXe1MflJoHw58wAUvxVlcM7WpKtijWG7I1UidcGh6g4= +github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= +github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/detectors/gcp v1.44.0 h1:NmLfL734pJhM0JKaYd2Y28+nY9dPRWYAAbxhRCrKXPw= +go.opentelemetry.io/contrib/detectors/gcp v1.44.0/go.mod h1:tNAsgd8avTGke1+MndXlU5Cru4PQ9Ai/cCNWQv/ZJ/s= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= +go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= +go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0 h1:5gn2urDL/FBnK8OkCfD1j3/ER79rUuTYmCvlXBKeYL8= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0/go.mod h1:0fBG6ZJxhqByfFZDwSwpZGzJU671HkwpWaNe2t4VUPI= +go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= +go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= +go.opentelemetry.io/otel/metric/x v0.67.0 h1:PcicCNZFkZ4bXfSooXdo3WN7RBOVOtjVdo1wD358Uns= +go.opentelemetry.io/otel/metric/x v0.67.0/go.mod h1:FBjCWZe6wgcqxcMtjdGiClDKXb2YxxXii0CXftE4QtI= +go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= +go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= +go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= +go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA= +go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= +go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/api v0.267.0 h1:w+vfWPMPYeRs8qH1aYYsFX68jMls5acWl/jocfLomwE= +google.golang.org/api v0.267.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0= +google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM= +google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= +gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= +gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +oras.land/oras-go/v2 v2.6.2 h1:N04RXngAp1LJKTG6ifz3xHPipasEkWr+hFmInja5YKo= +oras.land/oras-go/v2 v2.6.2/go.mod h1:PlTtg4JTDJkDe8yVHpM2wz7/YDc00GVas+i4jAW2TZ4= diff --git a/pkg/casync/casync.go b/pkg/casync/casync.go index 504c51e..ad072ab 100644 --- a/pkg/casync/casync.go +++ b/pkg/casync/casync.go @@ -16,6 +16,7 @@ package casync import ( + "bytes" "context" "crypto/sha256" "encoding/hex" @@ -83,6 +84,8 @@ type FileEntry struct { Size int64 `json:"size"` Mode uint32 `json:"mode"` Index string `json:"index"` + // IndexSHA256 is the SHA-256 of the index file, so the manifest digest covers the content. + IndexSHA256 string `json:"index_sha256"` } // Artifact is the content of artifact.json. @@ -134,18 +137,16 @@ func (e *Engine) Publish(ctx context.Context, dir string) (digest.Digest, error) return digest.Digest{}, fmt.Errorf("chop %s: %w", rel, err) } name := fmt.Sprintf("%d.caibx", i) - f, err := os.Create(filepath.Join(tmp, name)) - if err != nil { - return digest.Digest{}, err - } - if _, err := idx.WriteTo(f); err != nil { - f.Close() + var buf bytes.Buffer + if _, err := idx.WriteTo(&buf); err != nil { return digest.Digest{}, err } - if err := f.Close(); err != nil { + if err := os.WriteFile(filepath.Join(tmp, name), buf.Bytes(), 0o644); err != nil { return digest.Digest{}, err } - a.Files = append(a.Files, FileEntry{Path: rel, Size: fi.Size(), Mode: uint32(fi.Mode().Perm()), Index: name}) + isum := sha256.Sum256(buf.Bytes()) + a.Files = append(a.Files, FileEntry{Path: rel, Size: fi.Size(), Mode: uint32(fi.Mode().Perm()), Index: name, + IndexSHA256: hex.EncodeToString(isum[:])}) } data, _ := json.Marshal(a) sum := sha256.Sum256(data) @@ -183,12 +184,14 @@ func (e *Engine) Load(d digest.Digest) (*Artifact, []desync.Index, error) { } idxs := make([]desync.Index, len(a.Files)) for i, f := range a.Files { - r, err := os.Open(filepath.Join(dir, f.Index)) + raw, err := os.ReadFile(filepath.Join(dir, f.Index)) if err != nil { return nil, nil, err } - idxs[i], err = desync.IndexFromReader(r) - r.Close() + if isum := sha256.Sum256(raw); hex.EncodeToString(isum[:]) != f.IndexSHA256 { + return nil, nil, fmt.Errorf("%s: index does not match the manifest", f.Path) + } + idxs[i], err = desync.IndexFromReader(bytes.NewReader(raw)) if err != nil { return nil, nil, fmt.Errorf("%s: %w", f.Path, err) } From 4626388b71722192c354c85bd6e21019781c9dee Mon Sep 17 00:00:00 2001 From: gspark-etri Date: Wed, 11 Nov 2026 18:46:50 +0900 Subject: [PATCH 08/16] casync engine in the CSI plugin; ownership integration test onggi csi -engine casync -casync-store S -casync-index I materializes with the casync engine; kept chunks are read back only when the base is not attested unchanged (or with -casync-verify-always). A materializer test checks that a sibling in use is never consumed and that a released idle sibling is the one consumed. Co-Authored-By: Claude Opus 5.5 --- cmd/onggi/csi.go | 18 ++++++++ pkg/casync/materializer_test.go | 74 +++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 pkg/casync/materializer_test.go diff --git a/cmd/onggi/csi.go b/cmd/onggi/csi.go index 974dcaa..a8ce63c 100644 --- a/cmd/onggi/csi.go +++ b/cmd/onggi/csi.go @@ -17,6 +17,7 @@ import ( bfs "github.com/ai-computing/onggi/pkg/backend/fs" "github.com/ai-computing/onggi/pkg/csi" "github.com/ai-computing/onggi/pkg/digest" + "github.com/ai-computing/onggi/pkg/casync" "github.com/ai-computing/onggi/pkg/peer" "github.com/ai-computing/onggi/pkg/store" ) @@ -41,6 +42,10 @@ func csiNode(args []string, stderr io.Writer) int { maxArtifacts := fs.Int("max-artifacts", 0, "artifacts kept per node; a pull takes an idle one over only when the node holds this many, least recently used first (0: no limit)") verifyKept := fs.Bool("verify-kept", true, "read back and hash-check every chunk a pull leaves in place when it takes an artifact over") baseCopy := fs.String("base-copy", "remote", "read an idle artifact that is not taken over where it is: remote (only if the store is not local), always, never") + engine := fs.String("engine", "store", "how artifacts are materialized: store (onggi packs) or casync (desync chunk store with state-aware in-place reuse)") + casyncStore := fs.String("casync-store", "", "casync engine: desync local chunk store") + casyncIndex := fs.String("casync-index", "", "casync engine: directory of artifact indexes (onggi casync publish)") + casyncVerify := fs.Bool("casync-verify-always", false, "casync engine: read back kept chunks even when the base is attested unchanged") peerListen := fs.String("peer-listen", "", "serve the node cache to other nodes' pulls on this address, e.g. :7070 (#37)") peers := fs.String("peers", "", "comma-separated peer URLs asked for chunks before the store") peersDNS := fs.String("peers-dns", "", "host:port whose name resolves to the peers, e.g. a headless Service; re-resolved every 30 s") @@ -103,6 +108,19 @@ func csiNode(args []string, stderr io.Writer) int { } } m := &csi.Materializer{Root: *dataDir, Pull: csi.StorePullerPeers(s, openCache, peerList, tmp, *workers), Logf: logger.Printf, MaxPulls: *pulls} + if *engine == "casync" { + e, err := casync.New(*casyncStore, *casyncIndex, filepath.Join(*dataDir, "casync-attest"), *workers) + if err != nil || *casyncStore == "" || *casyncIndex == "" { + logger.Print("casync engine: needs -casync-store and -casync-index: ", err) + return 2 + } + e.VerifyKept, e.Logf = *casyncVerify, logger.Printf + m.Pull = e.Pull + *verifyKept = *casyncVerify // the engine decides from the attestation; the offer forces a read-back only on request + } else if *engine != "store" { + logger.Printf("-engine: %q is not store or casync", *engine) + return 2 + } if err := m.Init(); err != nil { logger.Print("data dir: ", err) return 1 diff --git a/pkg/casync/materializer_test.go b/pkg/casync/materializer_test.go new file mode 100644 index 0000000..70bb78c --- /dev/null +++ b/pkg/casync/materializer_test.go @@ -0,0 +1,74 @@ +package casync + +import ( + "bytes" + "context" + "math/rand" + "os" + "path/filepath" + "testing" + "time" + + "github.com/ai-computing/onggi/pkg/csi" + "github.com/ai-computing/onggi/pkg/store" +) + +// TestOwnership runs the materializer with the casync engine: a sibling a volume uses is never +// consumed (the new variant is built without destroying it), and once released and idle it is the +// one consumed, writing only what differs. +func TestOwnership(t *testing.T) { + ctx := context.Background() + e, root := setup(t) + base, aligned, _ := variants() + third := append([]byte(nil), aligned...) + rand.New(rand.NewSource(9)).Read(third[1<<20 : 1<<20+200<<10]) + mA, _ := e.Publish(ctx, publish(t, e, root, "a", base)) + mB, _ := e.Publish(ctx, publish(t, e, root, "b", aligned)) + mC, _ := e.Publish(ctx, publish(t, e, root, "c", third)) + + m := &csi.Materializer{Root: filepath.Join(root, "node"), Pull: e.Pull, RebaseIdle: time.Nanosecond, + MaxArtifacts: 1, BaseCopy: store.BaseCopyAlways} + if err := m.Init(); err != nil { + t.Fatal(err) + } + dA, err := m.Ensure(ctx, mA) + if err != nil { + t.Fatal(err) + } + if err := m.Acquire("vol-a", mA, "/pods/a"); err != nil { + t.Fatal(err) + } + // A is in use: B must not consume it + dB, err := m.Ensure(ctx, mB) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(readFile(t, filepath.Join(dA, "model.bin")), base) { + t.Fatal("a sibling in use was modified") + } + if !bytes.Equal(readFile(t, filepath.Join(dB, "model.bin")), aligned) { + t.Fatal("B is wrong") + } + // A released and idle, B idle: C consumes the least recently used idle sibling (A) + if _, _, _, err := m.Release("vol-a"); err != nil { + t.Fatal(err) + } + time.Sleep(20 * time.Millisecond) + dC, err := m.Ensure(ctx, mC) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(readFile(t, filepath.Join(dC, "model.bin")), third) { + t.Fatal("C is wrong") + } + _, errA := os.Stat(dA) + _, errB := os.Stat(dB) + if errA == nil && errB == nil { + t.Fatal("C consumed no idle sibling") + } + if errB != nil { + if !bytes.Equal(readFile(t, filepath.Join(dA, "model.bin")), base) { + t.Fatal("A changed although B was consumed") + } + } +} From ef2f7988bf94056904e547a2b5df8128b507e5a7 Mon Sep 17 00:00:00 2001 From: gspark-etri Date: Wed, 11 Nov 2026 19:56:08 +0900 Subject: [PATCH 09/16] casync: honour BaseCopyRemote with a local store; test the victim choice Audit findings: the engine read a sibling (Copy) whenever one was readable, although with a local chunk store BaseCopyRemote means assemble from the store. The ownership test accepted either idle sibling; it now checks that the least recently used idle sibling is consumed (Release counts as a use) and that a released sibling is consumed once it becomes the victim. Co-Authored-By: Claude Opus 5.5 --- pkg/casync/casync.go | 4 +++- pkg/casync/materializer_test.go | 26 +++++++++++++++++--------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/pkg/casync/casync.go b/pkg/casync/casync.go index ad072ab..4f2dc9d 100644 --- a/pkg/casync/casync.go +++ b/pkg/casync/casync.go @@ -376,7 +376,9 @@ func (e *Engine) Pull(ctx context.Context, m digest.Digest, dest string, offer * // Copy: a base that stays where it is seeds the files Keep did not produce var release func() var seedBase *keepPlan - if offer != nil && len(offer.Readable) > 0 && offer.BaseCopy != store.BaseCopyNever { + // The chunk store of this engine is local, so reading a sibling instead of the store is chosen + // only when asked for explicitly (BaseCopyAlways); BaseCopyRemote means "only for a remote store". + if offer != nil && len(offer.Readable) > 0 && offer.BaseCopy == store.BaseCopyAlways { var most keepPlan for _, b := range offer.Readable { if p, err := e.plan(a, idxs, b); err == nil && p.same > most.same { diff --git a/pkg/casync/materializer_test.go b/pkg/casync/materializer_test.go index 70bb78c..defad69 100644 --- a/pkg/casync/materializer_test.go +++ b/pkg/casync/materializer_test.go @@ -49,7 +49,7 @@ func TestOwnership(t *testing.T) { if !bytes.Equal(readFile(t, filepath.Join(dB, "model.bin")), aligned) { t.Fatal("B is wrong") } - // A released and idle, B idle: C consumes the least recently used idle sibling (A) + // Release marks A used now, so B is the least recently used idle sibling: C consumes B, not A if _, _, _, err := m.Release("vol-a"); err != nil { t.Fatal(err) } @@ -61,14 +61,22 @@ func TestOwnership(t *testing.T) { if !bytes.Equal(readFile(t, filepath.Join(dC, "model.bin")), third) { t.Fatal("C is wrong") } - _, errA := os.Stat(dA) - _, errB := os.Stat(dB) - if errA == nil && errB == nil { - t.Fatal("C consumed no idle sibling") + if _, err := os.Stat(dB); !os.IsNotExist(err) { + t.Fatal("C did not consume B, the least recently used idle sibling") } - if errB != nil { - if !bytes.Equal(readFile(t, filepath.Join(dA, "model.bin")), base) { - t.Fatal("A changed although B was consumed") - } + if !bytes.Equal(readFile(t, filepath.Join(dA, "model.bin")), base) { + t.Fatal("A changed although it was not the victim") + } + // next, the released A is the least recently used idle sibling and is the one consumed + time.Sleep(20 * time.Millisecond) + dB2, err := m.Ensure(ctx, mB) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(dA); !os.IsNotExist(err) { + t.Fatal("the released sibling A was not consumed when it became the victim") + } + if !bytes.Equal(readFile(t, filepath.Join(dB2, "model.bin")), aligned) { + t.Fatal("B (second time) is wrong") } } From 50867a778bc88d12e57f130200687ee57f643eaa Mon Sep 17 00:00:00 2001 From: gspark-etri Date: Wed, 11 Nov 2026 21:06:43 +0900 Subject: [PATCH 10/16] casync: pair renamed files by same-offset bytes so Keep works across shape-named variants Co-Authored-By: Claude Opus 5.5 --- pkg/casync/casync.go | 30 +++++++++++++++++++++++++++ pkg/casync/casync_test.go | 43 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/pkg/casync/casync.go b/pkg/casync/casync.go index 4f2dc9d..786b0a0 100644 --- a/pkg/casync/casync.go +++ b/pkg/casync/casync.go @@ -248,6 +248,36 @@ func (e *Engine) plan(target *Artifact, tidx []desync.Index, b store.Base) (keep p.kept[i], s = sameOffset(bidx[j].Chunks, tidx[i].Chunks) p.same += s } + // Variants of different shapes usually name their files differently (decode_S256_B2 vs _B8). + // A target file without a namesake takes the unused base file that holds the most of its + // bytes at the same offset, decided from the two indexes alone. + used := map[int]bool{} + for _, j := range p.pair { + if j >= 0 { + used[j] = true + } + } + for i := range target.Files { + if p.pair[i] >= 0 { + continue + } + best, bestSame := -1, int64(0) + var bestKept []bool + for j := range ba.Files { + if used[j] { + continue + } + k, s := sameOffset(bidx[j].Chunks, tidx[i].Chunks) + if s > bestSame { + best, bestSame, bestKept = j, s, k + } + } + if best >= 0 { + p.pair[i], p.kept[i] = best, bestKept + p.same += bestSame + used[best] = true + } + } return p, nil } diff --git a/pkg/casync/casync_test.go b/pkg/casync/casync_test.go index b7856bc..b6e9f6b 100644 --- a/pkg/casync/casync_test.go +++ b/pkg/casync/casync_test.go @@ -152,3 +152,46 @@ func TestSameOffset(t *testing.T) { t.Fatalf("same=%d of %d", same, len(aligned)) } } + +// TestKeepRenamed: variants of different shapes name their files differently; Keep pairs the +// target file with the base file that holds most of it at the same offset. +func TestKeepRenamed(t *testing.T) { + ctx := context.Background() + e, root := setup(t) + base, aligned, _ := variants() + write := func(name, file string, data []byte) string { + d := filepath.Join(root, "src-"+name) + os.MkdirAll(d, 0o755) + if err := os.WriteFile(filepath.Join(d, file), data, 0o644); err != nil { + t.Fatal(err) + } + return d + } + mA, err := e.Publish(ctx, write("a", "decode_B2.bin", base)) + if err != nil { + t.Fatal(err) + } + mB, err := e.Publish(ctx, write("b", "decode_B8.bin", aligned)) + if err != nil { + t.Fatal(err) + } + dA := filepath.Join(root, "art", "A") + os.MkdirAll(dA, 0o755) + if _, err := e.Pull(ctx, mA, dA, nil); err != nil { + t.Fatal(err) + } + offer := &csi.Offer{Bases: []store.Base{{Manifest: mA, Dir: dA}}, + Claim: func(b store.Base) (string, bool) { return b.Dir, true }} + dB := filepath.Join(root, "art", "B") + os.MkdirAll(dB, 0o755) + st, err := e.Pull(ctx, mB, dB, offer) + if err != nil || st.Base != mA.String() || st.Verified != 0 || st.Reused < int64(len(aligned))*8/10 { + t.Fatalf("keep renamed: %v %+v", err, st) + } + if !bytes.Equal(readFile(t, filepath.Join(dB, "decode_B8.bin")), aligned) { + t.Fatal("keep renamed: output differs") + } + if _, err := os.Stat(filepath.Join(dA, "decode_B2.bin")); !os.IsNotExist(err) { + t.Fatal("keep renamed must consume the base file") + } +} From a4f2730ce910c001830a2c8195323d3c51e19c12 Mon Sep 17 00:00:00 2001 From: gspark-etri Date: Wed, 11 Nov 2026 21:26:49 +0900 Subject: [PATCH 11/16] csi: with a capacity the eviction victim needs no idle window; casync Copy uses the tau threshold The victim is deleted by eviction anyway, so waiting only loses its bytes; leases still guard it. Copy now applies the same share as Keep (Algorithm 1 line 7). Co-Authored-By: Claude Opus 5.5 --- pkg/casync/casync.go | 3 ++- pkg/casync/materializer_test.go | 35 +++++++++++++++++++++++++++++++++ pkg/csi/materializer.go | 9 +++++++-- 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/pkg/casync/casync.go b/pkg/casync/casync.go index 786b0a0..bbff175 100644 --- a/pkg/casync/casync.go +++ b/pkg/casync/casync.go @@ -415,7 +415,8 @@ func (e *Engine) Pull(ctx context.Context, m digest.Digest, dest string, offer * most = p } } - if most.same > 0 { + // the same threshold as Keep (Algorithm 1): a sibling that holds less is not worth reading + if most.same > 0 && float64(most.same) >= minSame*float64(st.Bytes) { ok := true release = func() {} if offer.Hold != nil { diff --git a/pkg/casync/materializer_test.go b/pkg/casync/materializer_test.go index defad69..67c5159 100644 --- a/pkg/casync/materializer_test.go +++ b/pkg/casync/materializer_test.go @@ -80,3 +80,38 @@ func TestOwnership(t *testing.T) { t.Fatal("B (second time) is wrong") } } + +// TestVictimNeedsNoIdleWindow: with a capacity, the eviction victim is consumed as soon as no +// lease holds it, even if it was used a moment ago and the idle window is long. +func TestVictimNeedsNoIdleWindow(t *testing.T) { + ctx := context.Background() + e, root := setup(t) + base, aligned, _ := variants() + mA, _ := e.Publish(ctx, publish(t, e, root, "a", base)) + mB, _ := e.Publish(ctx, publish(t, e, root, "b", aligned)) + m := &csi.Materializer{Root: filepath.Join(root, "node"), Pull: e.Pull, RebaseIdle: time.Hour, + MaxArtifacts: 1} + if err := m.Init(); err != nil { + t.Fatal(err) + } + dA, err := m.Ensure(ctx, mA) + if err != nil { + t.Fatal(err) + } + if err := m.Acquire("vol-a", mA, "/pods/a"); err != nil { + t.Fatal(err) + } + if _, _, _, err := m.Release("vol-a"); err != nil { + t.Fatal(err) + } + dB, err := m.Ensure(ctx, mB) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(dA); !os.IsNotExist(err) { + t.Fatal("the just-released victim was not consumed") + } + if !bytes.Equal(readFile(t, filepath.Join(dB, "model.bin")), aligned) { + t.Fatal("B is wrong") + } +} diff --git a/pkg/csi/materializer.go b/pkg/csi/materializer.go index e6c6fc0..c5729e8 100644 --- a/pkg/csi/materializer.go +++ b/pkg/csi/materializer.go @@ -497,12 +497,17 @@ func (m *Materializer) offer(d digest.Digest) (*Offer, *[]string) { Readable: readable, Hold: hold}, claimed } -// idle says whether artifact d may be consumed: materialized, no volume, no pull, unused for -// RebaseIdle. The caller holds m.mu. +// idle says whether artifact d may be consumed: materialized, no volume, no pull, and, on a node +// without a capacity, unused for RebaseIdle. With a capacity the victim is the artifact eviction +// would delete anyway, so waiting would only lose its bytes: no lease is all it needs. +// The caller holds m.mu. func (m *Materializer) idle(d digest.Digest, now time.Time) bool { if m.busy(d) || !isDir(m.Dir(d)) { return false } + if m.MaxArtifacts > 0 { + return true + } return now.Sub(m.lastUsed(d)) >= m.RebaseIdle } From 53d842dcf96a3568b59e274f089de6ce2c396957 Mon Sep 17 00:00:00 2001 From: gspark-etri Date: Thu, 12 Nov 2026 08:59:41 +0900 Subject: [PATCH 12/16] casync: chunks subcommand dumps the chunk list of a manifest for capacity and transfer accounting Co-Authored-By: Claude Opus 5.5 --- cmd/onggi/casync.go | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/cmd/onggi/casync.go b/cmd/onggi/casync.go index 9faa280..4579eb2 100644 --- a/cmd/onggi/casync.go +++ b/cmd/onggi/casync.go @@ -19,6 +19,7 @@ import ( const casyncUsage = `onggi casync — artifacts in casync format (desync chunk store), with state-aware in-place reuse onggi casync publish --store S --index I chunk dir into S; print the manifest digest + onggi casync chunks --store S --index I print the chunk list of every file (JSON) onggi casync pull --store S --index I [--attest A] [--base M --base-dir D] take base M over (Keep): write only differing chunks; kept chunks are read back unless A attests the base unchanged since it was written @@ -76,6 +77,40 @@ func casyncCmd(args []string, stdout, stderr io.Writer) int { } fmt.Fprintf(stdout, "%s\t%.2fs\n", d, time.Since(start).Seconds()) return 0 + case "chunks": + // Dump the chunk list of every file of a manifest (start, size, id) as JSON, for capacity + // and transfer accounting outside the engine. + if fs.NArg() != 1 { + fmt.Fprint(stderr, casyncUsage) + return 2 + } + m, err := digest.Parse(fs.Arg(0)) + if err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + a, idxs, err := e.Load(m) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + type fileChunks struct { + Path string `json:"path"` + Size int64 `json:"size"` + Chunks [][3]string `json:"chunks"` + } + out := make([]fileChunks, len(a.Files)) + for i, f := range a.Files { + out[i] = fileChunks{Path: f.Path, Size: f.Size} + for _, c := range idxs[i].Chunks { + out[i].Chunks = append(out[i].Chunks, [3]string{fmt.Sprint(c.Start), fmt.Sprint(c.Size), c.ID.String()}) + } + } + if err := json.NewEncoder(stdout).Encode(out); err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + return 0 case "pull", "verify", "attest": if fs.NArg() != 2 { fmt.Fprint(stderr, casyncUsage) From 8aaa0c7d7419bc68be80bd999d7b3a623a33dca0 Mon Sep 17 00:00:00 2001 From: gspark-etri Date: Thu, 12 Nov 2026 09:16:40 +0900 Subject: [PATCH 13/16] casync: chunk source may be a remote HTTP chunk server (--remote); pulls count fetched chunks; Copy seeds from a sibling holding the target at any offset Co-Authored-By: Claude Opus 5.5 --- cmd/onggi/casync.go | 10 +++++ pkg/casync/casync.go | 94 +++++++++++++++++++++++++++++++-------- pkg/casync/casync_test.go | 35 +++++++++++++++ pkg/store/pull.go | 2 + 4 files changed, 123 insertions(+), 18 deletions(-) diff --git a/cmd/onggi/casync.go b/cmd/onggi/casync.go index 4579eb2..0ce5327 100644 --- a/cmd/onggi/casync.go +++ b/cmd/onggi/casync.go @@ -7,9 +7,12 @@ import ( "fmt" "io" "log" + "net/url" "os" "time" + "github.com/folbricht/desync" + "github.com/ai-computing/onggi/pkg/casync" "github.com/ai-computing/onggi/pkg/csi" "github.com/ai-computing/onggi/pkg/digest" @@ -47,11 +50,18 @@ func casyncCmd(args []string, stdout, stderr io.Writer) int { verifyKept := fs.Bool("verify-kept", false, "read back kept chunks even if the base is attested unchanged") noSync := fs.Bool("no-sync", false, "do not sync written files (desync's default behaviour)") logPhases := fs.Bool("log", false, "log phase durations to stderr") + remote := fs.String("remote", "", "pull: read chunks from this HTTP chunk server instead of --store") if err := fs.Parse(args[1:]); err != nil || *storeDir == "" || *indexDir == "" { fmt.Fprint(stderr, casyncUsage) return 2 } e, err := casync.New(*storeDir, *indexDir, *attestDir, *workers) + if err == nil && *remote != "" { + var u *url.URL + if u, err = url.Parse(*remote); err == nil { + e.Src, err = desync.NewRemoteHTTPStore(u, desync.StoreOptions{N: *workers, Timeout: 5 * time.Minute}) + } + } if err != nil { fmt.Fprintln(stderr, err) return 1 diff --git a/pkg/casync/casync.go b/pkg/casync/casync.go index bbff175..6da7d04 100644 --- a/pkg/casync/casync.go +++ b/pkg/casync/casync.go @@ -47,6 +47,9 @@ const ( // Engine reads and writes artifacts in casync format. type Engine struct { Store desync.LocalStore + // Src, if set, is where pulls read chunks from instead of Store (for example a remote HTTP + // chunk server); Store still receives published chunks. + Src desync.Store IndexDir string AttestDir string // "" : never trust kept chunks, always read them back Workers int @@ -261,17 +264,22 @@ func (e *Engine) plan(target *Artifact, tidx []desync.Index, b store.Base) (keep if p.pair[i] >= 0 { continue } - best, bestSame := -1, int64(0) + best, bestSame, bestShared := -1, int64(0), int64(0) var bestKept []bool for j := range ba.Files { if used[j] { continue } k, s := sameOffset(bidx[j].Chunks, tidx[i].Chunks) - if s > bestSame { - best, bestSame, bestKept = j, s, k + // prefer bytes at the same offset (Keep); otherwise bytes held anywhere (a seed for Copy) + sh := sharedBytes([]desync.Index{bidx[j]}, []desync.Index{tidx[i]}) + if s > bestSame || (s == bestSame && sh > bestShared) { + best, bestSame, bestShared, bestKept = j, s, sh, k } } + if bestSame == 0 && bestShared == 0 { + best = -1 + } if best >= 0 { p.pair[i], p.kept[i] = best, bestKept p.same += bestSame @@ -357,7 +365,34 @@ func (e *Engine) unchanged(d digest.Digest, path, p string) bool { // Pull writes artifact m into the empty directory dest. With an offer it takes an idle aligned // base over (Keep), or reads a base that stays intact (Copy); otherwise it assembles from the // store (Reconstruct). It satisfies csi.Puller. -func (e *Engine) Pull(ctx context.Context, m digest.Digest, dest string, offer *csi.Offer) (store.PullStats, error) { +func (e *Engine) Pull(ctx context.Context, m digest.Digest, dest string, offer *csi.Offer) (st store.PullStats, err error) { + var src desync.Store = e.Store + if e.Src != nil { + src = e.Src + } + cs := &countingStore{Store: src} + defer func() { st.FetchedChunks, st.FetchedBytes = cs.n.Load(), cs.b.Load() }() + return e.pull(ctx, m, dest, offer, cs) +} + +// countingStore counts the chunks a pull takes from its chunk source and their logical size. +type countingStore struct { + desync.Store + n, b atomic.Int64 +} + +func (c *countingStore) GetChunk(id desync.ChunkID) (*desync.Chunk, error) { + ch, err := c.Store.GetChunk(id) + if err == nil { + if d, derr := ch.Data(); derr == nil { + c.n.Add(1) + c.b.Add(int64(len(d))) + } + } + return ch, err +} + +func (e *Engine) pull(ctx context.Context, m digest.Digest, dest string, offer *csi.Offer, src desync.Store) (store.PullStats, error) { start := time.Now() a, idxs, err := e.Load(m) if err != nil { @@ -397,7 +432,7 @@ func (e *Engine) Pull(ctx context.Context, m digest.Digest, dest string, offer * if best.same > 0 && float64(best.same) >= minSame*float64(st.Bytes) { if dir, ok := offer.Claim(best.base); ok { st.Base = best.base.Manifest.String() - if err := e.keep(ctx, a, idxs, best, dir, finals, done, offer.VerifyKept || e.VerifyKept, &st); err != nil { + if err := e.keep(ctx, a, idxs, best, dir, finals, done, offer.VerifyKept || e.VerifyKept, &st, src); err != nil { return st, err } } @@ -406,17 +441,22 @@ func (e *Engine) Pull(ctx context.Context, m digest.Digest, dest string, offer * // Copy: a base that stays where it is seeds the files Keep did not produce var release func() var seedBase *keepPlan - // The chunk store of this engine is local, so reading a sibling instead of the store is chosen - // only when asked for explicitly (BaseCopyAlways); BaseCopyRemote means "only for a remote store". - if offer != nil && len(offer.Readable) > 0 && offer.BaseCopy == store.BaseCopyAlways { + // Reading a sibling instead of the chunk source pays off when that source is remote (Src set, + // BaseCopyRemote) or when asked for explicitly (BaseCopyAlways). + copyOK := offer != nil && (offer.BaseCopy == store.BaseCopyAlways || (offer.BaseCopy == store.BaseCopyRemote && e.Src != nil)) + if copyOK && len(offer.Readable) > 0 { var most keepPlan + var mostShared int64 for _, b := range offer.Readable { - if p, err := e.plan(a, idxs, b); err == nil && p.same > most.same { - most = p + if p, err := e.plan(a, idxs, b); err == nil { + // a seed serves chunks at any offset, so count the target bytes the sibling holds anywhere + if sh := sharedBytes(p.idxs, idxs); sh > mostShared { + most, mostShared = p, sh + } } } - // the same threshold as Keep (Algorithm 1): a sibling that holds less is not worth reading - if most.same > 0 && float64(most.same) >= minSame*float64(st.Bytes) { + // the same threshold as Keep (Algorithm 1), on bytes held anywhere in the sibling + if mostShared > 0 && float64(mostShared) >= minSame*float64(st.Bytes) { ok := true release = func() {} if offer.Hold != nil { @@ -446,11 +486,11 @@ func (e *Engine) Pull(ctx context.Context, m digest.Digest, dest string, offer * s, err := desync.NewIndexSeed(partial, src, seedBase.idxs[j]) if err == nil { seeds = append(seeds, s) - st.LocalBytes += countKept(seedBase.kept[i], idxs[i].Chunks) + st.LocalBytes += sharedBytes([]desync.Index{seedBase.idxs[j]}, []desync.Index{idxs[i]}) } } t0 := time.Now() - if _, err := desync.AssembleFile(ctx, partial, idxs[i], e.Store, seeds, desync.AssembleOptions{N: e.Workers, InvalidSeedAction: desync.InvalidSeedActionSkip}); err != nil { + if _, err := desync.AssembleFile(ctx, partial, idxs[i], src, seeds, desync.AssembleOptions{N: e.Workers, InvalidSeedAction: desync.InvalidSeedActionSkip}); err != nil { os.Remove(partial) return st, fmt.Errorf("%s: %w", a.Files[i].Path, err) } @@ -484,7 +524,7 @@ func countKept(kept []bool, chunks []desync.IndexChunk) int64 { // keep turns the claimed base's files into the target's: each paired file is moved into place, // resized, and only its differing chunks are written from the store. Kept chunks are read back and // hash-checked unless the file is attested unchanged since this engine wrote it. -func (e *Engine) keep(ctx context.Context, a *Artifact, idxs []desync.Index, p keepPlan, dir string, finals []string, done []bool, verify bool, st *store.PullStats) error { +func (e *Engine) keep(ctx context.Context, a *Artifact, idxs []desync.Index, p keepPlan, dir string, finals []string, done []bool, verify bool, st *store.PullStats, src desync.Store) error { for i, j := range p.pair { if j < 0 { continue @@ -515,7 +555,7 @@ func (e *Engine) keep(ctx context.Context, a *Artifact, idxs []desync.Index, p k st.Repaired += bad } t1 := time.Now() - written, err := e.writeChunks(ctx, f, idxs[i].Chunks, kept) + written, err := e.writeChunks(ctx, f, idxs[i].Chunks, kept, src) if err != nil { f.Close() return err @@ -583,7 +623,7 @@ func (e *Engine) verifyKept(ctx context.Context, f *os.File, chunks []desync.Ind // writeChunks writes every chunk that is not kept, from the store (each chunk's digest is checked // when it is read from the store). It returns the bytes written. -func (e *Engine) writeChunks(ctx context.Context, f *os.File, chunks []desync.IndexChunk, kept []bool) (int64, error) { +func (e *Engine) writeChunks(ctx context.Context, f *os.File, chunks []desync.IndexChunk, kept []bool, src desync.Store) (int64, error) { var written atomic.Int64 var firstErr error var once sync.Once @@ -595,7 +635,7 @@ func (e *Engine) writeChunks(ctx context.Context, f *os.File, chunks []desync.In defer wg.Done() for i := range work { c := chunks[i] - ch, err := e.Store.GetChunk(c.ID) + ch, err := src.GetChunk(c.ID) if err == nil { var b []byte if b, err = ch.Data(); err == nil { @@ -670,3 +710,21 @@ func (e *Engine) Verify(ctx context.Context, m digest.Digest, dir string) error return nil } +// sharedBytes counts the bytes of the target's chunks that also occur, at any offset, in the base. +func sharedBytes(base, target []desync.Index) int64 { + have := map[desync.ChunkID]bool{} + for _, x := range base { + for _, c := range x.Chunks { + have[c.ID] = true + } + } + var n int64 + for _, x := range target { + for _, c := range x.Chunks { + if have[c.ID] { + n += int64(c.Size) + } + } + } + return n +} diff --git a/pkg/casync/casync_test.go b/pkg/casync/casync_test.go index b6e9f6b..cd9330d 100644 --- a/pkg/casync/casync_test.go +++ b/pkg/casync/casync_test.go @@ -195,3 +195,38 @@ func TestKeepRenamed(t *testing.T) { t.Fatal("keep renamed must consume the base file") } } + +// TestCopyShiftedFetchesLittle: with a separate chunk source (as a remote store would be), Copy +// seeds from a sibling whose content sits at other offsets and fetches only what it lacks. +func TestCopyShiftedFetchesLittle(t *testing.T) { + ctx := context.Background() + e, root := setup(t) + base, _, shifted := variants() + mA, _ := e.Publish(ctx, publish(t, e, root, "a", base)) + mC, _ := e.Publish(ctx, publish(t, e, root, "c", shifted)) + dA := filepath.Join(root, "art", "A") + os.MkdirAll(dA, 0o755) + if _, err := e.Pull(ctx, mA, dA, nil); err != nil { + t.Fatal(err) + } + e.Src = e.Store // a distinct chunk source: Copy is chosen with BaseCopyRemote + dC := filepath.Join(root, "art", "C") + os.MkdirAll(dC, 0o755) + st, err := e.Pull(ctx, mC, dC, &csi.Offer{Readable: []store.Base{{Manifest: mA, Dir: dA}}, BaseCopy: store.BaseCopyRemote}) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(readFile(t, filepath.Join(dC, "model.bin")), shifted) { + t.Fatal("copy shifted: output differs") + } + if st.FetchedBytes*5 > int64(len(shifted)) || st.LocalBytes == 0 { + t.Fatalf("copy shifted fetched %d of %d B (local %d B)", st.FetchedBytes, len(shifted), st.LocalBytes) + } + // without a sibling the whole target comes from the chunk source + dC2 := filepath.Join(root, "art", "C2") + os.MkdirAll(dC2, 0o755) + st2, err := e.Pull(ctx, mC, dC2, nil) + if err != nil || st2.FetchedBytes < int64(len(shifted)) { + t.Fatalf("reconstruct fetched %d of %d B: %v", st2.FetchedBytes, len(shifted), err) + } +} diff --git a/pkg/store/pull.go b/pkg/store/pull.go index 44bba01..fc05c13 100644 --- a/pkg/store/pull.go +++ b/pkg/store/pull.go @@ -84,6 +84,8 @@ type PullStats struct { LocalBase string `json:"local_base,omitempty"` // a base read where it is, not consumed (#58) LocalBytes int64 `json:"local_bytes"` // bytes read from that base's files Ranges int `json:"ranges"` + FetchedChunks int64 `json:"fetched_chunks"` // chunks taken from the chunk source (casync engine) + FetchedBytes int64 `json:"fetched_bytes"` // their logical size Duration time.Duration `json:"duration_ns"` } From b87c6e4388ac6d787ca89e39243094c1df413ffc Mon Sep 17 00:00:00 2001 From: gspark-etri Date: Thu, 12 Nov 2026 09:17:11 +0900 Subject: [PATCH 14/16] casync: keep's chunk source parameter no longer shadowed by the base path Co-Authored-By: Claude Opus 5.5 --- pkg/casync/casync.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/casync/casync.go b/pkg/casync/casync.go index 6da7d04..be714e7 100644 --- a/pkg/casync/casync.go +++ b/pkg/casync/casync.go @@ -524,7 +524,7 @@ func countKept(kept []bool, chunks []desync.IndexChunk) int64 { // keep turns the claimed base's files into the target's: each paired file is moved into place, // resized, and only its differing chunks are written from the store. Kept chunks are read back and // hash-checked unless the file is attested unchanged since this engine wrote it. -func (e *Engine) keep(ctx context.Context, a *Artifact, idxs []desync.Index, p keepPlan, dir string, finals []string, done []bool, verify bool, st *store.PullStats, src desync.Store) error { +func (e *Engine) keep(ctx context.Context, a *Artifact, idxs []desync.Index, p keepPlan, dir string, finals []string, done []bool, verify bool, st *store.PullStats, chunkSrc desync.Store) error { for i, j := range p.pair { if j < 0 { continue @@ -555,7 +555,7 @@ func (e *Engine) keep(ctx context.Context, a *Artifact, idxs []desync.Index, p k st.Repaired += bad } t1 := time.Now() - written, err := e.writeChunks(ctx, f, idxs[i].Chunks, kept, src) + written, err := e.writeChunks(ctx, f, idxs[i].Chunks, kept, chunkSrc) if err != nil { f.Close() return err From 892ab546ce74c506f79e5c564b760dbea1696290 Mon Sep 17 00:00:00 2001 From: gspark-etri Date: Thu, 12 Nov 2026 14:49:50 +0900 Subject: [PATCH 15/16] tools/varmorph: layout coder, artifact classifier, device rebuild (in place, batched), NPU copy benchmark, NPU-B batched-copy shim Co-Authored-By: Claude Opus 5.5 --- tools/varmorph/README.md | 7 + tools/varmorph/classify.py | 157 +++++++++ tools/varmorph/device_rebuild.py | 522 +++++++++++++++++++++++++++++ tools/varmorph/layout_coder.py | 548 +++++++++++++++++++++++++++++++ tools/varmorph/npu_d2d_bench.py | 210 ++++++++++++ tools/varmorph/rbln_vmshim.cpp | 38 +++ 6 files changed, 1482 insertions(+) create mode 100644 tools/varmorph/README.md create mode 100644 tools/varmorph/classify.py create mode 100644 tools/varmorph/device_rebuild.py create mode 100644 tools/varmorph/layout_coder.py create mode 100644 tools/varmorph/npu_d2d_bench.py create mode 100644 tools/varmorph/rbln_vmshim.cpp diff --git a/tools/varmorph/README.md b/tools/varmorph/README.md new file mode 100644 index 0000000..108064a --- /dev/null +++ b/tools/varmorph/README.md @@ -0,0 +1,7 @@ +# VarMorph layout coding and device rebuild (prototype) + +- `classify.py`: classifies each 64 KiB window of a compiled artifact (weights by numeric type, code, padding, text) from byte statistics, without parsing the format. +- `layout_coder.py`: codes a target variant against sibling(s) at any byte offset below chunk size (sub-block shift search, motion prediction with merge candidates, run merging, XOR residual, intra codec with bf16 byte planes, fill). It decodes every target and compares it byte for byte. With `VCODEC_DUMP=1` it writes the descriptor (`*.runs.npz`). +- `device_rebuild.py`: rebuilds a target on the device from a resident sibling. `replay` is out of place. `inplace` uses a ring of saved windows, KEEP for unmoved runs, absorb below a measured threshold, folding, and batched copies on NPU-B. +- `npu_d2d_bench.py`: device copy and upload costs per platform. +- `rbln_vmshim.cpp`: C shim over `rbln_memcpy_v2v_multi` / `rbln_memcpy_h2v_multi` of the NPU-B runtime API. diff --git a/tools/varmorph/classify.py b/tools/varmorph/classify.py new file mode 100644 index 0000000..cd6db16 --- /dev/null +++ b/tools/varmorph/classify.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Cohort 2026-11-12 (11): what a compiled binary holds, by content class, without parsing its format. + +Every 64 KiB window of a file is classified from byte statistics alone, so the same rule applies to +any vendor's artifact: + fill one or two distinct byte values (padding, zero-initialized buffers) + text >= 85 % printable ASCII (metadata, names, JSON) + fp16/bf16 16-bit float weights: >= 90 % of 16-bit words have a high byte in 0x04-0x3F or + 0x84-0xBF (|x| < 2 as fp16 or bf16), the two signs are balanced within 10 %, and the + high byte has <= 5.5 bits of entropy. The median magnitude byte tells the two apart + (bf16 weights sit at 0x3A-0x3D, fp16 weights below 0x38). + fp32 the same test on the top byte of 32-bit words (fp32 tensors: norms, tables, biases); + every byte phase is tried, since tensors start at odd offsets in some containers + bf16_in_fp32 fp32 words whose low 16 bits are zero in >= 95 % of words: bf16 values widened + code byte entropy < 7.0 bits (instructions, tables, descriptors; structured, compressible) + dense the rest (high entropy, not weight-like: packed or quantized data, compressed sections) +Per class: bytes, mean byte entropy, mean entropy of the high and low byte of 16-bit words. + +Optional join with a coding result: MODES.npy holds, per 64 KiB window of the target, the number of +64-byte blocks coded as match / fill / residual / intra (from vcodec_server.py). With it, the +output also gives how each class of the target was found in its sibling. +argv: OUT.json FILE [FILE ...] [--modes FILE=MODES.npy ...] +""" +import json +import os +import sys +import time + +import numpy as np + +WIN = 1 << 16 +CLASSES = ["fill", "text", "bf16", "fp32", "code", "dense", "fp16", "bf16_in_fp32"] + + +def entropy_rows(counts): + tot = counts.sum(axis=1, keepdims=True).astype(np.float64) + p = counts / np.maximum(tot, 1) + with np.errstate(divide="ignore", invalid="ignore"): + e = -(np.where(p > 0, p * np.log2(p), 0.0)).sum(axis=1) + return e + + +def hist_rows(a, nw): + """Per-row 256-bin histogram of a (nw, m) uint8 array.""" + off = (np.arange(nw, dtype=np.int64) * 256)[:, None] + return np.bincount((a.astype(np.int64) + off).ravel(), minlength=nw * 256).reshape(nw, 256) + + +def classify(path): + n = os.path.getsize(path) // WIN + mm = np.memmap(path, dtype=np.uint8, mode="r", shape=(n * WIN,)) + cls = np.empty(n, dtype=np.uint8) + feats = np.empty((n, 3), dtype=np.float32) + step = (1 << 28) // WIN + wl_hi = np.zeros(256, dtype=bool) + wl_hi[0x04:0x40] = True + wl_hi[0x84:0xC0] = True + printable = np.zeros(256, dtype=bool) + printable[0x20:0x7F] = True + printable[[9, 10, 13]] = True + for i in range(0, n, step): + j = min(n, i + step) + a = np.asarray(mm[i * WIN:j * WIN]).reshape(j - i, WIN) + nw = j - i + hb = hist_rows(a, nw) + h_odd = hist_rows(a[:, 1::2], nw) # high byte of 16-bit words at phase 0 + h_even = hist_rows(a[:, 0::2], nw) # high byte of 16-bit words at phase 1 + e = entropy_rows(hb) + distinct = (hb > 0).sum(axis=1) + asc = (hb[:, printable].sum(axis=1)) / WIN + + def test16(hh): + w = hh[:, wl_hi].sum(axis=1) / (WIN // 2) + sym = 1 - np.abs(hh[:, :128].sum(axis=1) - hh[:, 128:].sum(axis=1)) / (WIN // 2) + mag = hh[:, :128] + hh[:, 128:] + med = (np.cumsum(mag, axis=1) >= (WIN // 4)).argmax(axis=1) + return (w >= 0.9) & (sym >= 0.9) & (entropy_rows(hh) <= 5.5), med, entropy_rows(hh) + ok0, med0, e0 = test16(h_odd) + ok1, med1, e1 = test16(h_even) + is16 = ok0 | ok1 + med = np.where(ok0, med0, med1) + ehi = np.where(ok1 & ~ok0, e1, e0) + elo = np.where(ok1 & ~ok0, e0, e1) + is32 = np.zeros(nw, dtype=bool) + zlow = np.zeros(nw) + for p in range(4): + h32 = hist_rows(a[:, p::4], nw) + w32 = h32[:, wl_hi].sum(axis=1) / (WIN // 4) + sym32 = 1 - np.abs(h32[:, :128].sum(axis=1) - h32[:, 128:].sum(axis=1)) / (WIN // 4) + ok = (w32 >= 0.9) & (sym32 >= 0.9) + if ok.any(): + lo = (hist_rows(a[:, (p - 3) % 4::4], nw)[:, 0] + hist_rows(a[:, (p - 2) % 4::4], nw)[:, 0]) / (WIN // 2) + zlow = np.where(ok & ~is32, lo, zlow) + is32 |= ok + c = np.full(nw, 5, dtype=np.uint8) + c[e < 7.0] = 4 + c[is32 & ~is16] = 3 + c[is32 & ~is16 & (zlow >= 0.95)] = 7 # fp32 holding bf16 values (low half zero) + c[is16 & (med >= 0x38)] = 2 + c[is16 & (med < 0x38)] = 6 + c[asc >= 0.85] = 1 + c[distinct <= 2] = 0 + cls[i:j] = c + feats[i:j, 0] = e + feats[i:j, 1] = ehi + feats[i:j, 2] = elo + return cls, feats + + +def summary(path, cls, feats, modes=None): + out = {"file": path, "bytes": os.path.getsize(path), "windows": int(len(cls)), "classes": {}} + for k, name in enumerate(CLASSES): + sel = cls == k + if not sel.any(): + continue + row = {"bytes": int(sel.sum()) * WIN, "share": float(sel.mean()), + "entropy": float(feats[sel, 0].mean()), "entropy_hi": float(feats[sel, 1].mean()), + "entropy_lo": float(feats[sel, 2].mean())} + if modes is not None: + m = modes[: len(cls)][sel[: len(modes)]].sum(axis=0).astype(np.int64) * 64 + row["coded_as"] = {"match": int(m[0]), "fill": int(m[1]), "residual": int(m[2]), "intra": int(m[3])} + out["classes"][name] = row + # contiguous extents of one class, to show the layout + brk = np.nonzero(np.diff(cls.astype(np.int16)))[0] + 1 + st = np.concatenate([[0], brk]) + en = np.concatenate([brk, [len(cls)]]) + ext = [{"class": CLASSES[cls[s]], "offset": int(s) * WIN, "bytes": int(e - s) * WIN} for s, e in zip(st, en)] + big = sorted(ext, key=lambda x: -x["bytes"])[:12] + out["extents"] = int(len(ext)) + out["largest_extents"] = sorted(big, key=lambda x: x["offset"]) + return out + + +if __name__ == "__main__": + args = sys.argv[1:] + out = args[0] + files, modes = [], {} + k = 1 + while k < len(args): + if args[k] == "--modes": + f, m = args[k + 1].split("=", 1) + modes[f] = m + k += 2 + else: + files.append(args[k]) + k += 1 + res = [] + for f in files: + t = time.time() + cls, feats = classify(f) + md = np.load(modes[f]) if f in modes else None + res.append(summary(f, cls, feats, md)) + res[-1]["seconds"] = round(time.time() - t, 1) + with open(out, "w") as fh: + json.dump(res, fh, indent=1) + print(time.strftime("%H:%M:%S"), f, {c: round(v["share"], 3) for c, v in res[-1]["classes"].items()}, flush=True) + print("ALL-DONE", flush=True) diff --git a/tools/varmorph/device_rebuild.py b/tools/varmorph/device_rebuild.py new file mode 100644 index 0000000..435542e --- /dev/null +++ b/tools/varmorph/device_rebuild.py @@ -0,0 +1,522 @@ +#!/usr/bin/env python3 +"""Rebuild a target variant on the NPU from a resident sibling, with the coder's descriptor. + +The reference (sibling) bytes are uploaded once and left on the device: they stand for weights +that are already resident. The timed part rebuilds the target on the device: + 1. folded copies: match runs of equal length whose target and reference offsets advance by + constant strides are one strided copy (as_strided). The rest are single-run copies. + 2. patch: the 64-byte blocks that match nothing (residual, new content, padding) are uploaded + as one contiguous buffer and put in place with one index_copy_. +The baseline is an upload of the whole target from host memory, in the same 1 GiB pieces. +The result is read back and its SHA-256 compared with the target file. + +Device memory is a list of 1 GiB tensors (int16); a copy never crosses a piece, and copies are +cut at MAXCOPY bytes. Offsets must be even (int16 elements); odd runs are reported and fall +back to the patch. +argv: analyze RUNS.npz OUT.json (folding statistics only, no device) + inplace RUNS.npz OUT.json PLATFORM (in-place rebuild in the sibling's memory) + replay RUNS.npz OUT.json PLATFORM(rngd|rbln) (device replay) +""" +import hashlib +import json +import os +import sys +import time + +import numpy as np + +PIECE = 1 << 30 # bytes per device tensor +MAXCOPY = 256 << 20 # bytes per copy call +BLK = 64 + + +def split_runs(dst, src, ln): + """Cut runs at piece boundaries of both sides and at MAXCOPY.""" + out_d, out_s, out_l = [], [], [] + for d, s, l in zip(dst.tolist(), src.tolist(), ln.tolist()): + while l > 0: + take = min(l, PIECE - d % PIECE, PIECE - s % PIECE, MAXCOPY) + out_d.append(d) + out_s.append(s) + out_l.append(take) + d += take + s += take + l -= take + return np.array(out_d, np.int64), np.array(out_s, np.int64), np.array(out_l, np.int64) + + +GAP = int(os.environ.get("REPLAY_GAP", str(1 << 20))) # bytes of target a merged copy may cover + + +# Measured cost of one copy call and of host upload per platform (notes/npu-d2d-20261112): +# rngd-a3: copy_ per call 120 us, upload 3.7 GB/s; rbln-b3: v2v_multi 1.6 us per copy, upload 21.6 GB/s. +COSTS = {"rngd": dict(call_s=120e-6, h2d_Bps=3.7e9), "rbln": dict(call_s=1.6e-6, h2d_Bps=21.6e9), + "rbln-unbatched": dict(call_s=5.0e-6, h2d_Bps=21.6e9), "cpu": dict(call_s=1e-6, h2d_Bps=10e9)} + + +def absorb_threshold(platform): + """A run shorter than this is cheaper to upload in the patch than to copy with its own call: + the bytes the host link moves in the time of one call.""" + c = COSTS.get(platform, COSTS["cpu"]) + return int(c["call_s"] * c["h2d_Bps"]) // BLK * BLK + + +ABSORB = int(os.environ.get("REPLAY_ABSORB", str(64 << 10))) # bytes: shorter off-motion runs go to the patch +LOOK = int(os.environ.get("REPLAY_LOOK", str(1 << 20))) # bytes of target to look ahead for the motion + + +def absorb(dst, src, ln, limit=ABSORB, look=LOOK): + """Rate-distortion step: a short run whose motion differs from the surrounding segment is not + copied on its own; the segment's copy runs over it and its blocks join the patch. Returns the + kept runs and the absorbed byte ranges (target offset, length).""" + if limit <= 0 or len(dst) == 0: + return dst, src, ln, [] + o = np.argsort(dst, kind="stable") + dst, src, ln = dst[o].tolist(), src[o].tolist(), ln[o].tolist() + mv = [s_ - d for d, s_ in zip(dst, src)] + keep, gone = [], [] + i, n = 0, len(dst) + while i < n: + keep.append(i) + cur = mv[i] + j = i + 1 + while j < n: + if mv[j] == cur: + keep.append(j) + j += 1 + continue + # skip short off-motion runs if the segment's motion resumes within LOOK bytes + k, skipped = j, 0 + while k < n and mv[k] != cur and ln[k] <= limit and dst[k] - dst[j] <= look: + skipped += 1 + k += 1 + if k < n and mv[k] == cur and skipped: + gone.extend(range(j, k)) + j = k + else: + break + i = j + ka = np.array(keep, dtype=np.int64) + d, s_, l = np.array(dst)[ka], np.array(src)[ka], np.array(ln)[ka] + return d, s_, l, [(dst[g], ln[g]) for g in gone] + + +def coalesce(dst, src, ln, gap=GAP): + """Merge runs with the same motion (src - dst) whose target gap is <= gap: one copy covers the + gap, and the patch overwrites the gap blocks afterwards (skip + residual).""" + if gap <= 0 or len(dst) == 0: + return dst, src, ln + o = np.argsort(dst, kind="stable") + dst, src, ln = dst[o], src[o], ln[o] + out = [[dst[0], src[0], ln[0]]] + for d, s_, l in zip(dst[1:].tolist(), src[1:].tolist(), ln[1:].tolist()): + pd, ps, pl = out[-1] + if s_ - d == ps - pd and 0 <= d - (pd + pl) <= gap: + out[-1][2] = d + l - pd + else: + out.append([d, s_, l]) + a = np.array(out, dtype=np.int64) + return a[:, 0], a[:, 1], a[:, 2] + + +def fold(dst, src, ln): + """Greedy: consecutive runs (by target offset) with equal length and constant strides, inside + one piece on each side, form a group (d0, s0, len, count, dd, ds).""" + order = np.argsort(dst, kind="stable") + dst, src, ln = dst[order], src[order], ln[order] + groups = [] + i, n = 0, len(dst) + while i < n: + j = i + 1 + if j < n and ln[j] == ln[i] and dst[j] // PIECE == dst[i] // PIECE and src[j] // PIECE == src[i] // PIECE: + dd, ds = dst[j] - dst[i], src[j] - src[i] + if dd >= ln[i] and ds > 0: + while (j + 1 < n and ln[j + 1] == ln[i] and dst[j + 1] - dst[j] == dd and src[j + 1] - src[j] == ds + and dst[j + 1] // PIECE == dst[i] // PIECE and src[j + 1] // PIECE == src[i] // PIECE + and (j + 1 - i + 1) * ln[i] <= MAXCOPY): + j += 1 + j += 1 + else: + dd = ds = 0 + j = i + 1 + else: + dd = ds = 0 + groups.append((int(dst[i]), int(src[i]), int(ln[i]), j - i, int(dd), int(ds))) + i = j + return groups + + +def load(runs_path): + r = np.load(runs_path) + dst, src, ln = r["dst"].astype(np.int64), r["src"].astype(np.int64), r["len"].astype(np.int64) + odd = (dst % 2 != 0) | (src % 2 != 0) + return r, dst[~odd], src[~odd], ln[~odd], dst[odd], ln[odd] + + +def analyze(runs_path, out): + r, dst, src, ln, odd_d, odd_l = load(runs_path) + dst, src, ln, gone = absorb(dst, src, ln) + dst, src, ln = coalesce(dst, src, ln) + d2, s2, l2 = split_runs(dst, src, ln) + g = fold(d2, s2, l2) + cnt = np.array([x[3] for x in g]) + res = dict(gap=GAP, absorb=ABSORB, absorbed_runs=len(gone), absorbed_bytes=int(sum(x[1] for x in gone)), + coalesced_runs=int(len(dst)), runs=int(len(r["dst"])), odd_runs=int(len(odd_d)), odd_bytes=int(odd_l.sum()), + split_runs=int(len(d2)), groups=len(g), single_run_groups=int((cnt == 1).sum()), + bytes_in_multi_run_groups=int(sum(x[2] * x[3] for x in g if x[3] > 1)), + match_bytes=int(l2.sum()), other_blocks=int(len(r["other"])), v_bytes=int(r["v_bytes"])) + with open(out, "w") as f: + json.dump(res, f, indent=1) + print(json.dumps(res)) + return r, g, odd_d, odd_l, gone + + +def replay(runs_path, out, platform): + r, g, odd_d, odd_l, gone = analyze(runs_path, out + ".fold.json") + import torch + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + from npu_d2d_bench import device + dev, sync, pk = device(platform) + ref_path, tgt_path = str(r["refs"][0]), str(r["target"]) + vb = int(r["v_bytes"]) + res = dict(schema="npu-replay.v1", platform=platform, device=dev, ref=ref_path, target=tgt_path, v_bytes=vb, + groups=len(g), load=(os.getloadavg() if hasattr(os, "getloadavg") else None)) + + def pieces_of(nbytes): + return [min(PIECE, nbytes - o) for o in range(0, nbytes, PIECE)] + + def host_bytes(path, nbytes): + a = np.zeros(nbytes + (nbytes % 2), dtype=np.uint8) + with open(path, "rb") as f: + f.readinto(memoryview(a)[:os.path.getsize(path)]) + return a + + ref = host_bytes(ref_path, os.path.getsize(ref_path)) + tgt = host_bytes(tgt_path, vb) + # resident reference (untimed upload) + t = time.perf_counter() + S = [torch.from_numpy(ref[o:o + n].view(np.int16).copy()).to(dev) for o, n in zip(range(0, len(ref), PIECE), pieces_of(len(ref)))] + sync() + res["ref_upload_s"] = time.perf_counter() - t + al = (vb // BLK) * BLK # the device holds the block-aligned part; the tail (< 64 B) stays on the host + D = [torch.empty(n // 2, dtype=torch.int16, device=dev) for n in pieces_of(al)] + # baseline: upload the whole target + host_t = [torch.from_numpy(tgt[o:o + n].view(np.int16).copy()) for o, n in zip(range(0, al, PIECE), pieces_of(al))] + sync() + t = time.perf_counter() + for d, h in zip(D, host_t): + d.copy_(h) + sync() + res["baseline_upload_s"] = time.perf_counter() - t + # patch: every block not covered by an even match run + cover = np.zeros(vb // BLK + 1, dtype=bool) + for d0, s0, l, c, dd, ds in g: + for k in range(c): + a = (d0 + k * dd) // BLK + cover[a:a + l // BLK] = True + need = ~cover[: vb // BLK] + need[r["other"][r["other"] < vb // BLK]] = True + for d0, l in gone: # absorbed runs: the segment's copy wrote over them + need[d0 // BLK:(d0 + l) // BLK] = True + patch_blocks = np.nonzero(need)[0] + pb = tgt[: (vb // BLK) * BLK].reshape(-1, BLK)[patch_blocks].copy() + by_piece = {} + for i, b in enumerate(patch_blocks.tolist()): + by_piece.setdefault((b * BLK) // PIECE, []).append(i) + res["patch_blocks"] = int(len(patch_blocks)) + res["patch_bytes"] = int(len(patch_blocks) * BLK) + tail = tgt[(vb // BLK) * BLK:vb] + # clear the target so the replay cannot pass by accident + for d in D: + d.zero_() + sync() + path = {"strided": 0, "single": 0, "fallback": 0} + t0 = time.perf_counter() + for d0, s0, l, c, dd, ds in g: + dp, do = divmod(d0, PIECE) + sp, so = divmod(s0, PIECE) + le = l // 2 + if c == 1: + D[dp][do // 2:do // 2 + le].copy_(S[sp][so // 2:so // 2 + le]) + path["single"] += 1 + continue + try: + D[dp].as_strided((c, le), (dd // 2, 1), do // 2).copy_(S[sp].as_strided((c, le), (ds // 2, 1), so // 2)) + path["strided"] += 1 + except Exception: + for k in range(c): + a, b = (do + k * dd) // 2, (so + k * ds) // 2 + D[dp][a:a + le].copy_(S[sp][b:b + le]) + path["fallback"] += 1 + sync() + t1 = time.perf_counter() + for p, rows in by_piece.items(): + sel = np.array(rows) + local = (patch_blocks[sel] * BLK - p * PIECE) // BLK + buf = torch.from_numpy(pb[sel].view(np.int16)).to(dev) + idx = torch.from_numpy(local.astype(np.int64)).to(dev) + D[p].view(-1, BLK // 2).index_copy_(0, idx, buf) + sync() + t2 = time.perf_counter() + res.update(copy_s=t1 - t0, patch_s=t2 - t1, rebuild_s=t2 - t0, paths=path) + # verify + h = hashlib.sha256() + got = np.concatenate([d.cpu().numpy().view(np.uint8) for d in D])[: (vb // BLK) * BLK] + h.update(got.tobytes()) + h.update(tail.tobytes()) + want = hashlib.sha256(tgt[:vb].tobytes()).hexdigest() + res.update(sha256=h.hexdigest(), target_sha256=want, correct=h.hexdigest() == want) + res["mismatch_bytes"] = int((got != tgt[: (vb // BLK) * BLK]).sum()) if not res["correct"] else 0 + with open(out, "w") as f: + json.dump(res, f, indent=1) + print(json.dumps({k: v for k, v in res.items() if k != "load"})) + + +WIN = int(os.environ.get("REPLAY_WIN", str(64 << 20))) # bytes of target written per window +RING = int(os.environ.get("REPLAY_RING", str(256 << 20))) # bytes of original content kept behind the window + + +def replay_inplace(runs_path, out, platform): + """Rebuild the target in the memory that holds the sibling, with O(RING) scratch. + + One buffer holds the sibling's bytes and is overwritten with the target window by window, in + target order. Before a window is written, its original bytes are saved in a ring of + RING/WIN + 1 slots. A copy reads a source byte from: + - the live buffer if it lies after the current window (not yet overwritten); + - the ring if it lies in the RING bytes before the window, or in the window itself; + - a far scratch, filled before the first write, for the few sources further back. + Invariant: no original byte is overwritten before it is saved, and every copy that reads the + live buffer reads bytes after the current window. The patch is applied last.""" + import torch + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + from npu_d2d_bench import device + r, dst, src, ln, odd_d, odd_l = load(runs_path) + lim = ABSORB if "REPLAY_ABSORB" in os.environ else absorb_threshold(platform) + dst, src, ln, gone = absorb(dst, src, ln, limit=lim) + dst, src, ln = coalesce(dst, src, ln) + dev, sync, pk = device(platform) + ref_path, tgt_path = str(r["refs"][0]), str(r["target"]) + vb = int(r["v_bytes"]) + al = (vb // BLK) * BLK + ref = np.fromfile(ref_path, dtype=np.uint8) + tgt = np.fromfile(tgt_path, dtype=np.uint8) + total = max(len(ref), al) + total += (-total) % WIN + nslots = RING // WIN + 1 + pieces = [] + for d, s_, l in zip(dst.tolist(), src.tolist(), ln.tolist()): + while l > 0: + take = min(l, WIN - d % WIN) + pieces.append((d, s_, take)) + d, s_, l = d + take, s_ + take, l - take + pieces.sort() + kept = [x for x in pieces if x[0] == x[1]] # KEEP: already in place, no copy + pieces_cp = [x for x in pieces if x[0] != x[1]] + segs = [] + far = [] + far_bytes = 0 + for d, s_, l in pieces_cp: + w = d // WIN + a = w * WIN + ring_lo = max(a - (nslots - 1) * WIN, 0) + x, dd = s_, d + while x < s_ + l: + if x >= a + WIN: + take = min(s_ + l - x, PIECE - x % PIECE) + segs.append((w, dd, "live", x // PIECE, x % PIECE, take)) + elif x >= ring_lo: + take = min(s_ + l - x, WIN - x % WIN, a + WIN - x) + segs.append((w, dd, "ring", (x // WIN) % nslots, x % WIN, take)) + else: + take = min(s_ + l - x, ring_lo - x, PIECE - (far_bytes % PIECE)) + segs.append((w, dd, "far", far_bytes // PIECE, far_bytes % PIECE, take)) + far.append((x, take)) + far_bytes += take + x += take + dd += take + res = dict(schema="npu-replay-inplace.v1", platform=platform, device=dev, ref=ref_path, target=tgt_path, + v_bytes=vb, win=WIN, ring=RING, slots=nslots, copies=len(segs), far_bytes=far_bytes, + absorb_limit=lim, absorbed_bytes=int(sum(x[1] for x in gone)), + keep_bytes=int(sum(x[2] for x in kept)), keep_runs=len(kept), + far_copies=len(far), scratch_bytes=nslots * WIN + far_bytes, buffer_bytes=total, + load=(os.getloadavg() if hasattr(os, "getloadavg") else None)) + host = np.zeros(total, dtype=np.uint8) + host[:len(ref)] = ref + Bf = [torch.from_numpy(host[o:o + PIECE].view(np.int16).copy()).to(dev) for o in range(0, total, PIECE)] + ringt = [torch.empty(WIN // 2, dtype=torch.int16, device=dev) for _ in range(nslots)] + fart = [torch.empty(min(PIECE, far_bytes - o) // 2, dtype=torch.int16, device=dev) for o in range(0, far_bytes, PIECE)] + sync() + del host + # NPU-B: batched copies through the public runtime API (rbln_memcpy_v2v_multi / h2v_multi); + # copies inside one call are unordered, so a call never mixes a window's ring save with its + # copies, and no copy in a call writes where another reads + shim = None + if platform == "rbln" and os.environ.get("RBLN_VMSHIM") and os.environ.get("REPLAY_BATCH", "1") != "0": + import ctypes + shim = ctypes.CDLL(os.environ["RBLN_VMSHIM"]) + for t in ringt + fart: + shim.vmshim_layout_like(ctypes.c_uint64(t.data_ptr()), ctypes.c_uint64(Bf[0].data_ptr())) + if platform == "cpu" and os.environ.get("REPLAY_FAKE_BATCH"): # test the batched addressing on the host + import ctypes + + class _Fake: + @staticmethod + def _run(a_, b_, c_, n): + for i in range(n): + ctypes.memmove(b_[i], a_[i], c_[i]) + return 0 + vmshim_v2v_multi = _run + vmshim_h2v_multi = _run + shim = _Fake() + MAXB = 1024 + batch_calls = [0] + + def flush(ops, h2v=False): + import ctypes + for i in range(0, len(ops), MAXB): + part = ops[i:i + MAXB] + U = ctypes.c_uint64 * len(part) + a_, b_, c_ = U(*[x[0] for x in part]), U(*[x[1] for x in part]), U(*[x[2] for x in part]) + rc = (shim.vmshim_h2v_multi if h2v else shim.vmshim_v2v_multi)(a_, b_, c_, len(part)) + if rc != 0: + raise RuntimeError("batched copy rc=%d" % rc) + batch_calls[0] += 1 + need = np.ones(al // BLK, dtype=bool) + for d, s_, l in pieces: + need[d // BLK:(d + l) // BLK] = False + need[r["other"][r["other"] < al // BLK]] = True + for d0, l in gone: + need[d0 // BLK:(d0 + l) // BLK] = True + patch_blocks = np.nonzero(need)[0] + pb = tgt[:al].reshape(-1, BLK)[patch_blocks].copy() + res.update(patch_blocks=int(len(patch_blocks)), patch_bytes=int(len(patch_blocks) * BLK)) + + def E(b): + return b // 2 + + def view(origin, key, off, l): + t = {"live": Bf, "ring": ringt, "far": fart}[origin][key] + return t[E(off):E(off) + E(l)] + by_w = {} + for sg in segs: + by_w.setdefault(sg[0], []).append(sg) + # fold, per window: copies from one source tensor with equal length and constant target and + # source strides become one strided copy (as_strided on both sides) + plan = {} + nfold = 0 + for w, lst in by_w.items(): + lst.sort(key=lambda x: x[1]) + out_w, i = [], 0 + while i < len(lst): + _, d0, o0, k0, f0, l0 = lst[i] + j = i + 1 + if j < len(lst) and lst[j][2] == o0 and lst[j][3] == k0 and lst[j][5] == l0: + dd, ds = lst[j][1] - d0, lst[j][4] - f0 + if dd >= l0 and ds > 0 and d0 // PIECE == lst[j][1] // PIECE: + while (j + 1 < len(lst) and lst[j + 1][2] == o0 and lst[j + 1][3] == k0 and lst[j + 1][5] == l0 + and lst[j + 1][1] - lst[j][1] == dd and lst[j + 1][4] - lst[j][4] == ds + and lst[j + 1][1] // PIECE == d0 // PIECE): + j += 1 + j += 1 + else: + j = i + 1 + out_w.append((d0, o0, k0, f0, l0, j - i, (lst[i + 1][1] - d0) if j - i > 1 else 0, + (lst[i + 1][4] - f0) if j - i > 1 else 0)) + nfold += 1 + i = j + plan[w] = out_w + res["calls"] = nfold + path = {"strided": 0, "single": 0, "fallback": 0} + t0 = time.perf_counter() + fo = 0 + if shim is not None: + ops = [] + for x, l in far: + ops.append((Bf[x // PIECE].data_ptr() + x % PIECE, fart[fo // PIECE].data_ptr() + fo % PIECE, l)) + fo += l + flush(ops) + last_w = max(plan) if plan else -1 + for w in range(0, last_w + 1): + a = w * WIN + flush([(Bf[a // PIECE].data_ptr() + a % PIECE, ringt[w % nslots].data_ptr(), WIN)]) + ops = [] + for d, origin, key, off, l, c, dd, ds in plan.get(w, []): + st = {"live": Bf, "ring": ringt, "far": fart}[origin][key] + for k in range(c): + ops.append((st.data_ptr() + off + k * ds, Bf[d // PIECE].data_ptr() + d % PIECE + k * dd, l)) + flush(ops) + path["batched"] = batch_calls[0] + plan = {} + for x, l in far if shim is None else []: + fart[fo // PIECE][E(fo % PIECE):E(fo % PIECE) + E(l)].copy_(Bf[x // PIECE][E(x % PIECE):E(x % PIECE) + E(l)]) + fo += l + last_w = max(plan) if plan else -1 + for w in range(0, last_w + 1): + a = w * WIN + ringt[w % nslots].copy_(Bf[a // PIECE][E(a % PIECE):E(a % PIECE) + E(WIN)]) + for d, origin, key, off, l, c, dd, ds in plan.get(w, []): + dt = Bf[d // PIECE] + st = {"live": Bf, "ring": ringt, "far": fart}[origin][key] + if c == 1: + dt[E(d % PIECE):E(d % PIECE) + E(l)].copy_(st[E(off):E(off) + E(l)]) + path["single"] += 1 + continue + try: + dt.as_strided((c, E(l)), (E(dd), 1), E(d % PIECE)).copy_(st.as_strided((c, E(l)), (E(ds), 1), E(off))) + path["strided"] += 1 + except Exception: + for k in range(c): + a2, b2 = E(d % PIECE + k * dd), E(off + k * ds) + dt[a2:a2 + E(l)].copy_(st[b2:b2 + E(l)]) + path["fallback"] += 1 + sync() + res["paths"] = path + t1 = time.perf_counter() + if shim is not None and len(patch_blocks): + brk = np.nonzero(np.diff(patch_blocks) != 1)[0] + 1 + starts = np.concatenate([[0], brk]) + ends = np.concatenate([brk, [len(patch_blocks)]]) + ops = [] + base = pb.ctypes.data + for a_, b_ in zip(starts.tolist(), ends.tolist()): + off = int(patch_blocks[a_]) * BLK + n_ = (b_ - a_) * BLK + while n_ > 0: # do not cross a device piece + take = min(n_, PIECE - off % PIECE) + ops.append((base + a_ * BLK, Bf[off // PIECE].data_ptr() + off % PIECE, take)) + a_ += take // BLK + off += take + n_ -= take + res["patch_ranges"] = len(ops) + flush(ops, h2v=True) + path["batched"] = batch_calls[0] + else: + by_piece = {} + for i, b in enumerate(patch_blocks.tolist()): + by_piece.setdefault((b * BLK) // PIECE, []).append(i) + for p_, rows in by_piece.items(): + sel = np.array(rows) + local = (patch_blocks[sel] * BLK - p_ * PIECE) // BLK + Bf[p_].view(-1, BLK // 2).index_copy_(0, torch.from_numpy(local.astype(np.int64)).to(dev), + torch.from_numpy(pb[sel].view(np.int16)).to(dev)) + sync() + t2 = time.perf_counter() + got = np.concatenate([b.cpu().numpy().view(np.uint8) for b in Bf])[:al] + want = tgt[:al] + ok = bool(np.array_equal(got, want)) + res.update(copy_s=t1 - t0, patch_s=t2 - t1, rebuild_s=t2 - t0, correct=ok, + mismatch_bytes=0 if ok else int((got != want).sum()), + sha256=hashlib.sha256(got.tobytes() + tgt[al:vb].tobytes()).hexdigest(), + target_sha256=hashlib.sha256(tgt[:vb].tobytes()).hexdigest()) + with open(out, "w") as f: + json.dump(res, f, indent=1) + print(json.dumps({k: v for k, v in res.items() if k != "load"})) + + +if __name__ == "__main__": + if sys.argv[1] == "analyze": + analyze(sys.argv[2], sys.argv[3]) + elif sys.argv[1] == "inplace": + replay_inplace(sys.argv[2], sys.argv[3], sys.argv[4]) + else: + replay(sys.argv[2], sys.argv[3], sys.argv[4]) diff --git a/tools/varmorph/layout_coder.py b/tools/varmorph/layout_coder.py new file mode 100644 index 0000000..e87de20 --- /dev/null +++ b/tools/varmorph/layout_coder.py @@ -0,0 +1,548 @@ +#!/usr/bin/env python3 +"""Cohort 2026-11-12 (10): a video-coding-style lossless coder for a compiled variant against siblings. + +A target variant v is coded against reference variants (the siblings a node may hold), the way a +P-frame is coded against reference frames. Tools taken from video coding, and why: + +Matching +- block matching: v is split into 64-byte blocks, and each block is searched in the references by + exact hash, verified byte by byte. Exact matching replaces a SAD search, since the result must be + lossless. +- sub-block motion (the "sub-pel" step): content moves by offsets that are not multiples of 64 + bytes (a first, aligned-only run matched 211 KB of a prefill pair that content-defined chunks + share in GB). A rolling hash over every byte position of the references finds a sample of target + blocks. The phases (offset mod 64) of those shifts are then indexed, so the main search stays one + lookup per block. +- MV prediction: the block at the predicted position (the previous match plus the distance since) + is compared first. A hit is a "skip": no lookup and no new vector. +- variable blocks: consecutive blocks whose references are consecutive merge into one run. + +Coding +- MVD + entropy: each run is coded as its length and the difference of its motion vector from the + previous one (zigzag varints), then entropy-coded (zstd). +- inter residual: a block that matches nothing exactly but differs from its predicted block in at + most T bytes is coded as the XOR with that block. +- intra: any other block is raw content. Its bytes are coded with zstd, alone and after splitting + 2-byte words into byte planes (bf16). +- fill (DC): a block of one repeated 8-byte word is coded as that word. +- multi-reference: several references share one index; a vector names its reference through the + global offset. + +Not taken: transform and quantization (lossy) and loop filters. + +Every coded target is decoded and compared with v byte for byte before its size is reported. The +decode is timed: a single-threaded rebuild into host memory from references already in memory. +argv: OUT.json T REF[,REF...] TARGET [REF[,REF...] TARGET ...] +""" +import json +import os +import shutil +import subprocess +import sys +import time + +import numpy as np + +try: + from numba import njit +except ImportError: # slow fallback for small tests + def njit(*a, **k): + if a and callable(a[0]): + return a[0] + return lambda f: f + +B = 64 +W = B // 8 +K = (np.random.default_rng(12).integers(1, 2**63, size=W, dtype=np.uint64) | np.uint64(1)) +OP_MATCH, OP_FILL, OP_RES, OP_INTRA = 0, 1, 2, 3 +BASE = np.uint64(1099511628211) +SAMPLES = 1 << 16 +NMERGE = int(os.environ.get("VCODEC_NMERGE", "4")) # merge candidates (0: off) +NOMV = -(1 << 62) +BITS = 26 + + +def log(*a): + print(time.strftime("%H:%M:%S"), *a, flush=True) + + +def block_hashes(buf): + n = len(buf) // B + h = np.empty(n, dtype=np.uint64) + step = (1 << 28) // B + for i in range(0, n, step): + j = min(n, i + step) + w = np.frombuffer(buf[i * B:j * B].tobytes(), dtype=np.uint64).reshape(-1, W) if buf.ctypes.data % 8 else \ + buf[i * B:j * B].view(np.uint64).reshape(-1, W) + h[i:j] = ((w ^ (w >> np.uint64(29))) * K).sum(axis=1, dtype=np.uint64) + return h + + +@njit(cache=True) +def is_fill(V, o): + for k in range(8, B): + if V[o + k] != V[o + (k & 7)]: + return False + return True + + +@njit(cache=True) +def rk(V, o): + h = np.uint64(0) + for k in range(B): + h = h * BASE + np.uint64(V[o + k]) + return h + + +@njit(cache=True) +def find_shifts(R, V, sample_off, cap): + """Rolling hash over every byte of R; returns (ref_off, tgt_off) of verified sample hits.""" + m = len(sample_off) + sh = np.empty(m, dtype=np.uint64) + for s in range(m): + sh[s] = rk(V, sample_off[s]) + order = np.argsort(sh) + ss = sh[order] + so = sample_off[order] + bitmap = np.zeros(1 << (BITS - 3), dtype=np.uint8) + MIX = np.uint64(0x9E3779B97F4A7C15) + for s in range(m): + b = (ss[s] * MIX) >> np.uint64(64 - BITS) + bitmap[b >> np.uint64(3)] |= np.uint8(1) << np.uint8(b & np.uint64(7)) + pw = np.uint64(1) + for k in range(B): + pw = pw * BASE + out_r = np.empty(cap, dtype=np.int64) + out_t = np.empty(cap, dtype=np.int64) + nout = 0 + n = len(R) + if n < B: + return out_r[:0], out_t[:0] + h = rk(R, 0) + pos = 0 + while True: + b = (h * MIX) >> np.uint64(64 - BITS) + if bitmap[b >> np.uint64(3)] & (np.uint8(1) << np.uint8(b & np.uint64(7))): + lo, hi = 0, m + while lo < hi: + mid = (lo + hi) >> 1 + if ss[mid] < h: + lo = mid + 1 + else: + hi = mid + c = lo + while c < m and ss[c] == h and nout < cap: + t = so[c] + eq = True + for k in range(B): + if R[pos + k] != V[t + k]: + eq = False + break + if eq: + out_r[nout] = pos + out_t[nout] = t + nout += 1 + c += 1 + if pos + B >= n: + break + h = h * BASE + np.uint64(R[pos + B]) - np.uint64(R[pos]) * pw + pos += 1 + return out_r[:nout], out_t[:nout] + + +@njit(cache=True) +def scan(R, V, sr, spos, hv, T): + """Choose a mode per target block. ref holds a byte offset in R (or -1).""" + n = len(hv) + nr = len(sr) + nR = len(R) + mode = np.empty(n, dtype=np.uint8) + ref = np.full(n, -1, dtype=np.int64) + last_i = -1 + last_r = -1 + lookups = 0 + skips = 0 + merges = 0 + mc = np.full(NMERGE, NOMV, dtype=np.int64) + for i in range(n): + o = i * B + if is_fill(V, o): + mode[i] = OP_FILL + continue + p = -1 + if last_r >= 0: + p = last_r + (i - last_i) * B + if p + B > nR: + p = -1 + nz = B + 1 + if p >= 0: + nz = 0 + for k in range(B): + if R[p + k] != V[o + k]: + nz += 1 + if nz == 0: + mode[i] = OP_MATCH + ref[i] = p + last_i = i + last_r = p + skips += 1 + continue + # merge candidates (as in HEVC merge mode): the motions of the last few runs, tried + # before any hash lookup so that a block found elsewhere does not break the motion field + mfound = -1 + for c_ in range(NMERGE): + if mc[c_] == NOMV: + continue + q = o + mc[c_] + if q < 0 or q + B > nR or q == p: + continue + eq = True + for k in range(B): + if R[q + k] != V[o + k]: + eq = False + break + if eq: + mfound = q + break + if mfound >= 0: + mode[i] = OP_MATCH + ref[i] = mfound + last_i = i + last_r = mfound + merges += 1 + continue + lookups += 1 + lo, hi = 0, nr + x = hv[i] + while lo < hi: + mid = (lo + hi) >> 1 + if sr[mid] < x: + lo = mid + 1 + else: + hi = mid + found = -1 + c = lo + while c < nr and c < lo + 8 and sr[c] == x: + q = spos[c] + eq = True + for k in range(B): + if R[q + k] != V[o + k]: + eq = False + break + if eq: + found = q + break + c += 1 + if found >= 0: + mode[i] = OP_MATCH + ref[i] = found + last_i = i + last_r = found + d_ = found - o + if NMERGE > 0 and mc[0] != d_: + for c_ in range(NMERGE - 1, 0, -1): + mc[c_] = mc[c_ - 1] + mc[0] = d_ + continue + if p >= 0 and nz <= T: + mode[i] = OP_RES + ref[i] = p + last_i = i + last_r = p + continue + mode[i] = OP_INTRA + return mode, ref, lookups, skips, merges + + +@njit(cache=True) +def put_varint(buf, pos, x): + while x >= 0x80: + buf[pos] = (x & 0x7F) | 0x80 + x >>= 7 + pos += 1 + buf[pos] = x + return pos + 1 + + +@njit(cache=True) +def build(R, V, mode, ref): + """Merge blocks into runs; write the op stream, the residual stream and the intra stream.""" + n = len(mode) + ops = np.empty(n * 12 + 64, dtype=np.uint8) + res = np.empty(np.sum(mode == OP_RES) * B + 1, dtype=np.uint8) + intra = np.empty(np.sum(mode == OP_INTRA) * B + 1, dtype=np.uint8) + op = 0 + rp = 0 + ip = 0 + prev_d = 0 + runs = np.zeros(4, dtype=np.int64) + i = 0 + while i < n: + m = mode[i] + j = i + 1 + if m == OP_MATCH or m == OP_RES: + while j < n and mode[j] == m and ref[j] == ref[j - 1] + B: + j += 1 + elif m == OP_FILL: + while j < n and mode[j] == OP_FILL: + eq = True + for k in range(8): + if V[j * B + k] != V[i * B + k]: + eq = False + break + if not eq: + break + j += 1 + else: + while j < n and mode[j] == OP_INTRA: + j += 1 + L = j - i + runs[m] += 1 + ops[op] = m + op += 1 + op = put_varint(ops, op, L) + if m == OP_MATCH or m == OP_RES: + d = ref[i] - i * B + dd = d - prev_d + op = put_varint(ops, op, (dd << 1) ^ (dd >> 63)) + prev_d = d + if m == OP_RES: + for b in range(i, j): + for k in range(B): + res[rp] = R[ref[b] + k] ^ V[b * B + k] + rp += 1 + elif m == OP_FILL: + for k in range(8): + ops[op] = V[i * B + k] + op += 1 + else: + intra[ip:ip + L * B] = V[i * B:j * B] + ip += L * B + i = j + return ops[:op], res[:rp], intra[:ip], runs + + +@njit(cache=True) +def decode(R, ops, res, intra, n, tail): + out = np.empty(n * B + len(tail), dtype=np.uint8) + op = 0 + rp = 0 + ip = 0 + prev_d = 0 + i = 0 + while op < len(ops): + m = ops[op] + op += 1 + L = 0 + s = 0 + while True: + b = np.int64(ops[op]) + op += 1 + L |= (b & 0x7F) << s + s += 7 + if b < 0x80: + break + a = i * B + e = a + L * B + if m == OP_MATCH or m == OP_RES: + z = 0 + s = 0 + while True: + b = np.int64(ops[op]) + op += 1 + z |= (b & 0x7F) << s + s += 7 + if b < 0x80: + break + d = prev_d + ((z >> 1) ^ -(z & 1)) + prev_d = d + out[a:e] = R[a + d:e + d] + if m == OP_RES: + for k in range(L * B): + out[a + k] ^= res[rp + k] + rp += L * B + elif m == OP_FILL: + for k in range(a, e): + out[k] = ops[op + ((k - a) & 7)] + op += 8 + else: + out[a:e] = intra[ip:ip + L * B] + ip += L * B + i += L + out[n * B:] = tail + return out + + +def zsize(arr, level=19): + if len(arr) == 0: + return 0 + if shutil.which("zstd"): + lv = ["--ultra", "-22"] if level > 19 else ["-%d" % level] + p = subprocess.run(["zstd", "-q", "-c", "-T16", "--long=27"] + lv, input=arr.tobytes(), capture_output=True, check=True) + return len(p.stdout) + try: + import zstandard + return len(zstandard.ZstdCompressor(level=min(level, 19), threads=16).compress(arr.tobytes())) + except ImportError: + import lzma + return len(lzma.compress(arr.tobytes(), preset=6)) + + +def intra_size(arr): + """Best of zstd on the stream and zstd on its two byte planes (bf16 split).""" + plain = zsize(arr, 9) + if len(arr) < 2: + return plain, "zstd" + a = arr[: len(arr) // 2 * 2] + planes = zsize(np.ascontiguousarray(a[0::2]), 9) + zsize(np.ascontiguousarray(a[1::2]), 9) + (len(arr) - len(a)) + return (planes, "planes") if planes < plain else (plain, "zstd") + + +def load(paths): + sizes = [os.path.getsize(p) for p in paths] + pads = [(-s) % B for s in sizes] + R = np.empty(sum(sizes) + sum(pads), dtype=np.uint8) + o = 0 + for p, s, pd in zip(paths, sizes, pads): + with open(p, "rb") as f: + f.readinto(memoryview(R)[o:o + s]) + R[o + s:o + s + pd] = 0 + o += s + pd + return R + + +def run_stats(mode, ref, nontrivial_bytes): + idx = np.nonzero(mode == OP_MATCH)[0] + if len(idx) == 0: + return {} + r = ref[idx] + brk = np.ones(len(idx), dtype=bool) + brk[1:] = (idx[1:] != idx[:-1] + 1) | (r[1:] != r[:-1] + B) + st = np.nonzero(brk)[0] + run_bytes = np.diff(np.append(st, len(idx))) * B + d = r[st] - idx[st] * B + cov = {str(L): float(run_bytes[run_bytes >= L].sum() / nontrivial_bytes) + for L in [64, 256, 1024, 4096, 16384, 65536, 1 << 20]} + ud, inv = np.unique(d, return_inverse=True) + w = np.bincount(inv, weights=run_bytes) + o = np.argsort(-w) + cw = np.cumsum(w[o]) / run_bytes.sum() + jd = np.diff(d) + out = {"match_runs": int(len(st)), "median_run_bytes": float(np.median(run_bytes)), + "coverage_by_run_ge": cov, "distinct_deltas": int(len(ud)), + "deltas_for_90pct": int(np.searchsorted(cw, 0.9) + 1), + "top_deltas": [{"delta": int(ud[k]), "share": float(w[k] / run_bytes.sum())} for k in o[:10]]} + if len(jd): + uj, invj = np.unique(jd, return_inverse=True) + wj = np.bincount(invj, weights=run_bytes[1:]) + oj = np.argsort(-wj) + cj = np.cumsum(wj[oj]) / max(run_bytes[1:].sum(), 1) + out.update({"distinct_jumps": int(len(uj)), "jumps_for_90pct": int(np.searchsorted(cj, 0.9) + 1), + "top_jumps": [{"jump": int(uj[k]), "share": float(wj[k] / run_bytes[1:].sum())} for k in oj[:10]]}) + return out + + +def code_one(refs, target, T, modes_path=None): + t0 = time.time() + R = load(refs) + V = np.fromfile(target, dtype=np.uint8) + nb = len(V) // B + tail = V[nb * B:].copy() + Vb = V[: nb * B] + # sub-block motion: phases of the shifts found for a sample of target blocks + cand = np.linspace(0, nb - 1, SAMPLES * 2).astype(np.int64) + cand = np.array([c * B for c in cand if not is_fill(Vb, c * B)][:SAMPLES], dtype=np.int64) + hr_, ht_ = find_shifts(R, Vb, cand, 1 << 22) + shift = hr_ - ht_ + ph = np.bincount(((shift % B) + B) % B, minlength=B) + hit_blocks = len(np.unique(ht_)) + order_ph = np.argsort(-ph) + phases = [0] + acc = ph[0] + for p in order_ph: + if ph[p] == 0 or len(phases) >= 8 or acc >= 0.99 * max(ph.sum(), 1): + break + if p != 0: + phases.append(int(p)) + acc += ph[p] + us, cs = np.unique(shift, return_counts=True) + oc = np.argsort(-cs)[:10] + t1 = time.time() + log("phases", phases, "sample hits", int(hit_blocks), "of", len(cand)) + hs, ps = [], [] + for p in phases: + n = (len(R) - p) // B + hs.append(block_hashes(R[p:p + n * B])) + ps.append(p + np.arange(n, dtype=np.int64) * B) + hr = np.concatenate(hs) + pr = np.concatenate(ps) + del hs, ps + order = np.argsort(hr, kind="stable") + sr = hr[order] + spos = pr[order] + del hr, pr, order + hv = block_hashes(Vb) + t2 = time.time() + mode, ref, lookups, skips, merges = scan(R, Vb, sr, spos, hv, T) + ops, res, intra, runs = build(R, Vb, mode, ref) + t3 = time.time() + decode(R[:1 << 20], ops[:0], res, intra, 0, tail[:0]) # compile outside the timed call + t4 = time.time() + out = decode(R, ops, res, intra, nb, tail) + t5 = time.time() + ok = bool(np.array_equal(out, V)) + del out + log("decoded", ok, round(t5 - t4, 2), "s") + cnt = np.bincount(mode, minlength=4) * B + if modes_path and os.environ.get("VCODEC_DUMP"): + # the descriptor itself, for a device-side replay: match runs (target byte offset, + # reference byte offset, length) and the other blocks (target block index, mode) + idx = np.nonzero(mode == OP_MATCH)[0] + rr = ref[idx] + brk = np.ones(len(idx), dtype=bool) + brk[1:] = (idx[1:] != idx[:-1] + 1) | (rr[1:] != rr[:-1] + B) + st = np.nonzero(brk)[0] + ln = np.diff(np.append(st, len(idx))) * B + other = np.nonzero(mode != OP_MATCH)[0] + np.savez(modes_path.replace(".modes.npy", ".runs.npz"), dst=idx[st] * B, src=rr[st], len=ln, + other=other, other_mode=mode[other], v_bytes=len(V), refs=np.array(refs), target=np.array(target)) + if modes_path: # per 64 KiB window: blocks coded as match / fill / residual / intra + nw = nb // 1024 + wm = np.bincount((np.arange(nw * 1024) // 1024) * 4 + mode[:nw * 1024], minlength=nw * 4) + np.save(modes_path, wm.reshape(nw, 4).astype(np.uint32)) + s_ops = zsize(ops, 19) + s_res = zsize(res, 19) + s_intra, intra_how = intra_size(intra) + total = s_ops + s_res + s_intra + len(tail) + nontriv = int(len(V) - cnt[1]) + return { + "refs": refs, "target": target, "T": T, "v_bytes": int(len(V)), "lossless": ok, + "sample": {"blocks": int(len(cand)), "hit_blocks": int(hit_blocks), "hits": int(len(shift)), + "phase_hist_top": {int(p): int(ph[p]) for p in order_ph[:8]}, "phases_indexed": phases, + "top_shifts": [{"shift": int(us[k]), "count": int(cs[k])} for k in oc]}, + "bytes_by_mode": {"match": int(cnt[0]), "fill": int(cnt[1]), "residual": int(cnt[2]), "intra": int(cnt[3])}, + "runs_by_mode": {"match": int(runs[0]), "fill": int(runs[1]), "residual": int(runs[2]), "intra": int(runs[3])}, + "lookups": int(lookups), "skips": int(skips), "merges": int(merges), "nmerge": NMERGE, + "match_structure": run_stats(mode, ref, max(nontriv, 1)), + "coded": {"ops_raw": int(len(ops)), "ops": s_ops, "residual_raw": int(len(res)), "residual": s_res, + "intra_raw": int(len(intra)), "intra": s_intra, "intra_codec": intra_how, "total": total}, + "seconds": {"shift_search": round(t1 - t0, 1), "index": round(t2 - t1, 1), "encode": round(t3 - t2, 1), + "decode_host": round(t5 - t4, 2)}, + "decode_GBps": round(len(V) / max(t5 - t4, 1e-9) / 1e9, 2), + } + + +if __name__ == "__main__": + out, T = sys.argv[1], int(sys.argv[2]) + a = sys.argv[3:] + res = [] + for k in range(0, len(a), 2): + refs = a[k].split(",") + log("code", refs, "->", a[k + 1]) + res.append(code_one(refs, a[k + 1], T, "%s.%d.modes.npy" % (out, k // 2))) + with open(out, "w") as f: + json.dump(res, f, indent=1) + r = res[-1] + log("result", json.dumps(r["coded"]), json.dumps(r["bytes_by_mode"]), json.dumps(r["sample"]["phases_indexed"]), + "decode", r["seconds"]["decode_host"]) + log("ALL-DONE") diff --git a/tools/varmorph/npu_d2d_bench.py b/tools/varmorph/npu_d2d_bench.py new file mode 100644 index 0000000..4c19591 --- /dev/null +++ b/tools/varmorph/npu_d2d_bench.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +"""NPU micro-benchmark (NPU-A or NPU-B): can a target's weights be rebuilt on the device from +resident weights? + +A video-style descriptor turns a variant into copies of runs from a sibling. This measures what +such a decode costs on the card, against uploading the weights from the host. Cases: + h2d_ contiguous host -> device copy (the upload a restart pays) + d2d_ contiguous device -> device copy of one tensor into another + d2d_calls 512 device copies of one 32 KiB run each, one call per run + d2d_strided one call copying every other 32 KiB run of a 64 MiB region (a regular re-tiling + as one descriptor) + d2d_gather one call gathering the 32 KiB runs of a 64 MiB region in a random order + (index_select) + use_256 a device reduction over a 256 MiB tensor already on the device + d2duse_256 a device copy into that tensor, then the same reduction (timed together): if the + copy left the data on the host side, the reduction pays the upload + h2duse_256 a host upload into that tensor, then the reduction + +Every case runs in its own process, so a device error in one case does not leave the card in a bad +state for the next. Each case syncs the card around the timed region and repeats REPS times after +a warm-up. Values are finite bf16 (from a normal distribution), and results are compared as int16 +bit patterns: a fast path that changes the bytes is reported as wrong. +argv: OUT.json PLATFORM(rngd|rbln) [REPS, default 5] (driver) + --case NAME PLATFORM REPS OUT_CASE.json (one case, internal) +""" +import json +import os +import subprocess +import sys +import time +import traceback + +RUN = 16384 # bf16 elements per 32 KiB run +CASES = ["h2d_64", "h2d_1024", "d2d_1", "d2d_64", "d2d_256", "d2d_1024", + "d2d_calls", "d2d_strided", "d2d_gather", + "use_256", "d2duse_256", "h2duse_256"] +if os.environ.get("RBLN_VMSHIM"): # NPU-B batched copies through the public runtime API + CASES += ["v2vmulti_1", "v2vmulti_64", "v2vmulti_1024", "v2vmultis_1024", "h2vmulti_1024"] + + +def device(platform): + import torch + if platform == "cpu": # local test of the drivers + return "cpu", (lambda: None), ("torch",) + if platform == "rngd": + from rngd_pd import devices + from rngd_pd.engines.fsdk import dev_str, open_device, sync + chip = devices.resolve_chip(-1) + dev = dev_str(chip) + open_device(8, 2) + return dev, (lambda: sync(chip)), ("torch", "furiosa-torch") + import rebel # noqa: F401 + import torch_rbln # noqa: F401 registers the rbln device + dev = "rbln:%d" % int(os.environ.get("RBLN_BENCH_DEVICE", "0")) + return dev, (lambda: torch.rbln.synchronize(dev)), ("torch", "rebel-compiler") + + +def run_case(name, platform, reps, out): + import torch + from importlib.metadata import version + torch.set_num_threads(4) + row = dict(case=name, s=[]) + try: + dev, sync, pk = device(platform) + row.update(device=dev, packages={k: version(k) for k in pk}) + g = torch.Generator().manual_seed(7) + bits = lambda t: t.view(torch.int16) + + def host(nelem): + return torch.randn(nelem, generator=g, dtype=torch.float32).to(torch.bfloat16) + + kind, _, arg = name.partition("_") + if kind in ("v2vmulti", "v2vmultis", "h2vmulti"): + import ctypes + lib = ctypes.CDLL(os.environ["RBLN_VMSHIM"]) + k = int(arg) + rows = 2048 # 64 MiB region of 32 KiB runs + n = rows * RUN + h = host(n) + src = h.to(dev) + dst = torch.empty(n, dtype=torch.bfloat16, device=dev) + dst.copy_(src) # give dst a device view + sync() + if kind != "h2vmulti": + row["layout_like_rc"] = lib.vmshim_layout_like(ctypes.c_uint64(dst.data_ptr()), ctypes.c_uint64(src.data_ptr())) + perm = torch.randperm(rows, generator=g)[:k].tolist() + U64 = ctypes.c_uint64 * k + sz = U64(*([RUN * 2] * k)) + dsta = U64(*[dst.data_ptr() + i * RUN * 2 * (2 if kind == "v2vmultis" else 1) for i in range(k)]) + if kind == "h2vmulti": + hb = h.view(rows, RUN) + srca = U64(*[hb[r].data_ptr() for r in perm]) + fn = lambda: lib.vmshim_h2v_multi(srca, dsta, sz, k) + else: + srca = U64(*[src.data_ptr() + r * RUN * 2 for r in perm]) + fn = lambda: lib.vmshim_v2v_multi(srca, dsta, sz, k) + nbytes = k * RUN * 2 + row["calls"] = 1 + step = 2 if kind == "v2vmultis" else 1 + check = lambda: all(torch.equal(bits(dst[(i * step) * RUN:(i * step + 1) * RUN].cpu()), bits(h[perm[i] * RUN:(perm[i] + 1) * RUN])) + for i in sorted({0, k // 2, k - 1})) + rc = fn() + row["rc"] = rc + elif kind in ("use", "d2duse", "h2duse"): + n = int(arg) * (1 << 20) // 2 + h = host(n) + src = h.to(dev) + dst = torch.empty(n, dtype=torch.bfloat16, device=dev) + dst.copy_(src) + sync() + red = lambda: dst.sum() + if kind == "use": + fn = lambda: red() + elif kind == "d2duse": + fn = lambda: (dst.copy_(src), red()) + else: + fn = lambda: (dst.copy_(h), red()) + nbytes = n * 2 + check = lambda: torch.equal(bits(dst[-RUN:].cpu()), bits(h[-RUN:])) + elif kind == "h2d": + n = int(arg) * (1 << 20) // 2 + h = host(n) + dst = torch.empty(n, dtype=torch.bfloat16, device=dev) + fn = lambda: dst.copy_(h) + nbytes = n * 2 + check = lambda: torch.equal(bits(dst[-RUN:].cpu()), bits(h[-RUN:])) + else: + n = (int(arg) if arg.isdigit() else 64) * (1 << 20) // 2 + n = max(n // RUN, 2) * RUN + rows = n // RUN + h = host(n) + src = h.to(dev) + dst = torch.empty(n, dtype=torch.bfloat16, device=dev) + sync() + if arg.isdigit(): + fn = lambda: dst.copy_(src) + nbytes = n * 2 + check = lambda: torch.equal(bits(dst[-RUN:].cpu()), bits(h[-RUN:])) + elif arg == "calls": + k = min(512, rows) + perm = torch.randperm(rows, generator=g)[:k].tolist() + + def fn(): + for i, r in enumerate(perm): + dst[i * RUN:(i + 1) * RUN].copy_(src[r * RUN:(r + 1) * RUN]) + nbytes = k * RUN * 2 + row["calls"] = k + check = lambda: torch.equal(bits(dst[5 * RUN:6 * RUN].cpu()), bits(h[perm[5] * RUN:(perm[5] + 1) * RUN])) + elif arg == "strided": + s2, d2 = src.view(rows // 2, 2, RUN), dst.view(rows // 2, 2, RUN) + fn = lambda: d2[:, 0, :].copy_(s2[:, 1, :]) + nbytes = (rows // 2) * RUN * 2 + check = lambda: torch.equal(bits(d2[:, 0, :].cpu()), bits(h.view(rows // 2, 2, RUN)[:, 1, :])) + else: # gather + pidx = torch.randperm(rows, generator=g) + pdev = pidx.to(dev) + sv, dv = src.view(rows, RUN), dst.view(rows, RUN) + fn = lambda: dv.copy_(torch.index_select(sv, 0, pdev)) + nbytes = n * 2 + check = lambda: torch.equal(bits(dv[7].cpu()), bits(h.view(rows, RUN)[pidx[7]])) + row["bytes"] = nbytes + fn() + sync() + for _ in range(reps): + sync() + t = time.perf_counter() + fn() + sync() + row["s"].append(time.perf_counter() - t) + row["median_s"] = sorted(row["s"])[len(row["s"]) // 2] + row["GBps"] = nbytes / row["median_s"] / 1e9 + if "calls" in row: + row["us_per_call"] = row["median_s"] / row["calls"] * 1e6 + row["correct"] = bool(check()) + except Exception as e: + row["error"] = repr(e)[:400] + row["trace"] = traceback.format_exc()[-1500:] + with open(out, "w") as f: + json.dump(row, f, indent=1) + + +def driver(out, platform, reps): + res = dict(schema="npu-d2d.v2", platform=platform, reps=reps, load=os.getloadavg(), cases={}) + for name in CASES: + part = out + "." + name + ".json" + t = time.time() + try: + p = subprocess.run([sys.executable, __file__, "--case", name, platform, str(reps), part], + capture_output=True, text=True, timeout=600) + row = json.load(open(part)) if os.path.exists(part) else dict(error="no output", rc=p.returncode) + if p.returncode: + row["rc"] = p.returncode + row["stderr"] = p.stderr[-800:] + except subprocess.TimeoutExpired: + row = dict(error="timeout 600 s") + row["wall_s"] = round(time.time() - t, 1) + res["cases"][name] = row + with open(out, "w") as f: + json.dump(res, f, indent=1) + print(name, {k: v for k, v in row.items() if k not in ("s", "trace", "stderr")}, flush=True) + res["finished"] = time.time() + with open(out, "w") as f: + json.dump(res, f, indent=1) + + +if __name__ == "__main__": + if sys.argv[1] == "--case": + run_case(sys.argv[2], sys.argv[3], int(sys.argv[4]), sys.argv[5]) + else: + driver(sys.argv[1], sys.argv[2], int(sys.argv[3]) if len(sys.argv) > 3 else 5) diff --git a/tools/varmorph/rbln_vmshim.cpp b/tools/varmorph/rbln_vmshim.cpp new file mode 100644 index 0000000..547599c --- /dev/null +++ b/tools/varmorph/rbln_vmshim.cpp @@ -0,0 +1,38 @@ +// C shim over the public NPU-B runtime API (rebel/runtime/api/rbln_runtime_api.h, SDK 0.12.0) for +// batched copies, which torch-rbln does not bind to Python. Loaded with ctypes inside a process +// that has imported torch_rbln; tensor data pointers are vmem virtual addresses. +// g++ -O2 -shared -fPIC -I$SP/rebel/include rbln_vmshim.cpp -o librbln_vmshim.so \ +// -L$SP/tvm -l:librbln.so -Wl,-rpath,$SP/tvm +#include + +#include +#include +#include + +extern "C" { + +// n copies (src_vaddr[i] -> dst_vaddr[i], size[i]); on the device for <= 1024 per destination. +int vmshim_v2v_multi(const uint64_t* src, const uint64_t* dst, const uint64_t* size, int n) { + std::vector> v; + v.reserve(n); + for (int i = 0; i < n; ++i) v.emplace_back(src[i], dst[i], size[i]); + return static_cast(rbln::rbln_memcpy_v2v_multi(v)); +} + +// n patches (host pointer -> dst_vaddr, size). +int vmshim_h2v_multi(const uint64_t* host, const uint64_t* dst, const uint64_t* size, int n) { + std::vector> v; + v.reserve(n); + for (int i = 0; i < n; ++i) v.emplace_back(static_cast(host[i]), dst[i], size[i]); + return static_cast(rbln::rbln_memcpy_h2v_multi(v)); +} + +int vmshim_layout_like(uint64_t target, uint64_t ref) { + return static_cast(rbln::rbln_set_device_alloc_layout_like(target, ref)); +} + +int vmshim_sync(uint32_t torch_device_id) { + return static_cast(rbln::rbln_device_synchronize(torch_device_id)); +} + +} // extern "C" From ef1c2ea4773326f242049eeff8bcadc3eda8aee5 Mon Sep 17 00:00:00 2001 From: gspark-etri Date: Thu, 12 Nov 2026 17:05:31 +0900 Subject: [PATCH 16/16] casync backend: tests for repair of a changed base, a hard-linked base and a leased base under concurrent pulls; varmorph tools: descriptor file output, aligned-only switch, apply on host (new / in place), VCDIFF to copy list, reach analysis, ring saves only read windows, far-scratch budget Co-Authored-By: Claude Fable 5.2 --- pkg/casync/guards_test.go | 126 ++++++++++++++++++++++ tools/varmorph/device_rebuild.py | 49 ++++++++- tools/varmorph/host_restore.py | 157 ++++++++++++++++++++++++++++ tools/varmorph/layout_coder.py | 11 ++ tools/varmorph/reach.py | 24 +++++ tools/varmorph/vcdiff_to_runs.py | 153 +++++++++++++++++++++++++++ tools/varmorph/vm_apply.py | 173 +++++++++++++++++++++++++++++++ 7 files changed, 689 insertions(+), 4 deletions(-) create mode 100644 pkg/casync/guards_test.go create mode 100644 tools/varmorph/host_restore.py create mode 100644 tools/varmorph/reach.py create mode 100644 tools/varmorph/vcdiff_to_runs.py create mode 100644 tools/varmorph/vm_apply.py diff --git a/pkg/casync/guards_test.go b/pkg/casync/guards_test.go new file mode 100644 index 0000000..8e0f31e --- /dev/null +++ b/pkg/casync/guards_test.go @@ -0,0 +1,126 @@ +package casync + +import ( + "bytes" + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/ai-computing/onggi/pkg/csi" +) + +// TestKeepRepairsChangedBase: a base that was modified after the node wrote it no longer matches +// its record, so Keep reads the kept chunks back and replaces the ones that fail their digest. +func TestKeepRepairsChangedBase(t *testing.T) { + ctx := context.Background() + e, root := setup(t) + base, aligned, _ := variants() + mA, _ := e.Publish(ctx, publish(t, e, root, "a", base)) + mB, _ := e.Publish(ctx, publish(t, e, root, "b", aligned)) + m := &csi.Materializer{Root: filepath.Join(root, "node"), Pull: e.Pull, RebaseIdle: time.Nanosecond, MaxArtifacts: 1} + if err := m.Init(); err != nil { + t.Fatal(err) + } + dA, err := m.Ensure(ctx, mA) + if err != nil { + t.Fatal(err) + } + // damage a region that A and B share (B differs from A only around 3 MiB) + time.Sleep(20 * time.Millisecond) + f, err := os.OpenFile(filepath.Join(dA, "model.bin"), os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + if _, err := f.WriteAt(bytes.Repeat([]byte{0xEE}, 64<<10), 1<<20); err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + dB, err := m.Ensure(ctx, mB) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(readFile(t, filepath.Join(dB, "model.bin")), aligned) { + t.Fatal("Keep carried a changed base into the target") + } + if _, err := os.Stat(dA); !os.IsNotExist(err) { + t.Fatal("the idle victim was not consumed") + } +} + +// TestHardLinkedBaseIsNotOverwritten: a base whose file has a second name is never rebuilt in +// place, so the bytes behind the second name stay those of the base. +func TestHardLinkedBaseIsNotOverwritten(t *testing.T) { + ctx := context.Background() + e, root := setup(t) + base, aligned, _ := variants() + mA, _ := e.Publish(ctx, publish(t, e, root, "a", base)) + mB, _ := e.Publish(ctx, publish(t, e, root, "b", aligned)) + m := &csi.Materializer{Root: filepath.Join(root, "node"), Pull: e.Pull, RebaseIdle: time.Nanosecond, MaxArtifacts: 1} + if err := m.Init(); err != nil { + t.Fatal(err) + } + dA, err := m.Ensure(ctx, mA) + if err != nil { + t.Fatal(err) + } + snap := filepath.Join(m.Root, "snapshot-model.bin") + if err := os.Link(filepath.Join(dA, "model.bin"), snap); err != nil { + t.Skipf("hard links unavailable here: %v", err) + } + time.Sleep(20 * time.Millisecond) + dB, err := m.Ensure(ctx, mB) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(readFile(t, filepath.Join(dB, "model.bin")), aligned) { + t.Fatal("B is wrong") + } + if !bytes.Equal(readFile(t, snap), base) { + t.Fatal("a hard-linked base was rebuilt in place") + } +} + +// TestLeasedBaseSurvivesConcurrentPulls: while a volume holds A, several pulls of B run at once; +// A must stay intact and every pull must return the same correct B. +func TestLeasedBaseSurvivesConcurrentPulls(t *testing.T) { + ctx := context.Background() + e, root := setup(t) + base, aligned, _ := variants() + mA, _ := e.Publish(ctx, publish(t, e, root, "a", base)) + mB, _ := e.Publish(ctx, publish(t, e, root, "b", aligned)) + m := &csi.Materializer{Root: filepath.Join(root, "node"), Pull: e.Pull, RebaseIdle: time.Nanosecond, MaxArtifacts: 2} + if err := m.Init(); err != nil { + t.Fatal(err) + } + dA, err := m.Ensure(ctx, mA) + if err != nil { + t.Fatal(err) + } + if err := m.Acquire("vol-a", mA, "/pods/a"); err != nil { + t.Fatal(err) + } + errs := make(chan error, 4) + dirs := make(chan string, 4) + for i := 0; i < 4; i++ { + go func() { + d, err := m.Ensure(ctx, mB) + dirs <- d + errs <- err + }() + } + for i := 0; i < 4; i++ { + if err := <-errs; err != nil { + t.Fatal(err) + } + if d := <-dirs; !bytes.Equal(readFile(t, filepath.Join(d, "model.bin")), aligned) { + t.Fatal("a concurrent pull returned a wrong B") + } + } + if !bytes.Equal(readFile(t, filepath.Join(dA, "model.bin")), base) { + t.Fatal("a leased base changed under concurrent pulls") + } +} diff --git a/tools/varmorph/device_rebuild.py b/tools/varmorph/device_rebuild.py index 435542e..d2749cb 100644 --- a/tools/varmorph/device_rebuild.py +++ b/tools/varmorph/device_rebuild.py @@ -50,8 +50,18 @@ def split_runs(dst, src, ln): # Measured cost of one copy call and of host upload per platform (notes/npu-d2d-20261112): # rngd-a3: copy_ per call 120 us, upload 3.7 GB/s; rbln-b3: v2v_multi 1.6 us per copy, upload 21.6 GB/s. -COSTS = {"rngd": dict(call_s=120e-6, h2d_Bps=3.7e9), "rbln": dict(call_s=1.6e-6, h2d_Bps=21.6e9), - "rbln-unbatched": dict(call_s=5.0e-6, h2d_Bps=21.6e9), "cpu": dict(call_s=1e-6, h2d_Bps=10e9)} +COSTS = {"rngd": dict(call_s=120e-6, h2d_Bps=3.7e9, d2d_Bps=24.4e9), + "rbln": dict(call_s=1.6e-6, h2d_Bps=21.6e9, d2d_Bps=21.7e9), + "rbln-unbatched": dict(call_s=5.0e-6, h2d_Bps=21.6e9, d2d_Bps=21.7e9), + "cpu": dict(call_s=1e-6, h2d_Bps=10e9, d2d_Bps=10e9)} + + +def estimate(platform, n_copies, copy_bytes, patch_bytes, target_bytes): + """Predicted seconds of the rebuild and of a full upload, from the measured cost table.""" + c = COSTS.get(platform, COSTS["cpu"]) + rebuild = n_copies * c["call_s"] + copy_bytes / c["d2d_Bps"] + patch_bytes / c["h2d_Bps"] + upload = target_bytes / c["h2d_Bps"] + return rebuild, upload def absorb_threshold(platform): @@ -315,6 +325,7 @@ def replay_inplace(runs_path, out, platform): segs = [] far = [] far_bytes = 0 + need_ring = set() # windows whose original bytes a later copy reads: only these are saved for d, s_, l in pieces_cp: w = d // WIN a = w * WIN @@ -327,6 +338,7 @@ def replay_inplace(runs_path, out, platform): elif x >= ring_lo: take = min(s_ + l - x, WIN - x % WIN, a + WIN - x) segs.append((w, dd, "ring", (x // WIN) % nslots, x % WIN, take)) + need_ring.add(x // WIN) else: take = min(s_ + l - x, ring_lo - x, PIECE - (far_bytes % PIECE)) segs.append((w, dd, "far", far_bytes // PIECE, far_bytes % PIECE, take)) @@ -387,9 +399,24 @@ def flush(ops, h2v=False): need[r["other"][r["other"] < al // BLK]] = True for d0, l in gone: need[d0 // BLK:(d0 + l) // BLK] = True + # runs at odd offsets cannot be addressed in 16-bit device elements: their blocks are uploaded + for d0, l in zip(odd_d.tolist(), odd_l.tolist()): + need[d0 // BLK:(d0 + l + BLK - 1) // BLK] = True + res["odd_runs"] = int(len(odd_d)) + res["odd_bytes"] = int(odd_l.sum()) patch_blocks = np.nonzero(need)[0] pb = tgt[:al].reshape(-1, BLK)[patch_blocks].copy() res.update(patch_blocks=int(len(patch_blocks)), patch_bytes=int(len(patch_blocks) * BLK)) + nwin = (max(x[0] for x in pieces) // WIN + 1) if pieces else 0 + n_copies = len(segs) + len(far) + len(need_ring) + copy_bytes = sum(sg[5] for sg in segs) + sum(l for _, l in far) + len(need_ring) * WIN + res.update(windows=int(nwin), ring_saves=len(need_ring), relocate_bytes=int(sum(sg[5] for sg in segs))) + est_rebuild, est_upload = estimate(platform, n_copies, copy_bytes, res["patch_bytes"], al) + in_host = os.environ.get("REPLAY_TARGET_IN_HOST") == "1" + # the far scratch is known before the first write; beyond its budget the target is uploaded + far_budget = int(os.environ.get("REPLAY_FAR_BUDGET", str(1 << 30))) + res.update(est_rebuild_s=est_rebuild, est_upload_s=est_upload, target_in_host=in_host, far_budget=far_budget, + choice=("upload" if (in_host and est_upload < est_rebuild) or far_bytes > far_budget else "rebuild")) def E(b): return b // 2 @@ -438,7 +465,8 @@ def view(origin, key, off, l): last_w = max(plan) if plan else -1 for w in range(0, last_w + 1): a = w * WIN - flush([(Bf[a // PIECE].data_ptr() + a % PIECE, ringt[w % nslots].data_ptr(), WIN)]) + if w in need_ring: + flush([(Bf[a // PIECE].data_ptr() + a % PIECE, ringt[w % nslots].data_ptr(), WIN)]) ops = [] for d, origin, key, off, l, c, dd, ds in plan.get(w, []): st = {"live": Bf, "ring": ringt, "far": fart}[origin][key] @@ -453,7 +481,8 @@ def view(origin, key, off, l): last_w = max(plan) if plan else -1 for w in range(0, last_w + 1): a = w * WIN - ringt[w % nslots].copy_(Bf[a // PIECE][E(a % PIECE):E(a % PIECE) + E(WIN)]) + if w in need_ring: + ringt[w % nslots].copy_(Bf[a // PIECE][E(a % PIECE):E(a % PIECE) + E(WIN)]) for d, origin, key, off, l, c, dd, ds in plan.get(w, []): dt = Bf[d // PIECE] st = {"live": Bf, "ring": ringt, "far": fart}[origin][key] @@ -508,6 +537,18 @@ def view(origin, key, off, l): mismatch_bytes=0 if ok else int((got != want).sum()), sha256=hashlib.sha256(got.tobytes() + tgt[al:vb].tobytes()).hexdigest(), target_sha256=hashlib.sha256(tgt[:vb].tobytes()).hexdigest()) + del got + if res["target_in_host"]: + # the other path: upload the whole target from host memory into the same buffer + hostp = [torch.from_numpy(tgt[o:min(o + PIECE, al)].view(np.int16).copy()) for o in range(0, al, PIECE)] + sync() + t3 = time.perf_counter() + for dst_t, src_t in zip(Bf, hostp): + dst_t[:src_t.numel()].copy_(src_t) + sync() + res["upload_s"] = time.perf_counter() - t3 + res["upload_correct"] = bool(np.array_equal(np.concatenate([b.cpu().numpy().view(np.uint8) for b in Bf])[:al], want)) + res["measured_choice"] = "upload" if res["upload_s"] < res["rebuild_s"] else "rebuild" with open(out, "w") as f: json.dump(res, f, indent=1) print(json.dumps({k: v for k, v in res.items() if k != "load"})) diff --git a/tools/varmorph/host_restore.py b/tools/varmorph/host_restore.py new file mode 100644 index 0000000..8b78846 --- /dev/null +++ b/tools/varmorph/host_restore.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Restore a target file on the host from a sibling file and a copy list (runs.npz), timed. + +Modes: + new write the target into a new file: copy every run from the sibling, then the patch + blocks (blocks no run covers), then fsync. Same output contract as `xdelta3 -d`. + inplace turn a private copy of the sibling into the target in place, window by window in + target order, saving each window's original bytes in a ring first (as the device + rebuild does); runs whose move is zero are not touched; then the patch; then fsync. +The patch bytes are taken from the target file at the blocks no run covers (they stand for the +residual and new content that the descriptor carries). The result is compared with the target's +SHA-256. The copy of the sibling for `inplace` is made before the clock and is not timed. +argv: MODE RUNS.npz WORKDIR OUT.json +""" +import hashlib +import json +import os +import shutil +import sys +import time + +import numpy as np + +BLK = 64 +WIN = 64 << 20 +RING = 256 << 20 + + +def sha(path): + h = hashlib.sha256() + with open(path, "rb") as f: + while True: + b = f.read(64 << 20) + if not b: + break + h.update(b) + return h.hexdigest() + + +def main(): + mode, runs_path, work, out = sys.argv[1:5] + r = np.load(runs_path) + dst, src, ln = r["dst"].astype(np.int64), r["src"].astype(np.int64), r["len"].astype(np.int64) + ref, tgt = str(r["refs"][0]), str(r["target"]) + vb = int(r["v_bytes"]) + al = vb // BLK * BLK + T = np.memmap(tgt, dtype=np.uint8, mode="r") + need = np.ones(al // BLK, dtype=bool) + for d, l in zip(dst.tolist(), ln.tolist()): + need[d // BLK:(d + l) // BLK] = False + other = r["other"] + need[other[other < al // BLK]] = True + patch_blocks = np.nonzero(need)[0] + brk = np.nonzero(np.diff(patch_blocks) != 1)[0] + 1 + pr = list(zip(np.concatenate([[0], brk]).tolist(), np.concatenate([brk, [len(patch_blocks)]]).tolist())) + if "patch_data" in r.files: + # bytes carried by the descriptor (vm_apply.py expand): the target file is not read + pd, oth = r["patch_data"], r["other"] + pos = {int(b): i for i, b in enumerate(oth.tolist())} + assert all(int(b) in pos for b in patch_blocks.tolist()), "descriptor does not cover every block" + patch = [] + for a, b in pr: + i0 = pos[int(patch_blocks[a])] + patch.append((int(patch_blocks[a]) * BLK, pd[i0 * BLK:(i0 + (b - a)) * BLK])) + tail = r["tail"] + else: + patch = [(int(patch_blocks[a]) * BLK, np.array(T[int(patch_blocks[a]) * BLK:int(patch_blocks[b - 1]) * BLK + BLK])) + for a, b in pr] + tail = np.array(T[al:vb]) + os.makedirs(work, exist_ok=True) + path = os.path.join(work, os.path.basename(tgt) + "." + mode) + res = dict(mode=mode, runs=runs_path, ref=ref, target=tgt, v_bytes=vb, n_runs=int(len(dst)), + patch_bytes=int(len(patch_blocks) * BLK), patch_ranges=len(pr), + load=os.getloadavg()) + if mode == "inplace": + shutil.copyfile(ref, path) + os.truncate(path, max(vb, os.path.getsize(ref))) + with open(path, "rb") as f: + os.fsync(f.fileno()) + order = np.argsort(dst, kind="stable") + dst, src, ln = dst[order], src[order], ln[order] + t0 = time.perf_counter() + written = 0 + if mode == "new": + R = np.memmap(ref, dtype=np.uint8, mode="r") + O = np.memmap(path, dtype=np.uint8, mode="w+", shape=(vb,)) + for d, s_, l in zip(dst.tolist(), src.tolist(), ln.tolist()): + O[d:d + l] = R[s_:s_ + l] + for off, b in patch: + O[off:off + len(b)] = b + O[al:vb] = tail + written = vb + else: + O = np.memmap(path, dtype=np.uint8, mode="r+") + nslots = RING // WIN + 1 + ring = [None] * nslots + # far sources (before the ring) are saved before the first write + pieces = [] + for d, s_, l in zip(dst.tolist(), src.tolist(), ln.tolist()): + while l > 0: + take = min(l, WIN - d % WIN) + pieces.append((d, s_, take)) + d, s_, l = d + take, s_ + take, l - take + far = {} + for d, s_, l in pieces: + w = d // WIN + lo = max(w * WIN - (nslots - 1) * WIN, 0) + if s_ < lo and d != s_: + far[(s_, l)] = np.array(O[s_:s_ + l]) + by_w = {} + for p in pieces: + by_w.setdefault(p[0] // WIN, []).append(p) + last = (al - 1) // WIN + for w in range(last + 1): + a = w * WIN + lst = [p for p in by_w.get(w, []) if p[0] != p[1]] + ring[w % nslots] = (a, np.array(O[a:a + WIN])) if lst or True else None + for d, s_, l in lst: + if s_ >= a + WIN: + O[d:d + l] = O[s_:s_ + l] + elif (s_, l) in far: + O[d:d + l] = far[(s_, l)] + else: + x, dd, rem = s_, d, l + while rem > 0: + if x >= a + WIN: # not yet overwritten + O[dd:dd + rem] = O[x:x + rem] + break + sw = x // WIN + base, buf = ring[sw % nslots] + take = min(rem, base + WIN - x) + O[dd:dd + take] = buf[x - base:x - base + take] + x, dd, rem = x + take, dd + take, rem - take + written += l + for off, b in patch: + O[off:off + len(b)] = b + written += len(b) + O[al:vb] = tail + O.flush() + del O + with open(path, "rb+") as f: + if mode == "inplace": + f.truncate(vb) + os.fsync(f.fileno()) + t1 = time.perf_counter() + res.update(restore_s=t1 - t0, bytes_written=int(written)) + res["sha256"] = sha(path) + res["target_sha256"] = sha(tgt) + res["correct"] = res["sha256"] == res["target_sha256"] + os.remove(path) + with open(out, "w") as f: + json.dump(res, f, indent=1) + print(json.dumps({k: v for k, v in res.items() if k != "load"})) + + +if __name__ == "__main__": + main() diff --git a/tools/varmorph/layout_coder.py b/tools/varmorph/layout_coder.py index e87de20..2a7ce91 100644 --- a/tools/varmorph/layout_coder.py +++ b/tools/varmorph/layout_coder.py @@ -460,6 +460,8 @@ def code_one(refs, target, T, modes_path=None): phases = [0] acc = ph[0] for p in order_ph: + if os.environ.get("VCODEC_ALIGNED") == "1": # ablation: match only at block-aligned offsets + break if ph[p] == 0 or len(phases) >= 8 or acc >= 0.99 * max(ph.sum(), 1): break if p != 0: @@ -494,6 +496,14 @@ def code_one(refs, target, T, modes_path=None): del out log("decoded", ok, round(t5 - t4, 2), "s") cnt = np.bincount(mode, minlength=4) * B + shipped = None + if modes_path and os.environ.get("VCODEC_SAVE"): + # the descriptor as one file a node can fetch: the three streams and the tail, zstd -19 + dpath = modes_path.replace(".modes.npy", ".vmd.npz") + np.savez(dpath, ops=ops, res=res, intra=intra, tail=tail, nb=np.int64(nb), v_bytes=np.int64(len(V)), + refs=np.array(refs), target=np.array(target)) + subprocess.run(["zstd", "-q", "-f", "-19", "-T16", "--long=27", dpath, "-o", dpath + ".zst"], check=True) + shipped = os.path.getsize(dpath + ".zst") if modes_path and os.environ.get("VCODEC_DUMP"): # the descriptor itself, for a device-side replay: match runs (target byte offset, # reference byte offset, length) and the other blocks (target block index, mode) @@ -517,6 +527,7 @@ def code_one(refs, target, T, modes_path=None): nontriv = int(len(V) - cnt[1]) return { "refs": refs, "target": target, "T": T, "v_bytes": int(len(V)), "lossless": ok, + "shipped_bytes": shipped, "aligned_only": os.environ.get("VCODEC_ALIGNED") == "1", "sample": {"blocks": int(len(cand)), "hit_blocks": int(hit_blocks), "hits": int(len(shift)), "phase_hist_top": {int(p): int(ph[p]) for p in order_ph[:8]}, "phases_indexed": phases, "top_shifts": [{"shift": int(us[k]), "count": int(cs[k])} for k in oc]}, diff --git a/tools/varmorph/reach.py b/tools/varmorph/reach.py new file mode 100644 index 0000000..f577b73 --- /dev/null +++ b/tools/varmorph/reach.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +"""Backward reach of the moves of a copy list and the scratch an in-place rebuild needs. +argv: OUT.json RUNS.npz [RUNS.npz ...] (window 64 MiB, ring 256 MiB as in npu_replay.py)""" +import json, sys +import numpy as np +WIN, RING = 64 << 20, 256 << 20 +res = [] +for p in sys.argv[2:]: + r = np.load(p) + d, s, l = r["dst"].astype(np.int64), r["src"].astype(np.int64), r["len"].astype(np.int64) + mv = s - d + moved = mv != 0 + a = (d // WIN) * WIN # start of the window being written + back = np.where(moved, a - s, 0) # > 0: the source lies before the window (already overwritten) + ring_lo = np.maximum(a - RING, 0) + far = moved & (s < ring_lo) + far_bytes = int(np.minimum(l[far], ring_lo[far] - s[far]).sum()) + res.append(dict(runs=p, target=str(r["target"]), n_runs=int(len(d)), keep_bytes=int(l[~moved].sum()), + moved_bytes=int(l[moved].sum()), max_back_bytes=int(back.max()) if len(back) else 0, + back_p99_bytes=int(np.percentile(back[moved], 99)) if moved.any() else 0, + far_runs=int(far.sum()), far_bytes=far_bytes, scratch_bytes=RING + WIN + far_bytes, + v_bytes=int(r["v_bytes"]), ref_bytes=None)) +json.dump(res, open(sys.argv[1], "w"), indent=1) +for x in res: print(json.dumps(x)) diff --git a/tools/varmorph/vcdiff_to_runs.py b/tools/varmorph/vcdiff_to_runs.py new file mode 100644 index 0000000..f90481f --- /dev/null +++ b/tools/varmorph/vcdiff_to_runs.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Turn an xdelta3 (VCDIFF) patch into the copy list that the device rebuild reads (runs.npz). + +Reads `xdelta3 printdelta PATCH` on stdin. printdelta prints the target offset and S@a +(source offset) as absolute values, and T@a relative to the start of the current VCDIFF window +("window at offset"); a T@ copy repeats bytes already decoded in that window. A copy from the target (T@) is resolved to the source bytes it repeats, +piece by piece, so the result is generous to the delta: every byte a copy chain carries from the +source becomes a run. ADD and RUN bytes, and copies that resolve to them, have no source and go +to the patch. Runs are trimmed to whole 64-byte target blocks; blocks not fully covered form +`other`. The instruction counts of the patch as written are stored too. +argv: OUT.npz REF TARGET < printdelta.txt +""" +import bisect +import json +import os +import sys + +import numpy as np + +BLK = 64 + + +class Window: + def __init__(self, t0, src0): + self.t0, self.src0 = t0, src0 + self.st, self.en, self.kind, self.src = [], [], [], [] + + def emit(self, t, l, kind, s): + if l <= 0: + return + if self.st and self.en[-1] == t and self.kind[-1] == kind and \ + (kind == 0 or self.src[-1] + (self.en[-1] - self.st[-1]) == s): + self.en[-1] = t + l + return + self.st.append(t) + self.en.append(t + l) + self.kind.append(kind) + self.src.append(s) + + +def main(): + out, ref, target = sys.argv[1], sys.argv[2], sys.argv[3] + vb = os.path.getsize(target) + st = dict(windows=0, cpy_s=0, cpy_t=0, add=0, run=0, cpy_s_bytes=0, cpy_t_bytes=0, add_bytes=0, run_bytes=0) + rd, rs, rl = [], [], [] + w = None + next_t0, cw_off, tlen, win_at = 0, 0, 0, 0 + + def close(win): + if win is None: + return + for a, e, k, s in zip(win.st, win.en, win.kind, win.src): + if k == 1: + rd.append(win.t0 + a) + rs.append(win.src0 + s) + rl.append(e - a) + + for line in sys.stdin: + if line.startswith("VCDIFF window number"): + close(w) + w = None + st["windows"] += 1 + continue + if line.startswith("VCDIFF window at offset"): + win_at = int(line.split(":")[1]) + continue + if line.startswith("VCDIFF copy window offset"): + cw_off = int(line.split(":")[1]) + continue + if line.startswith("VCDIFF target window length"): + tlen = int(line.split(":")[1]) + w = Window(0, 0) # printdelta prints absolute target offsets and addresses + next_t0 += tlen + cw_off = 0 + continue + f = line.split() + if w is None or len(f) < 4 or not f[0].isdigit(): + continue + t = int(f[0]) + rest = f[2:] + while rest: + kind = rest[0] + if kind in ("ADD", "RUN"): + size, addr, rest = int(rest[1]), None, rest[2:] + st["add" if kind == "ADD" else "run"] += 1 + st[("add" if kind == "ADD" else "run") + "_bytes"] += size + w.emit(t, size, 0, 0) + else: + size, addr, rest = int(rest[1]), rest[2], rest[3:] + if addr.startswith("S@"): + st["cpy_s"] += 1 + st["cpy_s_bytes"] += size + w.emit(t, size, 1, int(addr[2:])) + else: + st["cpy_t"] += 1 + st["cpy_t_bytes"] += size + p, done = win_at + int(addr[2:]), 0 # T@ is relative to the window + while done < size: + i = bisect.bisect_right(w.st, p) - 1 + avail = min(w.en[i] - p, size - done, (t + done) - p) + if w.kind[i] == 1: + w.emit(t + done, avail, 1, w.src[i] + (p - w.st[i])) + else: + w.emit(t + done, avail, 0, 0) + p += avail + done += avail + t += size + close(w) + d = np.array(rd, np.int64) + s = np.array(rs, np.int64) + l = np.array(rl, np.int64) + st["source_runs_raw"] = int(len(d)) + st["source_bytes"] = int(l.sum()) + # merge across windows: consecutive runs with the same move + o = np.argsort(d, kind="stable") + d, s, l = d[o], s[o], l[o] + md, ms, ml = [], [], [] + for a, b, c in zip(d.tolist(), s.tolist(), l.tolist()): + if md and md[-1] + ml[-1] == a and ms[-1] + ml[-1] == b: + ml[-1] += c + else: + md.append(a), ms.append(b), ml.append(c) + d, s, l = np.array(md, np.int64), np.array(ms, np.int64), np.array(ml, np.int64) + st["source_runs_merged"] = int(len(d)) + # trim to whole target blocks + a = (d + BLK - 1) // BLK * BLK + e = (d + l) // BLK * BLK + keep = e > a + s = s + (a - d) + d, s, l = a[keep], s[keep], (e - a)[keep] + al = vb // BLK + covered = np.zeros(al, dtype=bool) + for a_, l_ in zip(d.tolist(), l.tolist()): + covered[a_ // BLK:(a_ + l_) // BLK] = True + other = np.nonzero(~covered)[0].astype(np.int64) + R = np.memmap(ref, dtype=np.uint8, mode="r") + T = np.memmap(target, dtype=np.uint8, mode="r") + bad = sum(1 for a_, b_, l_ in zip(d.tolist(), s.tolist(), l.tolist()) + if not np.array_equal(R[b_:b_ + l_], T[a_:a_ + l_])) + st["runs_mismatched"] = int(bad) + st["runs"] = int(len(d)) + st["run_bytes_trimmed"] = int(l.sum()) + st["other_blocks"] = int(len(other)) + st["v_bytes"] = vb + np.savez(out, dst=d, src=s, len=l, other=other, other_mode=np.zeros(len(other), np.uint8), + v_bytes=np.int64(vb), refs=np.array([ref]), target=np.array(target)) + with open(out + ".json", "w") as fh: + json.dump(st, fh, indent=1) + print(json.dumps(st)) + + +if __name__ == "__main__": + main() diff --git a/tools/varmorph/vm_apply.py b/tools/varmorph/vm_apply.py new file mode 100644 index 0000000..55c6294 --- /dev/null +++ b/tools/varmorph/vm_apply.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Apply a VarMorph descriptor file on a node: expand it, then restore the target file. + + expand DESC.vmd.npz[.zst] REF OUT.runs.npz + Read the descriptor (ops, residual, intra, tail) and the sibling REF and write the copy list + the restorers read: match runs (dst, src, len) and, for every other block, its bytes + (fill, residual XOR the sibling block it predicts from, new content). Only the sibling + blocks that residuals predict from are read. Timed: decompress + expand. + new|inplace DESC REF WORKDIR OUT.json [COLD] + expand, then restore the target into a new file / over a private copy of the sibling + (host_restore.py), sync, and compare with the SHA-256 recorded in the descriptor's name + file (the target itself, read after the clock). COLD=1 evicts the sibling from the page + cache first (posix_fadvise DONTNEED). +The copy of the sibling for `inplace` is made before the clock. +""" +import io +import json +import os +import subprocess +import sys +import time + +import numpy as np +from numba import njit + +B = 64 +OP_MATCH, OP_FILL, OP_RES, OP_INTRA = 0, 1, 2, 3 + + +@njit(cache=True) +def count(ops): + """Number of match runs, and of non-match blocks.""" + op, nm, nb_other = 0, 0, 0 + while op < len(ops): + m = ops[op] + op += 1 + L, s = 0, 0 + while True: + b = np.int64(ops[op]) + op += 1 + L |= (b & 0x7F) << s + s += 7 + if b < 0x80: + break + if m == OP_MATCH or m == OP_RES: + while True: + b = np.int64(ops[op]) + op += 1 + if b < 0x80: + break + if m == OP_MATCH: + nm += 1 + else: + nb_other += L + elif m == OP_FILL: + op += 8 + nb_other += L + else: + nb_other += L + return nm, nb_other + + +@njit(cache=True) +def expand(R, ops, res, intra, nm, nb_other): + dst = np.empty(nm, np.int64) + src = np.empty(nm, np.int64) + ln = np.empty(nm, np.int64) + other = np.empty(nb_other, np.int64) + data = np.empty(nb_other * B, np.uint8) + op, rp, ip, prev_d, i, km, ko = 0, 0, 0, 0, 0, 0, 0 + while op < len(ops): + m = ops[op] + op += 1 + L, s = 0, 0 + while True: + b = np.int64(ops[op]) + op += 1 + L |= (b & 0x7F) << s + s += 7 + if b < 0x80: + break + a = i * B + if m == OP_MATCH or m == OP_RES: + z, s = 0, 0 + while True: + b = np.int64(ops[op]) + op += 1 + z |= (b & 0x7F) << s + s += 7 + if b < 0x80: + break + d = prev_d + ((z >> 1) ^ -(z & 1)) + prev_d = d + if m == OP_MATCH: + dst[km] = a + src[km] = a + d + ln[km] = L * B + km += 1 + else: + for k in range(L * B): + data[ko * B + k] = R[a + d + k] ^ res[rp + k] + rp += L * B + for k in range(L): + other[ko + k] = i + k + ko += L + elif m == OP_FILL: + for k in range(L * B): + data[ko * B + k] = ops[op + (k & 7)] + op += 8 + for k in range(L): + other[ko + k] = i + k + ko += L + else: + data[ko * B:(ko + L) * B] = intra[ip:ip + L * B] + ip += L * B + for k in range(L): + other[ko + k] = i + k + ko += L + i += L + return dst, src, ln, other, data + + +def read_desc(path): + if path.endswith(".zst"): + raw = subprocess.run(["zstd", "-q", "-d", "-c", "--long=27", path], capture_output=True, check=True).stdout + return np.load(io.BytesIO(raw)) + return np.load(path) + + +def do_expand(desc, ref, out): + t0 = time.perf_counter() + d = read_desc(desc) + ops, res, intra, tail = d["ops"], d["res"], d["intra"], d["tail"] + t1 = time.perf_counter() + R = np.memmap(ref, dtype=np.uint8, mode="r") + nm, no = count(ops) + dst, src, ln, other, data = expand(R, ops, res, intra, nm, no) + t2 = time.perf_counter() + np.savez(out, dst=dst, src=src, len=ln, other=other, other_mode=np.zeros(len(other), np.uint8), + patch_data=data, tail=tail, v_bytes=d["v_bytes"], refs=np.array([ref]), target=d["target"]) + return dict(desc_bytes=os.path.getsize(desc), decompress_s=t1 - t0, expand_s=t2 - t1, runs=int(nm), + other_blocks=int(no), patch_bytes=int(no * B)) + + +if __name__ == "__main__": + mode = sys.argv[1] + here = os.path.dirname(os.path.abspath(__file__)) + if mode == "expand": + expand(np.zeros(64, np.uint8), np.zeros(0, np.uint8), np.zeros(0, np.uint8), np.zeros(0, np.uint8), 0, 0) + print(json.dumps(do_expand(sys.argv[2], sys.argv[3], sys.argv[4]))) + else: + desc, ref, work, out = sys.argv[2:6] + cold = len(sys.argv) > 6 and sys.argv[6] == "1" + os.makedirs(work, exist_ok=True) + expand(np.zeros(64, np.uint8), np.zeros(0, np.uint8), np.zeros(0, np.uint8), np.zeros(0, np.uint8), 0, 0) + count(np.zeros(0, np.uint8)) + runs = os.path.join(work, "apply.runs.npz") + if cold: + fd = os.open(ref, os.O_RDONLY) + os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED) + os.close(fd) + t0 = time.perf_counter() + e = do_expand(desc, ref, runs) + t1 = time.perf_counter() + p = subprocess.run([sys.executable, os.path.join(here, "host_restore.py"), mode, runs, work, out + ".restore"], + capture_output=True, text=True, check=True) + t2 = time.perf_counter() + r = json.load(open(out + ".restore")) + res = dict(mode=mode, cold=cold, expand=e, expand_total_s=t1 - t0, restore_s=r["restore_s"], + process_s=t2 - t1, apply_s=(t1 - t0) + r["restore_s"], correct=r["correct"], + bytes_written=r["bytes_written"], load=os.getloadavg()) + json.dump(res, open(out, "w"), indent=1) + print(json.dumps(res))