diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c220c3d..a2c3a1b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -15,7 +15,9 @@ jobs:
with:
go-version-file: go.mod
- name: gofmt
- run: test -z "$(gofmt -l .)" || { gofmt -l .; exit 1; }
+ run: test -z "$(gofmt -l .)" || { gofmt -d .; exit 1; }
+ - name: Owner admission and post-consumption recovery witnesses
+ run: go test -race -count=1 -timeout=2m -run '^TestOwner' -v ./pkg/csi
- run: go vet ./...
- run: go test -race ./...
- name: E11 payload accounting and validity regressions
diff --git a/cmd/onggi/casync.go b/cmd/onggi/casync.go
new file mode 100644
index 0000000..0ce5327
--- /dev/null
+++ b/cmd/onggi/casync.go
@@ -0,0 +1,184 @@
+package main
+
+import (
+ "context"
+ "encoding/json"
+ "flag"
+ "fmt"
+ "io"
+ "log"
+ "net/url"
+ "os"
+ "time"
+
+ "github.com/folbricht/desync"
+
+ "github.com/ai-computing/onggi/pkg/casync"
+ "github.com/ai-computing/onggi/pkg/csi"
+ "github.com/ai-computing/onggi/pkg/digest"
+ "github.com/ai-computing/onggi/pkg/store"
+)
+
+const casyncUsage = `onggi casync — artifacts in casync format (desync chunk store), with state-aware in-place reuse
+
+ onggi casync publish --store S --index I
chunk dir into S; print the manifest digest
+ onggi casync chunks --store S --index I print the chunk list of every file (JSON)
+ onggi casync pull --store S --index I [--attest A]
+ [--base M --base-dir D] take base M over (Keep): write only differing chunks; kept chunks are
+ read back unless A attests the base unchanged since it was written
+ [--copy M --copy-dir D] read base M where it is (Copy); it stays intact
+ [--verify-kept] [--no-sync] [--workers N] [--log]
+ onggi casync verify --store S --index I hash-check every chunk of dir
+ onggi casync attest --store S --index I --attest A verify dir, then record it as unchanged
+`
+
+func casyncCmd(args []string, stdout, stderr io.Writer) int {
+ if len(args) < 1 {
+ fmt.Fprint(stderr, casyncUsage)
+ return 2
+ }
+ fs := flag.NewFlagSet("casync "+args[0], flag.ContinueOnError)
+ fs.SetOutput(stderr)
+ storeDir := fs.String("store", "", "desync local chunk store")
+ indexDir := fs.String("index", "", "directory of artifact indexes")
+ attestDir := fs.String("attest", "", "directory of attestations (none: kept chunks are always read back)")
+ workers := fs.Int("workers", 12, "parallel chunk reads and writes")
+ base := fs.String("base", "", "Keep: manifest of the artifact to take over")
+ baseDir := fs.String("base-dir", "", "Keep: directory holding the base's files")
+ cp := fs.String("copy", "", "Copy: manifest of an artifact to read where it is")
+ copyDir := fs.String("copy-dir", "", "Copy: directory holding that artifact's files")
+ verifyKept := fs.Bool("verify-kept", false, "read back kept chunks even if the base is attested unchanged")
+ noSync := fs.Bool("no-sync", false, "do not sync written files (desync's default behaviour)")
+ logPhases := fs.Bool("log", false, "log phase durations to stderr")
+ remote := fs.String("remote", "", "pull: read chunks from this HTTP chunk server instead of --store")
+ if err := fs.Parse(args[1:]); err != nil || *storeDir == "" || *indexDir == "" {
+ fmt.Fprint(stderr, casyncUsage)
+ return 2
+ }
+ e, err := casync.New(*storeDir, *indexDir, *attestDir, *workers)
+ if err == nil && *remote != "" {
+ var u *url.URL
+ if u, err = url.Parse(*remote); err == nil {
+ e.Src, err = desync.NewRemoteHTTPStore(u, desync.StoreOptions{N: *workers, Timeout: 5 * time.Minute})
+ }
+ }
+ if err != nil {
+ fmt.Fprintln(stderr, err)
+ return 1
+ }
+ e.Durable = !*noSync
+ e.VerifyKept = *verifyKept
+ if *logPhases {
+ l := log.New(stderr, "", log.Lmicroseconds)
+ e.Logf = l.Printf
+ }
+ ctx := context.Background()
+ switch args[0] {
+ case "publish":
+ if fs.NArg() != 1 {
+ fmt.Fprint(stderr, casyncUsage)
+ return 2
+ }
+ start := time.Now()
+ d, err := e.Publish(ctx, fs.Arg(0))
+ if err != nil {
+ fmt.Fprintln(stderr, err)
+ return 1
+ }
+ fmt.Fprintf(stdout, "%s\t%.2fs\n", d, time.Since(start).Seconds())
+ return 0
+ case "chunks":
+ // Dump the chunk list of every file of a manifest (start, size, id) as JSON, for capacity
+ // and transfer accounting outside the engine.
+ if fs.NArg() != 1 {
+ fmt.Fprint(stderr, casyncUsage)
+ return 2
+ }
+ m, err := digest.Parse(fs.Arg(0))
+ if err != nil {
+ fmt.Fprintln(stderr, err)
+ return 2
+ }
+ a, idxs, err := e.Load(m)
+ if err != nil {
+ fmt.Fprintln(stderr, err)
+ return 1
+ }
+ type fileChunks struct {
+ Path string `json:"path"`
+ Size int64 `json:"size"`
+ Chunks [][3]string `json:"chunks"`
+ }
+ out := make([]fileChunks, len(a.Files))
+ for i, f := range a.Files {
+ out[i] = fileChunks{Path: f.Path, Size: f.Size}
+ for _, c := range idxs[i].Chunks {
+ out[i].Chunks = append(out[i].Chunks, [3]string{fmt.Sprint(c.Start), fmt.Sprint(c.Size), c.ID.String()})
+ }
+ }
+ if err := json.NewEncoder(stdout).Encode(out); err != nil {
+ fmt.Fprintln(stderr, err)
+ return 1
+ }
+ return 0
+ case "pull", "verify", "attest":
+ if fs.NArg() != 2 {
+ fmt.Fprint(stderr, casyncUsage)
+ return 2
+ }
+ m, err := digest.Parse(fs.Arg(0))
+ if err != nil {
+ fmt.Fprintln(stderr, err)
+ return 2
+ }
+ dest := fs.Arg(1)
+ if args[0] != "pull" {
+ if err := e.Verify(ctx, m, dest); err != nil {
+ fmt.Fprintln(stderr, err)
+ return 1
+ }
+ if args[0] == "attest" {
+ if err := e.Attest(m, dest); err != nil {
+ fmt.Fprintln(stderr, err)
+ return 1
+ }
+ }
+ return 0
+ }
+ if err := os.MkdirAll(dest, 0o755); err != nil {
+ fmt.Fprintln(stderr, err)
+ return 1
+ }
+ var offer *csi.Offer
+ if *base != "" {
+ b, err := digest.Parse(*base)
+ if err != nil {
+ fmt.Fprintln(stderr, err)
+ return 2
+ }
+ offer = &csi.Offer{Bases: []store.Base{{Manifest: b, Dir: *baseDir}},
+ Claim: func(sb store.Base) (string, bool) { return sb.Dir, true }}
+ }
+ if *cp != "" {
+ c, err := digest.Parse(*cp)
+ if err != nil {
+ fmt.Fprintln(stderr, err)
+ return 2
+ }
+ if offer == nil {
+ offer = &csi.Offer{}
+ }
+ offer.Readable, offer.BaseCopy = []store.Base{{Manifest: c, Dir: *copyDir}}, store.BaseCopyAlways
+ }
+ st, err := e.Pull(ctx, m, dest, offer)
+ if err != nil {
+ fmt.Fprintln(stderr, err)
+ return 1
+ }
+ js, _ := json.Marshal(st)
+ fmt.Fprintln(stdout, string(js))
+ return 0
+ }
+ fmt.Fprint(stderr, casyncUsage)
+ return 2
+}
diff --git a/cmd/onggi/csi.go b/cmd/onggi/csi.go
index 3c86d4e..a8ce63c 100644
--- a/cmd/onggi/csi.go
+++ b/cmd/onggi/csi.go
@@ -17,6 +17,7 @@ import (
bfs "github.com/ai-computing/onggi/pkg/backend/fs"
"github.com/ai-computing/onggi/pkg/csi"
"github.com/ai-computing/onggi/pkg/digest"
+ "github.com/ai-computing/onggi/pkg/casync"
"github.com/ai-computing/onggi/pkg/peer"
"github.com/ai-computing/onggi/pkg/store"
)
@@ -38,6 +39,13 @@ func csiNode(args []string, stderr io.Writer) int {
keepUsed := fs.Duration("keep-used", 7*24*time.Hour, "the node cache keeps what artifacts used within this period need")
gcGrace := fs.Duration("gc-grace", time.Hour, "grace period of the node cache's GC")
rebaseIdle := fs.Duration("rebase-idle", 5*time.Minute, "a pull may turn an artifact no volume used for this long into the new one, writing only what differs (0: never)")
+ maxArtifacts := fs.Int("max-artifacts", 0, "artifacts kept per node; a pull takes an idle one over only when the node holds this many, least recently used first (0: no limit)")
+ verifyKept := fs.Bool("verify-kept", true, "read back and hash-check every chunk a pull leaves in place when it takes an artifact over")
+ baseCopy := fs.String("base-copy", "remote", "read an idle artifact that is not taken over where it is: remote (only if the store is not local), always, never")
+ engine := fs.String("engine", "store", "how artifacts are materialized: store (onggi packs) or casync (desync chunk store with state-aware in-place reuse)")
+ casyncStore := fs.String("casync-store", "", "casync engine: desync local chunk store")
+ casyncIndex := fs.String("casync-index", "", "casync engine: directory of artifact indexes (onggi casync publish)")
+ casyncVerify := fs.Bool("casync-verify-always", false, "casync engine: read back kept chunks even when the base is attested unchanged")
peerListen := fs.String("peer-listen", "", "serve the node cache to other nodes' pulls on this address, e.g. :7070 (#37)")
peers := fs.String("peers", "", "comma-separated peer URLs asked for chunks before the store")
peersDNS := fs.String("peers-dns", "", "host:port whose name resolves to the peers, e.g. a headless Service; re-resolved every 30 s")
@@ -100,12 +108,31 @@ func csiNode(args []string, stderr io.Writer) int {
}
}
m := &csi.Materializer{Root: *dataDir, Pull: csi.StorePullerPeers(s, openCache, peerList, tmp, *workers), Logf: logger.Printf, MaxPulls: *pulls}
+ if *engine == "casync" {
+ e, err := casync.New(*casyncStore, *casyncIndex, filepath.Join(*dataDir, "casync-attest"), *workers)
+ if err != nil || *casyncStore == "" || *casyncIndex == "" {
+ logger.Print("casync engine: needs -casync-store and -casync-index: ", err)
+ return 2
+ }
+ e.VerifyKept, e.Logf = *casyncVerify, logger.Printf
+ m.Pull = e.Pull
+ *verifyKept = *casyncVerify // the engine decides from the attestation; the offer forces a read-back only on request
+ } else if *engine != "store" {
+ logger.Printf("-engine: %q is not store or casync", *engine)
+ return 2
+ }
if err := m.Init(); err != nil {
logger.Print("data dir: ", err)
return 1
}
mounter := csi.NewMounter()
m.RebaseIdle, m.Unmount = *rebaseIdle, mounter.Unmount
+ m.MaxArtifacts, m.VerifyKept = *maxArtifacts, *verifyKept
+ var ok bool
+ if m.BaseCopy, ok = parseBaseCopy(*baseCopy); !ok {
+ logger.Printf("-base-copy: %q is not remote, always or never", *baseCopy)
+ return 2
+ }
n := &csi.Node{
Version: version,
NodeID: *nodeID,
diff --git a/cmd/onggi/main.go b/cmd/onggi/main.go
index ae9f0bc..e63bf1e 100644
--- a/cmd/onggi/main.go
+++ b/cmd/onggi/main.go
@@ -34,6 +34,7 @@ usage:
evict from a node cache what recently used
artifacts do not need
onggi dedup-report [flags] ... measure how much the files would shrink in onggi
+ onggi casync publish|pull|verify|attest ... casync-format artifacts with state-aware in-place reuse
onggi csi --store S [flags] CSI node plugin: serve artifacts as read-only volumes (Linux)
onggi peer serve --cache DIR [--listen :7070] serve a node cache to other nodes' pulls (--peers)
onggi version
@@ -61,6 +62,8 @@ func main() {
os.Exit(inspect(os.Args[2:], os.Stdout, os.Stderr))
case "diff":
os.Exit(diff(os.Args[2:], os.Stdout, os.Stderr))
+ case "casync":
+ os.Exit(casyncCmd(os.Args[2:], os.Stdout, os.Stderr))
case "csi":
os.Exit(csiNode(os.Args[2:], os.Stderr))
case "rm-ref":
diff --git a/cmd/onggi/store.go b/cmd/onggi/store.go
index 21ba66c..d327dfe 100644
--- a/cmd/onggi/store.go
+++ b/cmd/onggi/store.go
@@ -26,6 +26,7 @@ import (
type storeFlags struct {
url, durability, min, avg, max, sub, subKinds, packSize, frame *string
+ frameFlat *bool
}
func addStoreFlags(fs *flag.FlagSet) storeFlags {
@@ -38,6 +39,7 @@ func addStoreFlags(fs *flag.FlagSet) storeFlags {
sub: fs.String("sub", "4KiB", "average sub-chunk size for two-level chunking, 0 for one level (only when creating a store)"),
subKinds: fs.String("sub-kinds", "atom-prefill,atom-decode,atom-other", "file kinds that get sub-chunks, or all (only when creating a store)"),
packSize: fs.String("pack-size", "64MiB", "pack object size (only when creating a store)"),
+ frameFlat: fs.Bool("frame-flat", false, "with --sub 0, compress the new chunks of a one-level store in frames of --frame too (only when creating a store)"),
frame: fs.String("frame", "256KiB", "compress new chunks of sub-chunked files together up to this size, 0 for one by one (only when creating a store)"),
}
}
@@ -101,6 +103,13 @@ func (f storeFlags) open(ctx context.Context) (*store.Store, backend.Refs, error
}
}
}
+ if c.Sub == nil && *f.frameFlat && *f.frame != "0" {
+ fr, err := size(*f.frame)
+ if err != nil {
+ return nil, nil, err
+ }
+ c.Frame, c.FrameFlat = int64(fr), true
+ }
s, err := store.OpenWith(ctx, b, c)
return s, b, err
}
@@ -192,6 +201,10 @@ func pull(args []string, stdout, stderr io.Writer) int {
peers := fs.String("peers", "", "comma-separated peer URLs (onggi peer serve) asked for chunks before the store")
baseRef := fs.String("base", "", "an artifact already restored in --base-dir that the pull may turn into this one, writing only what differs; its files are moved away (#41)")
baseDir := fs.String("base-dir", "", "the directory holding --base")
+ baseIntact := fs.Bool("base-intact", false, "never move --base's files away: read from them where they are")
+ baseCopy := fs.String("base-copy", "always", "read a base that is not taken over where it is: always, remote (only if the store is not local), never")
+ baseRange := fs.Int64("base-range", 0, "largest single read from the base's files in bytes (0: 16 MiB; 1: chunk by chunk)")
+ verifyKept := fs.Bool("verify-kept", false, "read back and hash-check every chunk left in place when --base is taken over")
asJSON := fs.Bool("json", false, "print JSON")
if err := fs.Parse(args); err != nil || fs.NArg() != 2 {
fmt.Fprintln(stderr, "usage: onggi pull --store S [--cache DIR] [--peers URLS] [--base REF --base-dir DIR] [ ")
@@ -220,8 +233,14 @@ func pull(args []string, stdout, stderr io.Writer) int {
return 1
}
o.Bases = []store.Base{{Manifest: bd, Dir: *baseDir}}
- o.Claim = func(b store.Base) (string, bool) { return b.Dir, true }
+ o.Claim = func(b store.Base) (string, bool) { return b.Dir, !*baseIntact }
}
+ var ok bool
+ if o.BaseCopy, ok = parseBaseCopy(*baseCopy); !ok {
+ fmt.Fprintf(stderr, "onggi: --base-copy: %q is not always, remote or never\n", *baseCopy)
+ return 2
+ }
+ o.VerifyKept, o.BaseRange = *verifyKept, *baseRange
if *cacheDir != "" {
cb, err := bfs.Open(*cacheDir, backend.DurabilityNone)
if err == nil {
@@ -430,3 +449,16 @@ func diff(args []string, stdout, stderr io.Writer) int {
}
return 0
}
+
+// parseBaseCopy maps a --base-copy value to the rule.
+func parseBaseCopy(v string) (store.BaseCopy, bool) {
+ switch v {
+ case "always":
+ return store.BaseCopyAlways, true
+ case "remote":
+ return store.BaseCopyRemote, true
+ case "never":
+ return store.BaseCopyNever, true
+ }
+ return store.BaseCopyAlways, false
+}
diff --git a/go.mod b/go.mod
index 4fe7267..c55c69e 100644
--- a/go.mod
+++ b/go.mod
@@ -1,32 +1,87 @@
module github.com/ai-computing/onggi
-go 1.24
+go 1.25.0
require (
github.com/container-storage-interface/spec v1.12.0
- github.com/klauspost/compress v1.18.0
- github.com/minio/minio-go/v7 v7.0.95
- golang.org/x/sys v0.33.0
- google.golang.org/grpc v1.75.1
+ github.com/folbricht/desync v1.1.5
+ github.com/klauspost/compress v1.19.2
+ github.com/minio/minio-go/v7 v7.3.0
+ golang.org/x/sys v0.47.0
+ google.golang.org/grpc v1.83.2
)
require (
- github.com/davecgh/go-spew v1.1.1 // indirect
+ cel.dev/expr v0.25.2 // indirect
+ cloud.google.com/go v0.123.0 // indirect
+ cloud.google.com/go/auth v0.18.2 // indirect
+ cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
+ cloud.google.com/go/compute/metadata v0.9.0 // indirect
+ cloud.google.com/go/iam v1.5.3 // indirect
+ cloud.google.com/go/monitoring v1.24.3 // indirect
+ cloud.google.com/go/storage v1.60.0 // indirect
+ github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 // indirect
+ github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect
+ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect
+ github.com/boljen/go-bitmap v0.0.0-20151001105940-23cd2fb0ce7d // indirect
+ github.com/cespare/xxhash/v2 v2.3.0 // indirect
+ github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
+ github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect
+ github.com/dchest/siphash v1.2.3 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
- github.com/go-ini/ini v1.67.0 // indirect
- github.com/goccy/go-json v0.10.5 // indirect
+ github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect
+ github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect
+ github.com/felixge/httpsnoop v1.0.4 // indirect
+ github.com/folbricht/tempfile v0.0.1 // indirect
+ github.com/go-jose/go-jose/v4 v4.1.4 // indirect
+ github.com/go-logr/logr v1.4.4 // indirect
+ github.com/go-logr/stdr v1.2.2 // indirect
+ github.com/google/s2a-go v0.1.9 // indirect
github.com/google/uuid v1.6.0 // indirect
- github.com/klauspost/cpuid/v2 v2.2.11 // indirect
- github.com/minio/crc64nvme v1.0.2 // indirect
+ github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect
+ github.com/googleapis/gax-go/v2 v2.17.0 // indirect
+ github.com/hanwen/go-fuse/v2 v2.11.0 // indirect
+ github.com/klauspost/cpuid/v2 v2.4.0 // indirect
+ github.com/klauspost/crc32 v1.3.0 // indirect
+ github.com/kr/fs v0.1.0 // indirect
+ github.com/mattn/go-runewidth v0.0.23 // indirect
+ github.com/minio/crc64nvme v1.1.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
+ github.com/opencontainers/go-digest v1.0.0 // indirect
+ github.com/opencontainers/image-spec v1.1.1 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
- github.com/pmezard/go-difflib v1.0.0 // indirect
+ github.com/pkg/errors v0.9.1 // indirect
+ github.com/pkg/sftp v1.13.5 // indirect
+ github.com/pkg/xattr v0.4.9 // indirect
+ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
github.com/rs/xid v1.6.0 // indirect
- github.com/tinylib/msgp v1.3.0 // indirect
- golang.org/x/crypto v0.39.0 // indirect
- golang.org/x/net v0.41.0 // indirect
- golang.org/x/text v0.26.0 // indirect
- google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect
- google.golang.org/protobuf v1.36.6 // indirect
- gopkg.in/yaml.v3 v3.0.1 // indirect
+ github.com/sirupsen/logrus v1.9.4 // indirect
+ github.com/spiffe/go-spiffe/v2 v2.7.0 // indirect
+ github.com/tinylib/msgp v1.6.4 // indirect
+ github.com/zeebo/xxh3 v1.1.0 // indirect
+ go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+ go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect
+ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
+ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
+ go.opentelemetry.io/otel v1.45.0 // indirect
+ go.opentelemetry.io/otel/metric v1.45.0 // indirect
+ go.opentelemetry.io/otel/sdk v1.45.0 // indirect
+ go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect
+ go.opentelemetry.io/otel/trace v1.45.0 // indirect
+ go.yaml.in/yaml/v3 v3.0.5 // indirect
+ golang.org/x/crypto v0.55.0 // indirect
+ golang.org/x/net v0.58.0 // indirect
+ golang.org/x/oauth2 v0.36.0 // indirect
+ golang.org/x/sync v0.22.0 // indirect
+ golang.org/x/term v0.45.0 // indirect
+ golang.org/x/text v0.41.0 // indirect
+ golang.org/x/time v0.14.0 // indirect
+ google.golang.org/api v0.267.0 // indirect
+ google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect
+ google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect
+ google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect
+ google.golang.org/protobuf v1.36.11 // indirect
+ gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
+ gopkg.in/ini.v1 v1.67.3 // indirect
+ oras.land/oras-go/v2 v2.6.2 // indirect
)
diff --git a/go.sum b/go.sum
index 85baef5..36902fa 100644
--- a/go.sum
+++ b/go.sum
@@ -1,72 +1,225 @@
+cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs=
+cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
+cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE=
+cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU=
+cloud.google.com/go/auth v0.18.2 h1:+Nbt5Ev0xEqxlNjd6c+yYUeosQ5TtEUaNcN/3FozlaM=
+cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M=
+cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
+cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
+cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
+cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
+cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc=
+cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU=
+cloud.google.com/go/logging v1.13.1 h1:O7LvmO0kGLaHY/gq8cV7T0dyp6zJhYAOtZPX4TF3QtY=
+cloud.google.com/go/logging v1.13.1/go.mod h1:XAQkfkMBxQRjQek96WLPNze7vsOmay9H5PqfsNYDqvw=
+cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8=
+cloud.google.com/go/longrunning v0.8.0/go.mod h1:UmErU2Onzi+fKDg2gR7dusz11Pe26aknR4kHmJJqIfk=
+cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE=
+cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI=
+cloud.google.com/go/storage v1.60.0 h1:oBfZrSOCimggVNz9Y/bXY35uUcts7OViubeddTTVzQ8=
+cloud.google.com/go/storage v1.60.0/go.mod h1:q+5196hXfejkctrnx+VYU8RKQr/L3c0cBIlrjmiAKE0=
+cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U=
+cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s=
+github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 h1:l7+6kwRMJNwdCvYdDl7Eax+wzEYHSnNY7zrrfbhDdTA=
+github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0/go.mod h1:pJTkW8hEUIIi3Pf65lPZOnn4Y81yCllX6IWk2jNXdkM=
+github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8=
+github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc=
+github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0 h1:7t/qx5Ost0s0wbA/VDrByOooURhp+ikYwv20i9Y07TQ=
+github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA=
+github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk=
+github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc=
+github.com/boljen/go-bitmap v0.0.0-20151001105940-23cd2fb0ce7d h1:zsO4lp+bjv5XvPTF58Vq+qgmZEYZttJK+CWtSZhKenI=
+github.com/boljen/go-bitmap v0.0.0-20151001105940-23cd2fb0ce7d/go.mod h1:f1iKL6ZhUWvbk7PdWVmOaak10o86cqMUYEmn1CZNGEI=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
+github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
+github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik=
+github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4=
github.com/container-storage-interface/spec v1.12.0 h1:zrFOEqpR5AghNaaDG4qyedwPBqU2fU0dWjLQMP/azK0=
github.com/container-storage-interface/spec v1.12.0/go.mod h1:txsm+MA2B2WDa5kW69jNbqPnvTtfvZma7T/zsAZ9qX8=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/siphash v1.2.3 h1:QXwFc8cFOR2dSa/gE6o/HokBMWtLUaNDVd+22aKHeEA=
+github.com/dchest/siphash v1.2.3/go.mod h1:0NvQU092bT0ipiFN++/rXm69QG9tVxLAlQHIXMPAkHc=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
-github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
-github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
-github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
-github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA=
+github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU=
+github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ=
+github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A=
+github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI=
+github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4=
+github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds=
+github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0=
+github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w=
+github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE=
+github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
+github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/folbricht/desync v1.1.5 h1:FxWghh6PzTglDW6zh6Gx235H33EffX+yxgJu2bBtc+A=
+github.com/folbricht/desync v1.1.5/go.mod h1:AvU6wBGs+PrGrInuNo1FX9k+AGNPohdeB6/O4mgke7c=
+github.com/folbricht/tempfile v0.0.1 h1:kB3DubP2Fm5e3W7TrWFNZBfzFEHBoKL7Pjn0HvqKxSQ=
+github.com/folbricht/tempfile v0.0.1/go.mod h1:/Flpxx/6U+clQJ61jQ3y6Z7L2l6j1/ZSiU4B9EDPgWw=
+github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
+github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
+github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
-github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
-github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc=
+github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0=
+github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
+github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
-github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao=
+github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8=
+github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc=
+github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY=
+github.com/hanwen/go-fuse/v2 v2.11.0 h1:CGVkJh9gRz0pTRMADNcqdFl3ec/5QbE/Vx1Gl7ESozM=
+github.com/hanwen/go-fuse/v2 v2.11.0/go.mod h1:aU7NkGYZUmuJrZapoI3mEcNve7PZTySUOLBuch/vR6U=
+github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
+github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
-github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
-github.com/minio/crc64nvme v1.0.2 h1:6uO1UxGAD+kwqWWp7mBFsi5gAse66C4NXO8cmcVculg=
-github.com/minio/crc64nvme v1.0.2/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
+github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
+github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
+github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
+github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
+github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
+github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw=
+github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
+github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
+github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
-github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU=
-github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo=
+github.com/minio/minio-go/v7 v7.3.0 h1:HM4pFCSQq/TK+j0/zmorSh5ddh81iDgRgU0BG0Vz/YU=
+github.com/minio/minio-go/v7 v7.3.0/go.mod h1:KUPWdecEO1LWyUz+sTGXAuf2jZHrPh5fCsRH86QbPfk=
+github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg=
+github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4=
+github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
+github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.13.5 h1:a3RLUqkyjYRtBTZJZ1VRrKbN3zhuPLlUc3sphVz81go=
+github.com/pkg/sftp v1.13.5/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg=
+github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
+github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
+github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
+github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww=
-github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0=
-go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
-go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
-go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
-go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
-go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
-go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
-go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=
-go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=
-go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc=
-go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps=
-go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
-go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
-golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
-golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
-golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
-golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
-golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
-golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
-golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
-golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
-gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
-gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
-google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI=
-google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ=
-google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
-google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
+github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
+github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
+github.com/spiffe/go-spiffe/v2 v2.7.0 h1:uXe1MflJoHw58wAUvxVlcM7WpKtijWG7I1UidcGh6g4=
+github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ=
+github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
+github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
+github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
+github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
+github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
+go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
+go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
+go.opentelemetry.io/contrib/detectors/gcp v1.44.0 h1:NmLfL734pJhM0JKaYd2Y28+nY9dPRWYAAbxhRCrKXPw=
+go.opentelemetry.io/contrib/detectors/gcp v1.44.0/go.mod h1:tNAsgd8avTGke1+MndXlU5Cru4PQ9Ai/cCNWQv/ZJ/s=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q=
+go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU=
+go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ=
+go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0 h1:5gn2urDL/FBnK8OkCfD1j3/ER79rUuTYmCvlXBKeYL8=
+go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0/go.mod h1:0fBG6ZJxhqByfFZDwSwpZGzJU671HkwpWaNe2t4VUPI=
+go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M=
+go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s=
+go.opentelemetry.io/otel/metric/x v0.67.0 h1:PcicCNZFkZ4bXfSooXdo3WN7RBOVOtjVdo1wD358Uns=
+go.opentelemetry.io/otel/metric/x v0.67.0/go.mod h1:FBjCWZe6wgcqxcMtjdGiClDKXb2YxxXii0CXftE4QtI=
+go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw=
+go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA=
+go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o=
+go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA=
+go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag=
+go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc=
+go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
+go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
+golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
+golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
+golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
+golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
+golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
+golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
+golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
+golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
+golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
+golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
+golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
+gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
+google.golang.org/api v0.267.0 h1:w+vfWPMPYeRs8qH1aYYsFX68jMls5acWl/jocfLomwE=
+google.golang.org/api v0.267.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0=
+google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM=
+google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM=
+google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8=
+google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
+google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU=
+google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8=
+google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
+google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk=
+gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
+gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw=
+gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+oras.land/oras-go/v2 v2.6.2 h1:N04RXngAp1LJKTG6ifz3xHPipasEkWr+hFmInja5YKo=
+oras.land/oras-go/v2 v2.6.2/go.mod h1:PlTtg4JTDJkDe8yVHpM2wz7/YDc00GVas+i4jAW2TZ4=
diff --git a/pkg/backend/backend.go b/pkg/backend/backend.go
index eb03630..5defad2 100644
--- a/pkg/backend/backend.go
+++ b/pkg/backend/backend.go
@@ -49,6 +49,8 @@ type Capabilities struct {
RangeReads bool
AtomicCreate bool // a second Put of the same key can never expose partial or mixed bytes
Durability Durability
+ // Local says the objects are files on this machine, so a range read costs no round trip.
+ Local bool
}
// Backend stores immutable objects.
diff --git a/pkg/backend/fs/fs.go b/pkg/backend/fs/fs.go
index 40d71d1..778767f 100644
--- a/pkg/backend/fs/fs.go
+++ b/pkg/backend/fs/fs.go
@@ -180,7 +180,7 @@ func (s *Store) Delete(ctx context.Context, k backend.Key) error {
// Caps: range reads; atomic create through link or rename.
func (s *Store) Caps() backend.Capabilities {
- return backend.Capabilities{RangeReads: true, AtomicCreate: true, Durability: s.durability}
+ return backend.Capabilities{RangeReads: true, AtomicCreate: true, Durability: s.durability, Local: true}
}
// ---- refs: a lock file per ref makes SetRef a compare-and-swap on a single mount.
diff --git a/pkg/casync/casync.go b/pkg/casync/casync.go
new file mode 100644
index 0000000..be714e7
--- /dev/null
+++ b/pkg/casync/casync.go
@@ -0,0 +1,730 @@
+// Package casync materializes artifacts from a casync-format chunk store (desync), and adds what a
+// stateless tool cannot do: it knows which sibling on the node is idle, aligned and unchanged since
+// it was written, and turns that sibling into the requested artifact by writing only the chunks that
+// differ, without reading the ones it keeps.
+//
+// Layout under Engine.IndexDir:
+//
+// /artifact.json files of the artifact: path, size, mode, index file
+// /.caibx casync index of file i
+//
+// The manifest digest is the SHA-256 of artifact.json. Chunks live in a desync local store
+// (one zstd-compressed file per chunk, named by its SHA512/256). Engine.AttestDir records, for every
+// file this engine wrote, the inode, size and modification time it had when the write completed;
+// a later Keep trusts the chunks it leaves in place only if the file still has them (state-aware
+// in-place), and otherwise reads them back and checks each digest.
+package casync
+
+import (
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+ "sort"
+ "sync"
+ "sync/atomic"
+ "syscall"
+ "time"
+
+ "github.com/folbricht/desync"
+
+ "github.com/ai-computing/onggi/pkg/csi"
+ "github.com/ai-computing/onggi/pkg/digest"
+ "github.com/ai-computing/onggi/pkg/store"
+)
+
+// Chunk sizes of desync's defaults (16/64/256 KiB).
+const (
+ MinChunk = 16 << 10
+ AvgChunk = 64 << 10
+ MaxChunk = 256 << 10
+)
+
+// Engine reads and writes artifacts in casync format.
+type Engine struct {
+ Store desync.LocalStore
+ // Src, if set, is where pulls read chunks from instead of Store (for example a remote HTTP
+ // chunk server); Store still receives published chunks.
+ Src desync.Store
+ IndexDir string
+ AttestDir string // "" : never trust kept chunks, always read them back
+ Workers int
+ MinSame float64 // share of bytes at the same offset needed to take a base (0: store.DefaultMinSame)
+ // VerifyKept forces reading back kept chunks even when the base is attested unchanged.
+ VerifyKept bool
+ // Durable syncs every written file before the pull returns (default true via New).
+ Durable bool
+ // Logf, if set, receives one line per phase with its duration.
+ Logf func(format string, args ...any)
+}
+
+// New opens the chunk store at storeDir and the index and attestation directories.
+func New(storeDir, indexDir, attestDir string, workers int) (*Engine, error) {
+ for _, d := range []string{storeDir, indexDir} {
+ if err := os.MkdirAll(d, 0o755); err != nil {
+ return nil, err
+ }
+ }
+ if attestDir != "" {
+ if err := os.MkdirAll(attestDir, 0o755); err != nil {
+ return nil, err
+ }
+ }
+ s, err := desync.NewLocalStore(storeDir, desync.StoreOptions{N: max(workers, 1)})
+ if err != nil {
+ return nil, err
+ }
+ return &Engine{Store: s, IndexDir: indexDir, AttestDir: attestDir, Workers: max(workers, 1), Durable: true}, nil
+}
+
+// FileEntry is one file of an artifact.
+type FileEntry struct {
+ Path string `json:"path"`
+ Size int64 `json:"size"`
+ Mode uint32 `json:"mode"`
+ Index string `json:"index"`
+ // IndexSHA256 is the SHA-256 of the index file, so the manifest digest covers the content.
+ IndexSHA256 string `json:"index_sha256"`
+}
+
+// Artifact is the content of artifact.json.
+type Artifact struct {
+ Files []FileEntry `json:"files"`
+}
+
+func (e *Engine) logf(format string, args ...any) {
+ if e.Logf != nil {
+ e.Logf(format, args...)
+ }
+}
+
+// Publish chunks every regular file under dir into the store and writes the artifact's indexes.
+// It returns the manifest digest.
+func (e *Engine) Publish(ctx context.Context, dir string) (digest.Digest, error) {
+ var paths []string
+ err := filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ if d.Type().IsRegular() {
+ rel, _ := filepath.Rel(dir, p)
+ paths = append(paths, filepath.ToSlash(rel))
+ }
+ return nil
+ })
+ if err != nil {
+ return digest.Digest{}, err
+ }
+ sort.Strings(paths)
+ tmp, err := os.MkdirTemp(e.IndexDir, ".publish-")
+ if err != nil {
+ return digest.Digest{}, err
+ }
+ defer os.RemoveAll(tmp)
+ var a Artifact
+ for i, rel := range paths {
+ src := filepath.Join(dir, filepath.FromSlash(rel))
+ fi, err := os.Stat(src)
+ if err != nil {
+ return digest.Digest{}, err
+ }
+ idx, _, err := desync.IndexFromFile(ctx, src, e.Workers, MinChunk, AvgChunk, MaxChunk, desync.NullProgressBar{})
+ if err != nil {
+ return digest.Digest{}, fmt.Errorf("index %s: %w", rel, err)
+ }
+ if err := desync.ChopFile(ctx, src, idx.Chunks, e.Store, e.Workers, desync.NullProgressBar{}); err != nil {
+ return digest.Digest{}, fmt.Errorf("chop %s: %w", rel, err)
+ }
+ name := fmt.Sprintf("%d.caibx", i)
+ var buf bytes.Buffer
+ if _, err := idx.WriteTo(&buf); err != nil {
+ return digest.Digest{}, err
+ }
+ if err := os.WriteFile(filepath.Join(tmp, name), buf.Bytes(), 0o644); err != nil {
+ return digest.Digest{}, err
+ }
+ isum := sha256.Sum256(buf.Bytes())
+ a.Files = append(a.Files, FileEntry{Path: rel, Size: fi.Size(), Mode: uint32(fi.Mode().Perm()), Index: name,
+ IndexSHA256: hex.EncodeToString(isum[:])})
+ }
+ data, _ := json.Marshal(a)
+ sum := sha256.Sum256(data)
+ d, err := digest.Parse("sha256:" + hex.EncodeToString(sum[:]))
+ if err != nil {
+ return digest.Digest{}, err
+ }
+ if err := os.WriteFile(filepath.Join(tmp, "artifact.json"), data, 0o644); err != nil {
+ return digest.Digest{}, err
+ }
+ final := filepath.Join(e.IndexDir, d.Hex())
+ if _, err := os.Stat(final); err == nil {
+ return d, nil // published before: same content
+ }
+ if err := os.Rename(tmp, final); err != nil {
+ return digest.Digest{}, err
+ }
+ return d, nil
+}
+
+// Load returns the artifact's file list and the casync index of each file.
+func (e *Engine) Load(d digest.Digest) (*Artifact, []desync.Index, error) {
+ dir := filepath.Join(e.IndexDir, d.Hex())
+ data, err := os.ReadFile(filepath.Join(dir, "artifact.json"))
+ if err != nil {
+ return nil, nil, err
+ }
+ sum := sha256.Sum256(data)
+ if hex.EncodeToString(sum[:]) != d.Hex() {
+ return nil, nil, fmt.Errorf("artifact %s: artifact.json does not match its digest", d)
+ }
+ var a Artifact
+ if err := json.Unmarshal(data, &a); err != nil {
+ return nil, nil, err
+ }
+ idxs := make([]desync.Index, len(a.Files))
+ for i, f := range a.Files {
+ raw, err := os.ReadFile(filepath.Join(dir, f.Index))
+ if err != nil {
+ return nil, nil, err
+ }
+ if isum := sha256.Sum256(raw); hex.EncodeToString(isum[:]) != f.IndexSHA256 {
+ return nil, nil, fmt.Errorf("%s: index does not match the manifest", f.Path)
+ }
+ idxs[i], err = desync.IndexFromReader(bytes.NewReader(raw))
+ if err != nil {
+ return nil, nil, fmt.Errorf("%s: %w", f.Path, err)
+ }
+ }
+ return &a, idxs, nil
+}
+
+// sameOffset marks the target chunks that the base file holds at the same offset. Both chunk
+// lists are sorted by offset, so one pass suffices; no file is read.
+func sameOffset(base, target []desync.IndexChunk) ([]bool, int64) {
+ kept := make([]bool, len(target))
+ var same int64
+ j := 0
+ for i, c := range target {
+ for j < len(base) && base[j].Start < c.Start {
+ j++
+ }
+ if j < len(base) && base[j].Start == c.Start && base[j].ID == c.ID && base[j].Size == c.Size {
+ kept[i] = true
+ same += int64(c.Size)
+ }
+ }
+ return kept, same
+}
+
+// keepPlan pairs the target's files with a base's files by path.
+type keepPlan struct {
+ base store.Base
+ files *Artifact
+ idxs []desync.Index
+ pair []int // base file per target file, -1: none
+ kept [][]bool // per target file, chunks the paired base file holds at the same offset
+ same int64
+}
+
+func (e *Engine) plan(target *Artifact, tidx []desync.Index, b store.Base) (keepPlan, error) {
+ ba, bidx, err := e.Load(b.Manifest)
+ if err != nil {
+ return keepPlan{}, err
+ }
+ byPath := map[string]int{}
+ for j, f := range ba.Files {
+ byPath[f.Path] = j
+ }
+ p := keepPlan{base: b, files: ba, idxs: bidx, pair: make([]int, len(target.Files)), kept: make([][]bool, len(target.Files))}
+ for i, f := range target.Files {
+ j, ok := byPath[f.Path]
+ if !ok {
+ p.pair[i] = -1
+ continue
+ }
+ p.pair[i] = j
+ var s int64
+ p.kept[i], s = sameOffset(bidx[j].Chunks, tidx[i].Chunks)
+ p.same += s
+ }
+ // Variants of different shapes usually name their files differently (decode_S256_B2 vs _B8).
+ // A target file without a namesake takes the unused base file that holds the most of its
+ // bytes at the same offset, decided from the two indexes alone.
+ used := map[int]bool{}
+ for _, j := range p.pair {
+ if j >= 0 {
+ used[j] = true
+ }
+ }
+ for i := range target.Files {
+ if p.pair[i] >= 0 {
+ continue
+ }
+ best, bestSame, bestShared := -1, int64(0), int64(0)
+ var bestKept []bool
+ for j := range ba.Files {
+ if used[j] {
+ continue
+ }
+ k, s := sameOffset(bidx[j].Chunks, tidx[i].Chunks)
+ // prefer bytes at the same offset (Keep); otherwise bytes held anywhere (a seed for Copy)
+ sh := sharedBytes([]desync.Index{bidx[j]}, []desync.Index{tidx[i]})
+ if s > bestSame || (s == bestSame && sh > bestShared) {
+ best, bestSame, bestShared, bestKept = j, s, sh, k
+ }
+ }
+ if bestSame == 0 && bestShared == 0 {
+ best = -1
+ }
+ if best >= 0 {
+ p.pair[i], p.kept[i] = best, bestKept
+ p.same += bestSame
+ used[best] = true
+ }
+ }
+ return p, nil
+}
+
+// attestation is what a file looked like when this engine finished writing it.
+type attestation struct {
+ Ino uint64 `json:"ino"`
+ Size int64 `json:"size"`
+ MtimeNs int64 `json:"mtime_ns"`
+}
+
+func stat(p string) (attestation, error) {
+ fi, err := os.Stat(p)
+ if err != nil {
+ return attestation{}, err
+ }
+ a := attestation{Size: fi.Size(), MtimeNs: fi.ModTime().UnixNano()}
+ if st, ok := fi.Sys().(*syscall.Stat_t); ok {
+ a.Ino = uint64(st.Ino)
+ }
+ return a, nil
+}
+
+func (e *Engine) attestFile(d digest.Digest) string {
+ return filepath.Join(e.AttestDir, d.Hex()+".json")
+}
+
+func (e *Engine) recordAttestation(d digest.Digest, dir string, a *Artifact) error {
+ if e.AttestDir == "" {
+ return nil
+ }
+ rec := map[string]attestation{}
+ for _, f := range a.Files {
+ s, err := stat(filepath.Join(dir, filepath.FromSlash(f.Path)))
+ if err != nil {
+ return err
+ }
+ rec[f.Path] = s
+ }
+ data, _ := json.Marshal(rec)
+ tmp := e.attestFile(d) + ".tmp"
+ if err := os.WriteFile(tmp, data, 0o644); err != nil {
+ return err
+ }
+ return os.Rename(tmp, e.attestFile(d))
+}
+
+// Attest records the files of manifest d under dir as unchanged; call it only after Verify.
+func (e *Engine) Attest(d digest.Digest, dir string) error {
+ a, _, err := e.Load(d)
+ if err != nil {
+ return err
+ }
+ return e.recordAttestation(d, dir, a)
+}
+
+// unchanged reports whether the base file at p is the one this engine wrote for manifest d.
+func (e *Engine) unchanged(d digest.Digest, path, p string) bool {
+ if e.AttestDir == "" {
+ return false
+ }
+ data, err := os.ReadFile(e.attestFile(d))
+ if err != nil {
+ return false
+ }
+ var rec map[string]attestation
+ if json.Unmarshal(data, &rec) != nil {
+ return false
+ }
+ want, ok := rec[path]
+ if !ok {
+ return false
+ }
+ got, err := stat(p)
+ return err == nil && got == want
+}
+
+// Pull writes artifact m into the empty directory dest. With an offer it takes an idle aligned
+// base over (Keep), or reads a base that stays intact (Copy); otherwise it assembles from the
+// store (Reconstruct). It satisfies csi.Puller.
+func (e *Engine) Pull(ctx context.Context, m digest.Digest, dest string, offer *csi.Offer) (st store.PullStats, err error) {
+ var src desync.Store = e.Store
+ if e.Src != nil {
+ src = e.Src
+ }
+ cs := &countingStore{Store: src}
+ defer func() { st.FetchedChunks, st.FetchedBytes = cs.n.Load(), cs.b.Load() }()
+ return e.pull(ctx, m, dest, offer, cs)
+}
+
+// countingStore counts the chunks a pull takes from its chunk source and their logical size.
+type countingStore struct {
+ desync.Store
+ n, b atomic.Int64
+}
+
+func (c *countingStore) GetChunk(id desync.ChunkID) (*desync.Chunk, error) {
+ ch, err := c.Store.GetChunk(id)
+ if err == nil {
+ if d, derr := ch.Data(); derr == nil {
+ c.n.Add(1)
+ c.b.Add(int64(len(d)))
+ }
+ }
+ return ch, err
+}
+
+func (e *Engine) pull(ctx context.Context, m digest.Digest, dest string, offer *csi.Offer, src desync.Store) (store.PullStats, error) {
+ start := time.Now()
+ a, idxs, err := e.Load(m)
+ if err != nil {
+ return store.PullStats{}, err
+ }
+ st := store.PullStats{Files: len(a.Files)}
+ for _, f := range a.Files {
+ st.Bytes += f.Size
+ }
+ finals := make([]string, len(a.Files))
+ for i, f := range a.Files {
+ p := filepath.Join(dest, filepath.FromSlash(f.Path))
+ if !filepath.IsLocal(filepath.FromSlash(f.Path)) {
+ return st, fmt.Errorf("file path %q leaves the artifact", f.Path)
+ }
+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+ return st, err
+ }
+ finals[i] = p
+ }
+ done := make([]bool, len(a.Files))
+ minSame := e.MinSame
+ if minSame <= 0 {
+ minSame = store.DefaultMinSame
+ }
+
+ // Keep: the idle base that holds the most bytes at the same offsets
+ if offer != nil && offer.Claim != nil && len(offer.Bases) > 0 {
+ var best keepPlan
+ for _, b := range offer.Bases {
+ p, err := e.plan(a, idxs, b)
+ if err == nil && p.same > best.same {
+ best = p
+ }
+ }
+ st.BaseBest = best.same
+ if best.same > 0 && float64(best.same) >= minSame*float64(st.Bytes) {
+ if dir, ok := offer.Claim(best.base); ok {
+ st.Base = best.base.Manifest.String()
+ if err := e.keep(ctx, a, idxs, best, dir, finals, done, offer.VerifyKept || e.VerifyKept, &st, src); err != nil {
+ return st, err
+ }
+ }
+ }
+ }
+ // Copy: a base that stays where it is seeds the files Keep did not produce
+ var release func()
+ var seedBase *keepPlan
+ // Reading a sibling instead of the chunk source pays off when that source is remote (Src set,
+ // BaseCopyRemote) or when asked for explicitly (BaseCopyAlways).
+ copyOK := offer != nil && (offer.BaseCopy == store.BaseCopyAlways || (offer.BaseCopy == store.BaseCopyRemote && e.Src != nil))
+ if copyOK && len(offer.Readable) > 0 {
+ var most keepPlan
+ var mostShared int64
+ for _, b := range offer.Readable {
+ if p, err := e.plan(a, idxs, b); err == nil {
+ // a seed serves chunks at any offset, so count the target bytes the sibling holds anywhere
+ if sh := sharedBytes(p.idxs, idxs); sh > mostShared {
+ most, mostShared = p, sh
+ }
+ }
+ }
+ // the same threshold as Keep (Algorithm 1), on bytes held anywhere in the sibling
+ if mostShared > 0 && float64(mostShared) >= minSame*float64(st.Bytes) {
+ ok := true
+ release = func() {}
+ if offer.Hold != nil {
+ release, ok = offer.Hold(most.base)
+ }
+ if ok {
+ seedBase = &most
+ st.LocalBase = most.base.Manifest.String()
+ }
+ }
+ }
+ if release != nil {
+ defer release()
+ }
+ // whatever is left: assemble from the store, seeded by the held base if there is one
+ for i := range a.Files {
+ if done[i] {
+ continue
+ }
+ partial := finals[i] + ".casync-partial"
+ os.Remove(partial)
+ var seeds []desync.Seed
+ if seedBase != nil && seedBase.pair[i] >= 0 {
+ j := seedBase.pair[i]
+ dir, _ := baseDir(offer, seedBase.base)
+ src := filepath.Join(dir, filepath.FromSlash(seedBase.files.Files[j].Path))
+ s, err := desync.NewIndexSeed(partial, src, seedBase.idxs[j])
+ if err == nil {
+ seeds = append(seeds, s)
+ st.LocalBytes += sharedBytes([]desync.Index{seedBase.idxs[j]}, []desync.Index{idxs[i]})
+ }
+ }
+ t0 := time.Now()
+ if _, err := desync.AssembleFile(ctx, partial, idxs[i], src, seeds, desync.AssembleOptions{N: e.Workers, InvalidSeedAction: desync.InvalidSeedActionSkip}); err != nil {
+ os.Remove(partial)
+ return st, fmt.Errorf("%s: %w", a.Files[i].Path, err)
+ }
+ e.logf("assemble %s (%d seeds): %s", a.Files[i].Path, len(seeds), time.Since(t0).Round(time.Millisecond))
+ if err := e.finish(partial, finals[i], a.Files[i]); err != nil {
+ return st, err
+ }
+ done[i] = true
+ }
+ if err := e.recordAttestation(m, dest, a); err != nil {
+ return st, err
+ }
+ st.Duration = time.Since(start)
+ return st, nil
+}
+
+// baseDir is where a readable base's files are: the materializer keeps them under its root, and
+// the offer's Base.Dir says where.
+func baseDir(_ *csi.Offer, b store.Base) (string, bool) { return b.Dir, b.Dir != "" }
+
+func countKept(kept []bool, chunks []desync.IndexChunk) int64 {
+ var n int64
+ for i, k := range kept {
+ if k {
+ n += int64(chunks[i].Size)
+ }
+ }
+ return n
+}
+
+// keep turns the claimed base's files into the target's: each paired file is moved into place,
+// resized, and only its differing chunks are written from the store. Kept chunks are read back and
+// hash-checked unless the file is attested unchanged since this engine wrote it.
+func (e *Engine) keep(ctx context.Context, a *Artifact, idxs []desync.Index, p keepPlan, dir string, finals []string, done []bool, verify bool, st *store.PullStats, chunkSrc desync.Store) error {
+ for i, j := range p.pair {
+ if j < 0 {
+ continue
+ }
+ src := filepath.Join(dir, filepath.FromSlash(p.files.Files[j].Path))
+ trusted := !verify && e.unchanged(p.base.Manifest, p.files.Files[j].Path, src)
+ partial := finals[i] + ".casync-partial"
+ if err := os.Rename(src, partial); err != nil {
+ continue // left for the store
+ }
+ f, err := os.OpenFile(partial, os.O_RDWR, 0)
+ if err != nil {
+ return err
+ }
+ if err := f.Chmod(0o644); err != nil {
+ f.Close()
+ return err
+ }
+ if err := f.Truncate(a.Files[i].Size); err != nil {
+ f.Close()
+ return err
+ }
+ kept := append([]bool(nil), p.kept[i]...)
+ t0 := time.Now()
+ if !trusted {
+ ok, bad := e.verifyKept(ctx, f, idxs[i].Chunks, kept)
+ st.Verified += ok
+ st.Repaired += bad
+ }
+ t1 := time.Now()
+ written, err := e.writeChunks(ctx, f, idxs[i].Chunks, kept, chunkSrc)
+ if err != nil {
+ f.Close()
+ return err
+ }
+ t2 := time.Now()
+ if e.Durable {
+ if err := f.Sync(); err != nil {
+ f.Close()
+ return err
+ }
+ }
+ if err := f.Close(); err != nil {
+ return err
+ }
+ st.Reused += a.Files[i].Size - written
+ st.ReadBytes += written
+ e.logf("keep %s: trusted=%v verify %s, write %d B %s, sync %s", a.Files[i].Path, trusted,
+ t1.Sub(t0).Round(time.Millisecond), written, t2.Sub(t1).Round(time.Millisecond), time.Since(t2).Round(time.Millisecond))
+ if err := e.finish(partial, finals[i], a.Files[i]); err != nil {
+ return err
+ }
+ done[i] = true
+ }
+ return nil
+}
+
+// verifyKept reads every kept chunk and checks its digest; a chunk that does not match is no
+// longer kept, so the write step fetches it. It returns the bytes that matched and that did not.
+func (e *Engine) verifyKept(ctx context.Context, f *os.File, chunks []desync.IndexChunk, kept []bool) (ok, bad int64) {
+ var okN, badN atomic.Int64
+ work := make(chan int)
+ var wg sync.WaitGroup
+ for w := 0; w < e.Workers; w++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ var buf []byte
+ for i := range work {
+ c := chunks[i]
+ if cap(buf) < int(c.Size) {
+ buf = make([]byte, c.Size)
+ }
+ b := buf[:c.Size]
+ if _, err := f.ReadAt(b, int64(c.Start)); err != nil || desync.ChunkID(desync.Digest.Sum(b)) != c.ID {
+ kept[i] = false
+ badN.Add(int64(c.Size))
+ continue
+ }
+ okN.Add(int64(c.Size))
+ }
+ }()
+ }
+ for i := range chunks {
+ if kept[i] {
+ select {
+ case work <- i:
+ case <-ctx.Done():
+ }
+ }
+ }
+ close(work)
+ wg.Wait()
+ return okN.Load(), badN.Load()
+}
+
+// writeChunks writes every chunk that is not kept, from the store (each chunk's digest is checked
+// when it is read from the store). It returns the bytes written.
+func (e *Engine) writeChunks(ctx context.Context, f *os.File, chunks []desync.IndexChunk, kept []bool, src desync.Store) (int64, error) {
+ var written atomic.Int64
+ var firstErr error
+ var once sync.Once
+ work := make(chan int)
+ var wg sync.WaitGroup
+ for w := 0; w < e.Workers; w++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for i := range work {
+ c := chunks[i]
+ ch, err := src.GetChunk(c.ID)
+ if err == nil {
+ var b []byte
+ if b, err = ch.Data(); err == nil {
+ if uint64(len(b)) != c.Size {
+ err = fmt.Errorf("chunk %s: %d bytes, index says %d", c.ID, len(b), c.Size)
+ } else if _, err = f.WriteAt(b, int64(c.Start)); err == nil {
+ written.Add(int64(len(b)))
+ }
+ }
+ }
+ if err != nil {
+ once.Do(func() { firstErr = err })
+ }
+ }
+ }()
+ }
+ for i := range chunks {
+ if !kept[i] {
+ select {
+ case work <- i:
+ case <-ctx.Done():
+ once.Do(func() { firstErr = ctx.Err() })
+ }
+ }
+ }
+ close(work)
+ wg.Wait()
+ return written.Load(), firstErr
+}
+
+// finish syncs a partial file (when the writer did not), sets its mode, and renames it into place.
+func (e *Engine) finish(partial, final string, f FileEntry) error {
+ if e.Durable {
+ h, err := os.OpenFile(partial, os.O_RDWR, 0)
+ if err != nil {
+ return err
+ }
+ if err := h.Sync(); err != nil {
+ h.Close()
+ return err
+ }
+ h.Close()
+ }
+ if err := os.Chmod(partial, os.FileMode(f.Mode)|0o400); err != nil {
+ return err
+ }
+ return os.Rename(partial, final)
+}
+
+// Verify hash-checks every chunk of the artifact's files under dir against its indexes.
+func (e *Engine) Verify(ctx context.Context, m digest.Digest, dir string) error {
+ a, idxs, err := e.Load(m)
+ if err != nil {
+ return err
+ }
+ for i, f := range a.Files {
+ h, err := os.Open(filepath.Join(dir, filepath.FromSlash(f.Path)))
+ if err != nil {
+ return err
+ }
+ kept := make([]bool, len(idxs[i].Chunks))
+ for k := range kept {
+ kept[k] = true
+ }
+ _, bad := e.verifyKept(ctx, h, idxs[i].Chunks, kept)
+ fi, _ := h.Stat()
+ h.Close()
+ if bad > 0 || fi.Size() != f.Size {
+ return fmt.Errorf("%s: %d bytes do not match the index", f.Path, bad)
+ }
+ }
+ return nil
+}
+
+// sharedBytes counts the bytes of the target's chunks that also occur, at any offset, in the base.
+func sharedBytes(base, target []desync.Index) int64 {
+ have := map[desync.ChunkID]bool{}
+ for _, x := range base {
+ for _, c := range x.Chunks {
+ have[c.ID] = true
+ }
+ }
+ var n int64
+ for _, x := range target {
+ for _, c := range x.Chunks {
+ if have[c.ID] {
+ n += int64(c.Size)
+ }
+ }
+ }
+ return n
+}
diff --git a/pkg/casync/casync_test.go b/pkg/casync/casync_test.go
new file mode 100644
index 0000000..cd9330d
--- /dev/null
+++ b/pkg/casync/casync_test.go
@@ -0,0 +1,232 @@
+package casync
+
+import (
+ "bytes"
+ "context"
+ "math/rand"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/ai-computing/onggi/pkg/csi"
+ "github.com/ai-computing/onggi/pkg/store"
+)
+
+// variants returns a base and an aligned sibling (a middle region rewritten in place) and a shifted
+// sibling (a header inserted at the front, so every later byte moves).
+func variants() (base, aligned, shifted []byte) {
+ r := rand.New(rand.NewSource(7))
+ base = make([]byte, 6<<20)
+ r.Read(base)
+ aligned = append([]byte(nil), base...)
+ r.Read(aligned[3<<20 : 3<<20+300<<10])
+ shifted = append(append(make([]byte, 0, len(base)+4096), bytes.Repeat([]byte{9}, 4096)...), base...)
+ return
+}
+
+func publish(t *testing.T, e *Engine, root, name string, data []byte) (dir string) {
+ t.Helper()
+ dir = filepath.Join(root, "src-"+name)
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(dir, "model.bin"), data, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ return dir
+}
+
+func setup(t *testing.T) (*Engine, string) {
+ t.Helper()
+ root := t.TempDir()
+ e, err := New(filepath.Join(root, "store"), filepath.Join(root, "index"), filepath.Join(root, "attest"), 4)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return e, root
+}
+
+func readFile(t *testing.T, p string) []byte {
+ t.Helper()
+ b, err := os.ReadFile(p)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return b
+}
+
+func TestPaths(t *testing.T) {
+ ctx := context.Background()
+ e, root := setup(t)
+ base, aligned, shifted := variants()
+ mA, err := e.Publish(ctx, publish(t, e, root, "a", base))
+ if err != nil {
+ t.Fatal(err)
+ }
+ mB, _ := e.Publish(ctx, publish(t, e, root, "b", aligned))
+ mC, _ := e.Publish(ctx, publish(t, e, root, "c", shifted))
+
+ // Reconstruct: no offer
+ dA := filepath.Join(root, "art", "A")
+ os.MkdirAll(dA, 0o755)
+ st, err := e.Pull(ctx, mA, dA, nil)
+ if err != nil || !bytes.Equal(readFile(t, filepath.Join(dA, "model.bin")), base) || st.Base != "" {
+ t.Fatalf("reconstruct: %v %+v", err, st)
+ }
+
+ // Keep, trusted: A is attested unchanged, so kept chunks are not read back
+ claimed := false
+ offer := &csi.Offer{Bases: []store.Base{{Manifest: mA, Dir: dA}},
+ Claim: func(b store.Base) (string, bool) { claimed = true; return b.Dir, true }}
+ dB := filepath.Join(root, "art", "B")
+ os.MkdirAll(dB, 0o755)
+ st, err = e.Pull(ctx, mB, dB, offer)
+ if err != nil || !claimed || st.Base != mA.String() || st.Verified != 0 || st.Reused < int64(len(aligned))*8/10 {
+ t.Fatalf("keep trusted: %v claimed=%v %+v", err, claimed, st)
+ }
+ if !bytes.Equal(readFile(t, filepath.Join(dB, "model.bin")), aligned) {
+ t.Fatal("keep trusted: output differs")
+ }
+ if _, err := os.Stat(filepath.Join(dA, "model.bin")); !os.IsNotExist(err) {
+ t.Fatal("keep must consume the base file")
+ }
+
+ // Keep, tampered: B is changed after it was written; the attestation no longer matches, so kept
+ // chunks are read back and the corrupted one is fetched again
+ p := filepath.Join(dB, "model.bin")
+ os.Chmod(p, 0o644)
+ f, _ := os.OpenFile(p, os.O_RDWR, 0)
+ f.WriteAt([]byte("corrupt"), 100)
+ f.Close()
+ future := time.Now().Add(time.Second)
+ os.Chtimes(p, future, future)
+ dA2 := filepath.Join(root, "art", "A2")
+ os.MkdirAll(dA2, 0o755)
+ st, err = e.Pull(ctx, mA, dA2, &csi.Offer{Bases: []store.Base{{Manifest: mB, Dir: dB}},
+ Claim: func(b store.Base) (string, bool) { return b.Dir, true }})
+ if err != nil || st.Verified == 0 || st.Repaired == 0 {
+ t.Fatalf("keep tampered: %v %+v", err, st)
+ }
+ if !bytes.Equal(readFile(t, filepath.Join(dA2, "model.bin")), base) {
+ t.Fatal("keep tampered: output differs")
+ }
+
+ // a shifted sibling keeps nothing at the same offset: no claim, assembled from the store
+ claimed = false
+ dC := filepath.Join(root, "art", "C")
+ os.MkdirAll(dC, 0o755)
+ st, err = e.Pull(ctx, mC, dC, &csi.Offer{Bases: []store.Base{{Manifest: mA, Dir: dA2}},
+ Claim: func(b store.Base) (string, bool) { claimed = true; return b.Dir, true }})
+ if err != nil || claimed || st.BaseBest != 0 || !bytes.Equal(readFile(t, filepath.Join(dC, "model.bin")), shifted) {
+ t.Fatalf("shifted: %v claimed=%v %+v", err, claimed, st)
+ }
+
+ // Copy: A2 stays intact and seeds B
+ held := false
+ dB2 := filepath.Join(root, "art", "B2")
+ os.MkdirAll(dB2, 0o755)
+ st, err = e.Pull(ctx, mB, dB2, &csi.Offer{Readable: []store.Base{{Manifest: mA, Dir: dA2}}, BaseCopy: store.BaseCopyAlways,
+ Hold: func(store.Base) (func(), bool) { held = true; return func() {}, true }})
+ if err != nil || !held || st.LocalBase != mA.String() || !bytes.Equal(readFile(t, filepath.Join(dB2, "model.bin")), aligned) {
+ t.Fatalf("copy: %v held=%v %+v", err, held, st)
+ }
+ if !bytes.Equal(readFile(t, filepath.Join(dA2, "model.bin")), base) {
+ t.Fatal("copy must leave the base intact")
+ }
+ if err := e.Verify(ctx, mB, dB2); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestSameOffset(t *testing.T) {
+ e, root := setup(t)
+ base, aligned, _ := variants()
+ ctx := context.Background()
+ mA, _ := e.Publish(ctx, publish(t, e, root, "a", base))
+ mB, _ := e.Publish(ctx, publish(t, e, root, "b", aligned))
+ _, ia, _ := e.Load(mA)
+ _, ib, _ := e.Load(mB)
+ _, same := sameOffset(ia[0].Chunks, ib[0].Chunks)
+ if same < int64(len(aligned))*8/10 || same == int64(len(aligned)) {
+ t.Fatalf("same=%d of %d", same, len(aligned))
+ }
+}
+
+// TestKeepRenamed: variants of different shapes name their files differently; Keep pairs the
+// target file with the base file that holds most of it at the same offset.
+func TestKeepRenamed(t *testing.T) {
+ ctx := context.Background()
+ e, root := setup(t)
+ base, aligned, _ := variants()
+ write := func(name, file string, data []byte) string {
+ d := filepath.Join(root, "src-"+name)
+ os.MkdirAll(d, 0o755)
+ if err := os.WriteFile(filepath.Join(d, file), data, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ return d
+ }
+ mA, err := e.Publish(ctx, write("a", "decode_B2.bin", base))
+ if err != nil {
+ t.Fatal(err)
+ }
+ mB, err := e.Publish(ctx, write("b", "decode_B8.bin", aligned))
+ if err != nil {
+ t.Fatal(err)
+ }
+ dA := filepath.Join(root, "art", "A")
+ os.MkdirAll(dA, 0o755)
+ if _, err := e.Pull(ctx, mA, dA, nil); err != nil {
+ t.Fatal(err)
+ }
+ offer := &csi.Offer{Bases: []store.Base{{Manifest: mA, Dir: dA}},
+ Claim: func(b store.Base) (string, bool) { return b.Dir, true }}
+ dB := filepath.Join(root, "art", "B")
+ os.MkdirAll(dB, 0o755)
+ st, err := e.Pull(ctx, mB, dB, offer)
+ if err != nil || st.Base != mA.String() || st.Verified != 0 || st.Reused < int64(len(aligned))*8/10 {
+ t.Fatalf("keep renamed: %v %+v", err, st)
+ }
+ if !bytes.Equal(readFile(t, filepath.Join(dB, "decode_B8.bin")), aligned) {
+ t.Fatal("keep renamed: output differs")
+ }
+ if _, err := os.Stat(filepath.Join(dA, "decode_B2.bin")); !os.IsNotExist(err) {
+ t.Fatal("keep renamed must consume the base file")
+ }
+}
+
+// TestCopyShiftedFetchesLittle: with a separate chunk source (as a remote store would be), Copy
+// seeds from a sibling whose content sits at other offsets and fetches only what it lacks.
+func TestCopyShiftedFetchesLittle(t *testing.T) {
+ ctx := context.Background()
+ e, root := setup(t)
+ base, _, shifted := variants()
+ mA, _ := e.Publish(ctx, publish(t, e, root, "a", base))
+ mC, _ := e.Publish(ctx, publish(t, e, root, "c", shifted))
+ dA := filepath.Join(root, "art", "A")
+ os.MkdirAll(dA, 0o755)
+ if _, err := e.Pull(ctx, mA, dA, nil); err != nil {
+ t.Fatal(err)
+ }
+ e.Src = e.Store // a distinct chunk source: Copy is chosen with BaseCopyRemote
+ dC := filepath.Join(root, "art", "C")
+ os.MkdirAll(dC, 0o755)
+ st, err := e.Pull(ctx, mC, dC, &csi.Offer{Readable: []store.Base{{Manifest: mA, Dir: dA}}, BaseCopy: store.BaseCopyRemote})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(readFile(t, filepath.Join(dC, "model.bin")), shifted) {
+ t.Fatal("copy shifted: output differs")
+ }
+ if st.FetchedBytes*5 > int64(len(shifted)) || st.LocalBytes == 0 {
+ t.Fatalf("copy shifted fetched %d of %d B (local %d B)", st.FetchedBytes, len(shifted), st.LocalBytes)
+ }
+ // without a sibling the whole target comes from the chunk source
+ dC2 := filepath.Join(root, "art", "C2")
+ os.MkdirAll(dC2, 0o755)
+ st2, err := e.Pull(ctx, mC, dC2, nil)
+ if err != nil || st2.FetchedBytes < int64(len(shifted)) {
+ t.Fatalf("reconstruct fetched %d of %d B: %v", st2.FetchedBytes, len(shifted), err)
+ }
+}
diff --git a/pkg/casync/guards_test.go b/pkg/casync/guards_test.go
new file mode 100644
index 0000000..8e0f31e
--- /dev/null
+++ b/pkg/casync/guards_test.go
@@ -0,0 +1,126 @@
+package casync
+
+import (
+ "bytes"
+ "context"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/ai-computing/onggi/pkg/csi"
+)
+
+// TestKeepRepairsChangedBase: a base that was modified after the node wrote it no longer matches
+// its record, so Keep reads the kept chunks back and replaces the ones that fail their digest.
+func TestKeepRepairsChangedBase(t *testing.T) {
+ ctx := context.Background()
+ e, root := setup(t)
+ base, aligned, _ := variants()
+ mA, _ := e.Publish(ctx, publish(t, e, root, "a", base))
+ mB, _ := e.Publish(ctx, publish(t, e, root, "b", aligned))
+ m := &csi.Materializer{Root: filepath.Join(root, "node"), Pull: e.Pull, RebaseIdle: time.Nanosecond, MaxArtifacts: 1}
+ if err := m.Init(); err != nil {
+ t.Fatal(err)
+ }
+ dA, err := m.Ensure(ctx, mA)
+ if err != nil {
+ t.Fatal(err)
+ }
+ // damage a region that A and B share (B differs from A only around 3 MiB)
+ time.Sleep(20 * time.Millisecond)
+ f, err := os.OpenFile(filepath.Join(dA, "model.bin"), os.O_WRONLY, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := f.WriteAt(bytes.Repeat([]byte{0xEE}, 64<<10), 1<<20); err != nil {
+ t.Fatal(err)
+ }
+ if err := f.Close(); err != nil {
+ t.Fatal(err)
+ }
+ dB, err := m.Ensure(ctx, mB)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(readFile(t, filepath.Join(dB, "model.bin")), aligned) {
+ t.Fatal("Keep carried a changed base into the target")
+ }
+ if _, err := os.Stat(dA); !os.IsNotExist(err) {
+ t.Fatal("the idle victim was not consumed")
+ }
+}
+
+// TestHardLinkedBaseIsNotOverwritten: a base whose file has a second name is never rebuilt in
+// place, so the bytes behind the second name stay those of the base.
+func TestHardLinkedBaseIsNotOverwritten(t *testing.T) {
+ ctx := context.Background()
+ e, root := setup(t)
+ base, aligned, _ := variants()
+ mA, _ := e.Publish(ctx, publish(t, e, root, "a", base))
+ mB, _ := e.Publish(ctx, publish(t, e, root, "b", aligned))
+ m := &csi.Materializer{Root: filepath.Join(root, "node"), Pull: e.Pull, RebaseIdle: time.Nanosecond, MaxArtifacts: 1}
+ if err := m.Init(); err != nil {
+ t.Fatal(err)
+ }
+ dA, err := m.Ensure(ctx, mA)
+ if err != nil {
+ t.Fatal(err)
+ }
+ snap := filepath.Join(m.Root, "snapshot-model.bin")
+ if err := os.Link(filepath.Join(dA, "model.bin"), snap); err != nil {
+ t.Skipf("hard links unavailable here: %v", err)
+ }
+ time.Sleep(20 * time.Millisecond)
+ dB, err := m.Ensure(ctx, mB)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(readFile(t, filepath.Join(dB, "model.bin")), aligned) {
+ t.Fatal("B is wrong")
+ }
+ if !bytes.Equal(readFile(t, snap), base) {
+ t.Fatal("a hard-linked base was rebuilt in place")
+ }
+}
+
+// TestLeasedBaseSurvivesConcurrentPulls: while a volume holds A, several pulls of B run at once;
+// A must stay intact and every pull must return the same correct B.
+func TestLeasedBaseSurvivesConcurrentPulls(t *testing.T) {
+ ctx := context.Background()
+ e, root := setup(t)
+ base, aligned, _ := variants()
+ mA, _ := e.Publish(ctx, publish(t, e, root, "a", base))
+ mB, _ := e.Publish(ctx, publish(t, e, root, "b", aligned))
+ m := &csi.Materializer{Root: filepath.Join(root, "node"), Pull: e.Pull, RebaseIdle: time.Nanosecond, MaxArtifacts: 2}
+ if err := m.Init(); err != nil {
+ t.Fatal(err)
+ }
+ dA, err := m.Ensure(ctx, mA)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := m.Acquire("vol-a", mA, "/pods/a"); err != nil {
+ t.Fatal(err)
+ }
+ errs := make(chan error, 4)
+ dirs := make(chan string, 4)
+ for i := 0; i < 4; i++ {
+ go func() {
+ d, err := m.Ensure(ctx, mB)
+ dirs <- d
+ errs <- err
+ }()
+ }
+ for i := 0; i < 4; i++ {
+ if err := <-errs; err != nil {
+ t.Fatal(err)
+ }
+ if d := <-dirs; !bytes.Equal(readFile(t, filepath.Join(d, "model.bin")), aligned) {
+ t.Fatal("a concurrent pull returned a wrong B")
+ }
+ }
+ if !bytes.Equal(readFile(t, filepath.Join(dA, "model.bin")), base) {
+ t.Fatal("a leased base changed under concurrent pulls")
+ }
+}
diff --git a/pkg/casync/materializer_test.go b/pkg/casync/materializer_test.go
new file mode 100644
index 0000000..67c5159
--- /dev/null
+++ b/pkg/casync/materializer_test.go
@@ -0,0 +1,117 @@
+package casync
+
+import (
+ "bytes"
+ "context"
+ "math/rand"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/ai-computing/onggi/pkg/csi"
+ "github.com/ai-computing/onggi/pkg/store"
+)
+
+// TestOwnership runs the materializer with the casync engine: a sibling a volume uses is never
+// consumed (the new variant is built without destroying it), and once released and idle it is the
+// one consumed, writing only what differs.
+func TestOwnership(t *testing.T) {
+ ctx := context.Background()
+ e, root := setup(t)
+ base, aligned, _ := variants()
+ third := append([]byte(nil), aligned...)
+ rand.New(rand.NewSource(9)).Read(third[1<<20 : 1<<20+200<<10])
+ mA, _ := e.Publish(ctx, publish(t, e, root, "a", base))
+ mB, _ := e.Publish(ctx, publish(t, e, root, "b", aligned))
+ mC, _ := e.Publish(ctx, publish(t, e, root, "c", third))
+
+ m := &csi.Materializer{Root: filepath.Join(root, "node"), Pull: e.Pull, RebaseIdle: time.Nanosecond,
+ MaxArtifacts: 1, BaseCopy: store.BaseCopyAlways}
+ if err := m.Init(); err != nil {
+ t.Fatal(err)
+ }
+ dA, err := m.Ensure(ctx, mA)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := m.Acquire("vol-a", mA, "/pods/a"); err != nil {
+ t.Fatal(err)
+ }
+ // A is in use: B must not consume it
+ dB, err := m.Ensure(ctx, mB)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(readFile(t, filepath.Join(dA, "model.bin")), base) {
+ t.Fatal("a sibling in use was modified")
+ }
+ if !bytes.Equal(readFile(t, filepath.Join(dB, "model.bin")), aligned) {
+ t.Fatal("B is wrong")
+ }
+ // Release marks A used now, so B is the least recently used idle sibling: C consumes B, not A
+ if _, _, _, err := m.Release("vol-a"); err != nil {
+ t.Fatal(err)
+ }
+ time.Sleep(20 * time.Millisecond)
+ dC, err := m.Ensure(ctx, mC)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(readFile(t, filepath.Join(dC, "model.bin")), third) {
+ t.Fatal("C is wrong")
+ }
+ if _, err := os.Stat(dB); !os.IsNotExist(err) {
+ t.Fatal("C did not consume B, the least recently used idle sibling")
+ }
+ if !bytes.Equal(readFile(t, filepath.Join(dA, "model.bin")), base) {
+ t.Fatal("A changed although it was not the victim")
+ }
+ // next, the released A is the least recently used idle sibling and is the one consumed
+ time.Sleep(20 * time.Millisecond)
+ dB2, err := m.Ensure(ctx, mB)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := os.Stat(dA); !os.IsNotExist(err) {
+ t.Fatal("the released sibling A was not consumed when it became the victim")
+ }
+ if !bytes.Equal(readFile(t, filepath.Join(dB2, "model.bin")), aligned) {
+ t.Fatal("B (second time) is wrong")
+ }
+}
+
+// TestVictimNeedsNoIdleWindow: with a capacity, the eviction victim is consumed as soon as no
+// lease holds it, even if it was used a moment ago and the idle window is long.
+func TestVictimNeedsNoIdleWindow(t *testing.T) {
+ ctx := context.Background()
+ e, root := setup(t)
+ base, aligned, _ := variants()
+ mA, _ := e.Publish(ctx, publish(t, e, root, "a", base))
+ mB, _ := e.Publish(ctx, publish(t, e, root, "b", aligned))
+ m := &csi.Materializer{Root: filepath.Join(root, "node"), Pull: e.Pull, RebaseIdle: time.Hour,
+ MaxArtifacts: 1}
+ if err := m.Init(); err != nil {
+ t.Fatal(err)
+ }
+ dA, err := m.Ensure(ctx, mA)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := m.Acquire("vol-a", mA, "/pods/a"); err != nil {
+ t.Fatal(err)
+ }
+ if _, _, _, err := m.Release("vol-a"); err != nil {
+ t.Fatal(err)
+ }
+ dB, err := m.Ensure(ctx, mB)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := os.Stat(dA); !os.IsNotExist(err) {
+ t.Fatal("the just-released victim was not consumed")
+ }
+ if !bytes.Equal(readFile(t, filepath.Join(dB, "model.bin")), aligned) {
+ t.Fatal("B is wrong")
+ }
+}
diff --git a/pkg/csi/candidates_test.go b/pkg/csi/candidates_test.go
new file mode 100644
index 0000000..6cba439
--- /dev/null
+++ b/pkg/csi/candidates_test.go
@@ -0,0 +1,172 @@
+package csi
+
+import (
+ "context"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/ai-computing/onggi/pkg/store"
+)
+
+// An artifact a volume uses is never offered for consumption, but a pull may read it where it is.
+func TestLeasedArtifactIsReadNotConsumed(t *testing.T) {
+ ctx := context.Background()
+ f := newFixture(t)
+ m, c := newMaterializer(t, f)
+ m.RebaseIdle = time.Hour
+ if _, err := m.Ensure(ctx, f.a); err != nil {
+ t.Fatal(err)
+ }
+ if err := m.Acquire("vol-a", f.a, "/target-a"); err != nil {
+ t.Fatal(err)
+ }
+ ageOwnerDonor(t, m, f.a)
+ before, err := os.Stat(filepath.Join(m.Dir(f.a), "model.bin"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ offer, _ := m.offer(f.b)
+ if offer == nil || len(offer.Bases) != 0 || len(offer.Readable) != 1 || offer.Readable[0].Manifest != f.a {
+ t.Fatalf("offer %+v: the leased artifact must be readable and not consumable", offer)
+ }
+ dir, err := m.Ensure(ctx, f.b)
+ if err != nil {
+ t.Fatal(err)
+ }
+ sameTree(t, f.srcB, dir)
+ sameTree(t, f.srcA, m.Dir(f.a))
+ last := c.pulls[len(c.pulls)-1]
+ if last.Base != "" || last.LocalBase != f.a.String() || last.LocalBytes == 0 {
+ t.Fatalf("the pull did not read the leased artifact where it is: %+v", last)
+ }
+ after, err := os.Stat(filepath.Join(m.Dir(f.a), "model.bin"))
+ if err != nil || !os.SameFile(before, after) {
+ t.Fatalf("the leased artifact changed: %v", err)
+ }
+ if n, err := m.users(f.a); err != nil || n != 1 {
+ t.Fatalf("lease lost: %d, %v", n, err)
+ }
+}
+
+// While a pull reads an artifact, another pull cannot take it over and eviction leaves it alone.
+func TestReadHoldBlocksClaimAndEviction(t *testing.T) {
+ ctx := context.Background()
+ f := newFixture(t)
+ m, _ := newMaterializer(t, f)
+ m.RebaseIdle = time.Hour
+ if _, err := m.Ensure(ctx, f.a); err != nil {
+ t.Fatal(err)
+ }
+ ageOwnerDonor(t, m, f.a)
+ offer, claimed := m.offer(f.b)
+ if offer == nil || len(offer.Bases) != 1 || offer.Hold == nil {
+ t.Fatalf("offer %+v", offer)
+ }
+ release, ok := offer.Hold(offer.Readable[0])
+ if !ok {
+ t.Fatal("could not hold a published artifact")
+ }
+ if dir, ok := offer.Claim(offer.Bases[0]); ok || dir != "" || len(*claimed) != 0 {
+ t.Fatal("an artifact that a pull is reading was taken over")
+ }
+ if gone, err := m.Evict(time.Now(), 0, nil); err != nil || len(gone) != 0 {
+ t.Fatalf("an artifact that a pull is reading was evicted: %v, %v", gone, err)
+ }
+ release()
+ ageOwnerDonor(t, m, f.a)
+ if dir, ok := offer.Claim(offer.Bases[0]); !ok || dir == "" {
+ t.Fatal("the artifact could not be taken over after the read ended")
+ }
+}
+
+// With a limit, the cache policy picks the victim: only the least recently used idle artifact is
+// offered for consumption, whatever the others would keep in place. All of them may be read.
+func TestOnlyTheEvictionVictimIsOffered(t *testing.T) {
+ ctx := context.Background()
+ f := newFixture(t)
+ m, _ := newMaterializer(t, f)
+ m.RebaseIdle = time.Hour
+ other := t.TempDir()
+ if err := os.WriteFile(filepath.Join(other, "model.bin"), random(7, 2<<20), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ o, _, err := f.s.Push(ctx, other, store.PushOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, d := range []struct {
+ name string
+ age time.Duration
+ }{{"a", 3 * time.Hour}, {"o", 2 * time.Hour}} {
+ x := f.a
+ if d.name == "o" {
+ x = o
+ }
+ if _, err := m.Ensure(ctx, x); err != nil {
+ t.Fatal(err)
+ }
+ when := time.Now().Add(-d.age)
+ if err := os.Chtimes(m.usedFile(x), when, when); err != nil {
+ t.Fatal(err)
+ }
+ }
+ offer, _ := m.offer(f.b)
+ if offer == nil || len(offer.Bases) != 2 || len(offer.Readable) != 2 {
+ t.Fatalf("without a limit both idle artifacts are offered: %+v", offer)
+ }
+ m.MaxArtifacts = 2
+ offer, _ = m.offer(f.b)
+ if offer == nil || len(offer.Bases) != 1 || offer.Bases[0].Manifest != f.a || len(offer.Readable) != 2 {
+ t.Fatalf("with a limit only the least recently used idle artifact is offered: %+v", offer)
+ }
+ // make the other one the least recently used: it becomes the victim, though it shares nothing with B
+ when := time.Now().Add(-4 * time.Hour)
+ if err := os.Chtimes(m.usedFile(o), when, when); err != nil {
+ t.Fatal(err)
+ }
+ offer, _ = m.offer(f.b)
+ if offer == nil || len(offer.Bases) != 1 || offer.Bases[0].Manifest != o {
+ t.Fatalf("the victim is the least recently used artifact, not the best aligned one: %+v", offer)
+ }
+}
+
+// Leases are files: a new materializer on the same root (a restarted plugin) still refuses to
+// take over an artifact a volume uses.
+func TestLeasesSurviveARestart(t *testing.T) {
+ ctx := context.Background()
+ f := newFixture(t)
+ m, _ := newMaterializer(t, f)
+ if _, err := m.Ensure(ctx, f.a); err != nil {
+ t.Fatal(err)
+ }
+ if err := m.Acquire("vol-a", f.a, "/target-a"); err != nil {
+ t.Fatal(err)
+ }
+ c := &counting{Puller: StorePuller(f.s, f.cache, filepath.Join(m.Root, "tmp"), 4)}
+ again := &Materializer{Root: m.Root, Pull: c.pull, Logf: t.Logf, RebaseIdle: time.Hour}
+ if err := again.Init(); err != nil {
+ t.Fatal(err)
+ }
+ ageOwnerDonor(t, again, f.a)
+ if n, err := again.Users(f.a); err != nil || n != 1 {
+ t.Fatalf("the restarted materializer sees %d leases, %v", n, err)
+ }
+ if offer, _ := again.offer(f.b); offer == nil || len(offer.Bases) != 0 {
+ t.Fatalf("a leased artifact was offered for consumption after a restart: %+v", offer)
+ }
+ dir, err := again.Ensure(ctx, f.b)
+ if err != nil {
+ t.Fatal(err)
+ }
+ sameTree(t, f.srcB, dir)
+ sameTree(t, f.srcA, again.Dir(f.a))
+ if _, _, ok, err := again.Release("vol-a"); err != nil || !ok {
+ t.Fatalf("the lease could not be released after the restart: %v", err)
+ }
+ ageOwnerDonor(t, again, f.a)
+ if offer, _ := again.offer(f.b); offer == nil || len(offer.Bases) != 1 {
+ t.Fatalf("the released artifact is not offered: %+v", offer)
+ }
+}
diff --git a/pkg/csi/capacity_test.go b/pkg/csi/capacity_test.go
new file mode 100644
index 0000000..9839204
--- /dev/null
+++ b/pkg/csi/capacity_test.go
@@ -0,0 +1,143 @@
+package csi
+
+import (
+ "context"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/ai-computing/onggi/pkg/digest"
+ "github.com/ai-computing/onggi/pkg/store"
+)
+
+// Below MaxArtifacts a pull consumes nothing, however idle the other artifact is: the node has
+// room for both. At the limit the idle artifact is taken over, and the node still holds as many
+// artifacts as before.
+func TestOwnerFreeSlotRefusesConsumption(t *testing.T) {
+ ctx := context.Background()
+ f := newFixture(t)
+ m, c := newMaterializer(t, f)
+ m.RebaseIdle = time.Hour
+ m.MaxArtifacts = 2
+ if _, err := m.Ensure(ctx, f.a); err != nil {
+ t.Fatal(err)
+ }
+ ageOwnerDonor(t, m, f.a)
+ dir, err := m.Ensure(ctx, f.b)
+ if err != nil {
+ t.Fatal(err)
+ }
+ sameTree(t, f.srcB, dir)
+ sameTree(t, f.srcA, m.Dir(f.a))
+ if last := c.pulls[len(c.pulls)-1]; last.Base != "" {
+ t.Fatalf("a pull consumed an artifact although a slot was free: %+v", last)
+ }
+ if ds, err := m.Materialized(); err != nil || len(ds) != 2 {
+ t.Fatalf("materialized %v, %v; want both", ds, err)
+ }
+
+ // The node is full now. Drop B and pull it again with the limit at one: A is consumed.
+ if err := os.RemoveAll(m.Dir(f.b)); err != nil {
+ t.Fatal(err)
+ }
+ m.MaxArtifacts = 1
+ ageOwnerDonor(t, m, f.a)
+ if dir, err = m.Ensure(ctx, f.b); err != nil {
+ t.Fatal(err)
+ }
+ sameTree(t, f.srcB, dir)
+ if last := c.pulls[len(c.pulls)-1]; last.Base != f.a.String() {
+ t.Fatalf("a full node did not take the idle artifact over: %+v", last)
+ }
+ if ds, err := m.Materialized(); err != nil || len(ds) != 1 || ds[0] != f.b {
+ t.Fatalf("materialized %v, %v; want only B", ds, err)
+ }
+}
+
+// Evict brings a node back under MaxArtifacts by removing the least recently used unused
+// artifact, and leaves an artifact a volume uses alone even if that keeps the node over.
+func TestEvictAboveMaxArtifacts(t *testing.T) {
+ ctx := context.Background()
+ f := newFixture(t)
+ m, _ := newMaterializer(t, f)
+ if _, err := m.Ensure(ctx, f.a); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := m.Ensure(ctx, f.b); err != nil {
+ t.Fatal(err)
+ }
+ now := time.Now()
+ older, newer := now.Add(-2*time.Minute), now.Add(-time.Minute)
+ if err := os.Chtimes(m.usedFile(f.a), older, older); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Chtimes(m.usedFile(f.b), newer, newer); err != nil {
+ t.Fatal(err)
+ }
+
+ // no limit: nothing is idle for an hour, nothing goes
+ if gone, err := m.Evict(now, time.Hour, nil); err != nil || len(gone) != 0 {
+ t.Fatalf("evicted %v, %v without a limit", gone, err)
+ }
+ // limit one, but the older artifact is in use: the newer, unused one goes instead
+ m.MaxArtifacts = 1
+ if err := m.Acquire("vol-a", f.a, "/target-a"); err != nil {
+ t.Fatal(err)
+ }
+ gone, err := m.Evict(now, time.Hour, nil)
+ if err != nil || len(gone) != 1 || gone[0] != f.b {
+ t.Fatalf("evicted %v, %v; want the unused B", gone, err)
+ }
+ if _, err := os.Stat(filepath.Join(m.Dir(f.a), "model.bin")); err != nil {
+ t.Fatalf("the artifact in use was removed: %v", err)
+ }
+
+ // both unused: the least recently used goes
+ if _, _, _, err := m.Release("vol-a"); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := m.Ensure(ctx, f.b); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Chtimes(m.usedFile(f.a), older, older); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Chtimes(m.usedFile(f.b), newer, newer); err != nil {
+ t.Fatal(err)
+ }
+ gone, err = m.Evict(now, time.Hour, nil)
+ if err != nil || len(gone) != 1 || gone[0] != f.a {
+ t.Fatalf("evicted %v, %v; want the least recently used A", gone, err)
+ }
+}
+
+// The materializer hands its policies to the pull.
+func TestOfferCarriesPolicies(t *testing.T) {
+ ctx := context.Background()
+ f := newFixture(t)
+ m, _ := newMaterializer(t, f)
+ m.RebaseIdle = time.Hour
+ m.VerifyKept, m.BaseCopy = true, store.BaseCopyRemote
+ if _, err := m.Ensure(ctx, f.a); err != nil {
+ t.Fatal(err)
+ }
+ ageOwnerDonor(t, m, f.a)
+ offer, _ := m.offer(f.b)
+ if offer == nil || !offer.VerifyKept || offer.BaseCopy != store.BaseCopyRemote {
+ t.Fatalf("offer %+v does not carry the policies", offer)
+ }
+ var seen store.PullStats
+ inner := m.Pull
+ m.Pull = func(ctx context.Context, d digest.Digest, dest string, o *Offer) (store.PullStats, error) {
+ st, err := inner(ctx, d, dest, o)
+ seen = st
+ return st, err
+ }
+ if _, err := m.Ensure(ctx, f.b); err != nil {
+ t.Fatal(err)
+ }
+ if seen.Base != f.a.String() || seen.Verified <= 0 || seen.Verified != seen.Reused {
+ t.Fatalf("the pull did not verify what it kept: %+v", seen)
+ }
+}
diff --git a/pkg/csi/evict.go b/pkg/csi/evict.go
index 446502a..a3ff98e 100644
--- a/pkg/csi/evict.go
+++ b/pkg/csi/evict.go
@@ -1,6 +1,7 @@
package csi
import (
+ "sort"
"os"
"path/filepath"
"strings"
@@ -61,7 +62,10 @@ func (m *Materializer) materialized() ([]digest.Digest, error) {
return out, nil
}
-// Evict removes the artifacts that no volume uses and that nothing used for at least idle.
+// Evict removes the artifacts that no volume uses and that nothing used for at least idle. With
+// MaxArtifacts set it then removes the least recently used of the remaining unused artifacts
+// until the node holds at most that many; artifacts a volume or a pull uses always stay, so the
+// node can be over the limit while they are all in use.
// unmount is called on the directory first: on Linux it holds the read-only self-bind that
// publishing made. The directory leaves artifacts/ by a rename under the materializer's lock,
// so Ensure either finds it complete or pulls it again; deleting the files happens after.
@@ -74,38 +78,73 @@ func (m *Materializer) Evict(now time.Time, idle time.Duration, unmount func(dir
}
var gone []digest.Digest
var trash []string
- for _, d := range ds {
- if _, busy := m.inflight[d]; busy {
- continue
- }
- if n, err := m.users(d); err != nil || n > 0 {
- continue
- }
- if now.Sub(m.lastUsed(d)) < idle {
- continue
- }
+ over := map[digest.Digest]bool{}
+ // remove takes d out of artifacts/. ok false: it stays (logged); err: stop.
+ remove := func(d digest.Digest) (bool, error) {
dir := m.Dir(d)
if unmount != nil {
if err := unmount(dir); err != nil {
m.logf("evict %s: %v", d, err)
- continue
+ return false, nil
}
}
t, err := os.MkdirTemp(filepath.Join(m.Root, "tmp"), "evict-")
if err != nil {
- m.mu.Unlock()
- return gone, err
+ return false, err
}
dst := filepath.Join(t, d.Hex())
if err := os.Rename(dir, dst); err != nil {
os.Remove(t)
m.logf("evict %s: %v", d, err)
- continue
+ return false, nil
}
os.Remove(m.usedFile(d))
os.Remove(filepath.Join(m.Root, "leases", d.Hex()))
trash = append(trash, t)
gone = append(gone, d)
+ return true, nil
+ }
+ left := 0
+ var spare []digest.Digest // unused, but used too recently for idle
+ for _, d := range ds {
+ if _, busy := m.inflight[d]; busy || m.reading[d] > 0 {
+ left++
+ continue
+ }
+ if n, err := m.users(d); err != nil || n > 0 {
+ left++
+ continue
+ }
+ if now.Sub(m.lastUsed(d)) < idle {
+ left++
+ spare = append(spare, d)
+ continue
+ }
+ ok, err := remove(d)
+ if err != nil {
+ m.mu.Unlock()
+ return gone, err
+ }
+ if !ok {
+ left++
+ }
+ }
+ if m.MaxArtifacts > 0 && left > m.MaxArtifacts {
+ sort.Slice(spare, func(i, j int) bool { return m.lastUsed(spare[i]).Before(m.lastUsed(spare[j])) })
+ for _, d := range spare {
+ if left <= m.MaxArtifacts {
+ break
+ }
+ ok, err := remove(d)
+ if err != nil {
+ m.mu.Unlock()
+ return gone, err
+ }
+ if ok {
+ left--
+ over[d] = true
+ }
+ }
}
m.mu.Unlock()
for _, t := range trash {
@@ -114,7 +153,11 @@ func (m *Materializer) Evict(now time.Time, idle time.Duration, unmount func(dir
}
}
for _, d := range gone {
- m.logf("evicted %s (no volume, idle %s or more)", d, idle)
+ if over[d] {
+ m.logf("evicted %s (no volume, least recently used above the limit of %d)", d, m.MaxArtifacts)
+ } else {
+ m.logf("evicted %s (no volume, idle %s or more)", d, idle)
+ }
}
return gone, nil
}
diff --git a/pkg/csi/links_other.go b/pkg/csi/links_other.go
new file mode 100644
index 0000000..7698814
--- /dev/null
+++ b/pkg/csi/links_other.go
@@ -0,0 +1,6 @@
+//go:build !unix
+
+package csi
+
+// aliased reports no hard links where the platform does not expose link counts.
+func aliased(string) (string, error) { return "", nil }
diff --git a/pkg/csi/links_unix.go b/pkg/csi/links_unix.go
new file mode 100644
index 0000000..3db22e8
--- /dev/null
+++ b/pkg/csi/links_unix.go
@@ -0,0 +1,35 @@
+//go:build unix
+
+package csi
+
+import (
+ "io/fs"
+ "path/filepath"
+ "syscall"
+)
+
+// aliased returns a regular file under root that has more than one hard link, or "" if there is
+// none. A pull that takes a base over changes its files in place; a second link to one of them
+// (a snapshot, a copy made with cp -l) would change with it, outside the artifact and outside
+// every lease.
+func aliased(root string) (string, error) {
+ found := ""
+ err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ if !d.Type().IsRegular() {
+ return nil
+ }
+ fi, err := d.Info()
+ if err != nil {
+ return err
+ }
+ if st, ok := fi.Sys().(*syscall.Stat_t); ok && st.Nlink > 1 {
+ found = p
+ return fs.SkipAll
+ }
+ return nil
+ })
+ return found, err
+}
diff --git a/pkg/csi/links_unix_test.go b/pkg/csi/links_unix_test.go
new file mode 100644
index 0000000..0d83a35
--- /dev/null
+++ b/pkg/csi/links_unix_test.go
@@ -0,0 +1,67 @@
+//go:build unix
+
+package csi
+
+import (
+ "bytes"
+ "context"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+)
+
+// A base whose file has a second hard link is not taken over: changing it in place would change
+// the other name too. The pull then builds the target without the base, and both stay intact.
+func TestOwnerHardLinkedDonorRefusesConsumption(t *testing.T) {
+ ctx := context.Background()
+ f := newFixture(t)
+ m, c := newMaterializer(t, f)
+ m.RebaseIdle = time.Hour
+ if _, err := m.Ensure(ctx, f.a); err != nil {
+ t.Fatal(err)
+ }
+ ageOwnerDonor(t, m, f.a)
+ inside := filepath.Join(m.Dir(f.a), "model.bin")
+ want, err := os.ReadFile(inside)
+ if err != nil {
+ t.Fatal(err)
+ }
+ // The second name lives under the materializer root so that both are on one filesystem.
+ outside := filepath.Join(m.Root, "snapshot-model.bin")
+ if err := os.Link(inside, outside); err != nil {
+ t.Skipf("hard links unavailable here: %v", err)
+ }
+ if p, err := aliased(m.Dir(f.a)); err != nil || p != inside {
+ t.Fatalf("aliased = %q, %v; want %q", p, err, inside)
+ }
+
+ offer, claimed := m.offer(f.b)
+ if offer == nil || len(offer.Bases) != 1 || offer.Bases[0].Manifest != f.a {
+ t.Fatal("expected the idle A donor to be offered")
+ }
+ if dir, ok := offer.Claim(offer.Bases[0]); ok || dir != "" || len(*claimed) != 0 {
+ t.Fatalf("a hard-linked donor was consumed: dir=%q ok=%v", dir, ok)
+ }
+
+ dir, err := m.Ensure(ctx, f.b)
+ if err != nil {
+ t.Fatal(err)
+ }
+ sameTree(t, f.srcB, dir)
+ sameTree(t, f.srcA, m.Dir(f.a))
+ if got, err := os.ReadFile(outside); err != nil || !bytes.Equal(got, want) {
+ t.Fatalf("the second link changed: err=%v", err)
+ }
+ if last := c.pulls[len(c.pulls)-1]; last.Base != "" {
+ t.Fatalf("the pull took over a hard-linked base: %+v", last)
+ }
+
+ // Without the second link the same donor is consumed as before.
+ if err := os.Remove(outside); err != nil {
+ t.Fatal(err)
+ }
+ if p, err := aliased(m.Dir(f.a)); err != nil || p != "" {
+ t.Fatalf("aliased after unlink = %q, %v", p, err)
+ }
+}
diff --git a/pkg/csi/materializer.go b/pkg/csi/materializer.go
index 59b86e9..c5729e8 100644
--- a/pkg/csi/materializer.go
+++ b/pkg/csi/materializer.go
@@ -29,6 +29,13 @@ type Puller func(ctx context.Context, m digest.Digest, dest string, offer *Offer
type Offer struct {
Bases []store.Base
Claim func(store.Base) (dir string, ok bool)
+ // VerifyKept and BaseCopy are passed to the pull (store.PullOptions).
+ VerifyKept bool
+ BaseCopy store.BaseCopy
+ // Readable are artifacts the pull may read where they are, in use or not; Hold keeps one
+ // from being taken over or evicted while it is read.
+ Readable []store.Base
+ Hold func(store.Base) (release func(), ok bool)
}
// StorePuller pulls from s through the node cache (nil: no cache), assembling cache packs in tmp.
@@ -57,6 +64,8 @@ func StorePullerPeers(s *store.Store, openCache func(context.Context) (*store.St
}
if offer != nil {
o.Bases, o.Claim = offer.Bases, offer.Claim
+ o.VerifyKept, o.BaseCopy = offer.VerifyKept, offer.BaseCopy
+ o.ReadBases, o.Hold = offer.Readable, offer.Hold
}
return s.Pull(ctx, m, dest, o)
}
@@ -84,9 +93,22 @@ type Materializer struct {
// MaxPulls bounds distinct artifacts materialized concurrently (0: one).
// Set before the first Ensure; retries for one digest still share one pull.
MaxPulls int
+ // MaxArtifacts is how many artifacts the node keeps (0: no limit). With a limit, a pull takes
+ // an idle artifact over only when the node already holds that many, least recently used
+ // first: the artifact it consumes is the one that would be evicted to make room. Below the
+ // limit nothing is consumed, so a free slot is never filled by destroying a warm artifact.
+ // Evict removes the least recently used idle artifacts above the limit.
+ MaxArtifacts int
+ // VerifyKept makes a pull that takes an artifact over read back and hash-check every chunk it
+ // leaves in place (store.PullOptions.VerifyKept).
+ VerifyKept bool
+ // BaseCopy says when an idle artifact that is not taken over is read where it is
+ // (store.PullOptions.BaseCopy).
+ BaseCopy store.BaseCopy
mu sync.Mutex
inflight map[digest.Digest]*flight
+ reading map[digest.Digest]int // pulls reading an artifact where it is
slots chan struct{}
}
@@ -370,11 +392,14 @@ func (m *Materializer) offer(d digest.Digest) (*Offer, *[]string) {
}
m.mu.Lock()
ds, _ := m.materialized()
- var bases []store.Base
+ var bases, readable []store.Base
inUse, recent := 0, 0
for _, b := range ds {
+ if b == d {
+ continue
+ }
+ readable = append(readable, store.Base{Manifest: b, Dir: m.Dir(b)})
switch {
- case b == d:
case m.idle(b, time.Now()):
bases = append(bases, store.Base{Manifest: b, Dir: m.Dir(b)})
case m.busy(b):
@@ -384,23 +409,69 @@ func (m *Materializer) offer(d digest.Digest) (*Offer, *[]string) {
}
}
m.mu.Unlock()
+ if len(readable) == 0 {
+ return nil, claimed
+ }
if len(bases) == 0 {
- if len(ds) > 0 {
- m.logf("pull of %s: no artifact to take over (%d in use, %d used within %s)", d, inUse, recent, m.RebaseIdle)
+ m.logf("pull of %s: no artifact to take over (%d in use, %d used within %s)", d, inUse, recent, m.RebaseIdle)
+ }
+ // To consume: most recently used first. With a limit, only the least recently used idle
+ // artifact, the one eviction would remove: the cache policy picks the victim, not the pull.
+ sort.Slice(bases, func(i, j int) bool {
+ a, b := m.lastUsed(bases[i].Manifest), m.lastUsed(bases[j].Manifest)
+ if m.MaxArtifacts > 0 {
+ return a.Before(b)
}
- return nil, claimed
+ return a.After(b)
+ })
+ if m.MaxArtifacts > 0 && len(bases) > 1 {
+ bases = bases[:1]
}
- sort.Slice(bases, func(i, j int) bool { return m.lastUsed(bases[i].Manifest).After(m.lastUsed(bases[j].Manifest)) })
if len(bases) > 3 {
bases = bases[:3]
}
+ // To read: most recently used first, in use or not.
+ sort.Slice(readable, func(i, j int) bool {
+ return m.lastUsed(readable[i].Manifest).After(m.lastUsed(readable[j].Manifest))
+ })
+ if len(readable) > 3 {
+ readable = readable[:3]
+ }
+ hold := func(b store.Base) (func(), bool) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ if !isDir(m.Dir(b.Manifest)) {
+ return nil, false
+ }
+ if m.reading == nil {
+ m.reading = map[digest.Digest]int{}
+ }
+ m.reading[b.Manifest]++
+ return func() {
+ m.mu.Lock()
+ if m.reading[b.Manifest]--; m.reading[b.Manifest] <= 0 {
+ delete(m.reading, b.Manifest)
+ }
+ m.mu.Unlock()
+ }, true
+ }
claim := func(b store.Base) (string, bool) {
m.mu.Lock()
defer m.mu.Unlock()
if !m.idle(b.Manifest, time.Now()) {
return "", false
}
+ if m.MaxArtifacts > 0 {
+ if ds, err := m.materialized(); err != nil || len(ds) < m.MaxArtifacts {
+ return "", false // a free slot: consume nothing
+ }
+ }
dir := m.Dir(b.Manifest)
+ // A file with a second hard link would change outside the artifact too; leave it alone.
+ if p, err := aliased(dir); err != nil || p != "" {
+ m.logf("rebase from %s refused: %q has another hard link (err %v)", b.Manifest, p, err)
+ return "", false
+ }
if m.Unmount != nil {
if err := m.Unmount(dir); err != nil {
m.logf("rebase from %s: %v", b.Manifest, err)
@@ -422,21 +493,27 @@ func (m *Materializer) offer(d digest.Digest) (*Offer, *[]string) {
*claimed = append(*claimed, t)
return dst, true
}
- return &Offer{Bases: bases, Claim: claim}, claimed
+ return &Offer{Bases: bases, Claim: claim, VerifyKept: m.VerifyKept, BaseCopy: m.BaseCopy,
+ Readable: readable, Hold: hold}, claimed
}
-// idle says whether artifact d may be consumed: materialized, no volume, no pull, unused for
-// RebaseIdle. The caller holds m.mu.
+// idle says whether artifact d may be consumed: materialized, no volume, no pull, and, on a node
+// without a capacity, unused for RebaseIdle. With a capacity the victim is the artifact eviction
+// would delete anyway, so waiting would only lose its bytes: no lease is all it needs.
+// The caller holds m.mu.
func (m *Materializer) idle(d digest.Digest, now time.Time) bool {
if m.busy(d) || !isDir(m.Dir(d)) {
return false
}
+ if m.MaxArtifacts > 0 {
+ return true
+ }
return now.Sub(m.lastUsed(d)) >= m.RebaseIdle
}
// busy says whether a volume or a pull uses artifact d. The caller holds m.mu.
func (m *Materializer) busy(d digest.Digest) bool {
- if _, ok := m.inflight[d]; ok {
+ if _, ok := m.inflight[d]; ok || m.reading[d] > 0 {
return true
}
n, err := m.users(d)
diff --git a/pkg/csi/owner_protocol_test.go b/pkg/csi/owner_protocol_test.go
new file mode 100644
index 0000000..97cc2e5
--- /dev/null
+++ b/pkg/csi/owner_protocol_test.go
@@ -0,0 +1,261 @@
+package csi
+
+import (
+ "context"
+ "errors"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/ai-computing/onggi/pkg/digest"
+ "github.com/ai-computing/onggi/pkg/store"
+)
+
+func ageOwnerDonor(t *testing.T, m *Materializer, d digest.Digest) {
+ t.Helper()
+ old := time.Unix(1, 0)
+ if err := os.Chtimes(m.usedFile(d), old, old); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// A stale offer cannot override a lease acquired before its actual claim.
+func TestOwnerLeaseAfterOfferRefusesConsumption(t *testing.T) {
+ ctx := context.Background()
+ f := newFixture(t)
+ m, c := newMaterializer(t, f)
+ m.RebaseIdle = time.Hour
+ if _, err := m.Ensure(ctx, f.a); err != nil {
+ t.Fatal(err)
+ }
+ ageOwnerDonor(t, m, f.a)
+ before, err := os.Stat(filepath.Join(m.Dir(f.a), "model.bin"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ offer, claimed := m.offer(f.b)
+ if offer == nil || len(offer.Bases) != 1 || offer.Bases[0].Manifest != f.a {
+ t.Fatal("expected the idle A donor to be offered")
+ }
+ if err := m.Acquire("late-owner-lease", f.a, "/late-owner-target"); err != nil {
+ t.Fatal(err)
+ }
+ // Keep age eligible so refusal is due to the lease rather than recency.
+ ageOwnerDonor(t, m, f.a)
+ if dir, ok := offer.Claim(offer.Bases[0]); ok || dir != "" || len(*claimed) != 0 {
+ t.Fatalf("stale offer consumed a leased donor: dir=%q ok=%v", dir, ok)
+ }
+ if users, err := m.users(f.a); err != nil || users != 1 {
+ t.Fatalf("lease lost: users=%d err=%v", users, err)
+ }
+ dir, err := m.Ensure(ctx, f.b)
+ if err != nil {
+ t.Fatal(err)
+ }
+ sameTree(t, f.srcB, dir)
+ sameTree(t, f.srcA, m.Dir(f.a))
+ after, err := os.Stat(filepath.Join(m.Dir(f.a), "model.bin"))
+ if err != nil || !os.SameFile(before, after) || c.pulls[1].Base != "" {
+ t.Fatalf("leased donor changed or was consumed: err=%v stats=%+v", err, c.pulls[1])
+ }
+ t.Log("offer preceded lease; saved claim refused; lease/inode/base bytes retained; separate target verified")
+}
+
+// Two distinct target offers contend on the real owner claim, not a stub callback.
+func TestOwnerCompetingDonorClaims(t *testing.T) {
+ ctx := context.Background()
+ f := newFixture(t)
+ m, _ := newMaterializer(t, f)
+ m.RebaseIdle = time.Hour
+ m.MaxPulls = 2
+ if _, err := m.Ensure(ctx, f.a); err != nil {
+ t.Fatal(err)
+ }
+ ageOwnerDonor(t, m, f.a)
+ otherSource := t.TempDir()
+ shared, err := os.ReadFile(filepath.Join(f.srcA, "model.bin"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(otherSource, "model.bin"), append(shared, random(3, 100_000)...), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ other, _, err := f.s.Push(ctx, otherSource, store.PushOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ first, claimedFirst := m.offer(f.b)
+ second, claimedSecond := m.offer(other)
+ for _, offer := range []*Offer{first, second} {
+ if offer == nil || len(offer.Bases) != 1 || offer.Bases[0].Manifest != f.a {
+ t.Fatal("both distinct target offers must name A")
+ }
+ }
+ before, err := os.Stat(filepath.Join(m.Dir(f.a), "model.bin"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ type result struct {
+ dir string
+ ok bool
+ }
+ start := make(chan struct{})
+ results := make(chan result, 2)
+ for _, offer := range []*Offer{first, second} {
+ go func(o *Offer) {
+ <-start
+ dir, ok := o.Claim(o.Bases[0])
+ results <- result{dir: dir, ok: ok}
+ }(offer)
+ }
+ close(start)
+ winner := ""
+ wins := 0
+ for range 2 {
+ select {
+ case r := <-results:
+ if r.ok {
+ wins++
+ winner = r.dir
+ } else if r.dir != "" {
+ t.Fatalf("refused claim returned a directory: %q", r.dir)
+ }
+ case <-time.After(10 * time.Second):
+ t.Fatal("competing claims did not return")
+ }
+ }
+ if wins != 1 || len(*claimedFirst)+len(*claimedSecond) != 1 || isDir(m.Dir(f.a)) {
+ t.Fatalf("donor ownership moved incorrectly: wins=%d first=%v second=%v", wins, *claimedFirst, *claimedSecond)
+ }
+ sameTree(t, f.srcA, winner)
+ after, err := os.Stat(filepath.Join(winner, "model.bin"))
+ if err != nil || !os.SameFile(before, after) {
+ t.Fatalf("winner did not receive the original donor inode: %v", err)
+ }
+ for _, offer := range []*Offer{first, second} {
+ if _, ok := offer.Claim(offer.Bases[0]); ok {
+ t.Fatal("a later claim took the already-transferred donor")
+ }
+ }
+ sameTree(t, f.srcA, winner)
+ t.Log("two real claims released together; exactly one move; winner bytes/inode intact after repeated refusal")
+}
+
+func ownerPackPaths(t *testing.T, root string) map[string]bool {
+ t.Helper()
+ paths := make(map[string]bool)
+ err := filepath.WalkDir(filepath.Join(root, "packs"), func(path string, entry fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ if entry.Type().IsRegular() {
+ paths[path] = true
+ }
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ return paths
+}
+
+// Actual residual corruption follows a successful real claim; retry has no donor.
+func TestOwnerPostConsumptionFailureThenRetry(t *testing.T) {
+ ctx := context.Background()
+ storeDir := t.TempDir()
+ f := &fixture{s: openStore(t, storeDir), srcA: t.TempDir(), srcB: t.TempDir()}
+ shared := random(71, 3<<20)
+ if err := os.WriteFile(filepath.Join(f.srcA, "model.bin"), shared, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(f.srcB, "model.bin"), append(shared, random(72, 100_000)...), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ var err error
+ f.a, _, err = f.s.Push(ctx, f.srcA, store.PushOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ basePacks := ownerPackPaths(t, storeDir)
+ f.b, _, err = f.s.Push(ctx, f.srcB, store.PushOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ var residual []string
+ for path := range ownerPackPaths(t, storeDir) {
+ if !basePacks[path] {
+ residual = append(residual, path)
+ }
+ }
+ if len(residual) != 1 {
+ t.Fatalf("expected one target-only residual pack, got %d", len(residual))
+ }
+ pack := residual[0]
+ original, err := os.ReadFile(pack)
+ if err != nil || len(original) == 0 {
+ t.Fatalf("read residual pack: %v", err)
+ }
+ m, _ := newMaterializer(t, f)
+ m.RebaseIdle = time.Hour
+ if _, err := m.Ensure(ctx, f.a); err != nil {
+ t.Fatal(err)
+ }
+ ageOwnerDonor(t, m, f.a)
+ // No cache/peer can mask the corrupted target-only store bytes.
+ realPull := StorePuller(f.s, nil, filepath.Join(m.Root, "tmp"), 2)
+ claimed := false
+ var injectionErr, pullErr error
+ var failedStats store.PullStats
+ m.Pull = func(ctx context.Context, target digest.Digest, dest string, offer *Offer) (store.PullStats, error) {
+ if offer != nil {
+ realClaim := offer.Claim
+ offer.Claim = func(base store.Base) (string, bool) {
+ dir, ok := realClaim(base)
+ if ok {
+ claimed = true
+ corrupt := append([]byte(nil), original...)
+ corrupt[len(corrupt)/2] ^= 0xff
+ injectionErr = os.Chmod(pack, 0o644)
+ if injectionErr == nil {
+ injectionErr = os.WriteFile(pack, corrupt, 0o644)
+ }
+ }
+ return dir, ok
+ }
+ }
+ failedStats, pullErr = realPull(ctx, target, dest, offer)
+ return failedStats, pullErr
+ }
+ _, err = m.Ensure(ctx, f.b)
+ if !claimed || injectionErr != nil || pullErr == nil || !errors.Is(err, pullErr) {
+ t.Fatalf("expected actual post-claim construction failure: claimed=%v injection=%v pull=%v ensure=%v", claimed, injectionErr, pullErr, err)
+ }
+ if !strings.Contains(pullErr.Error(), "fails its hash check") || !strings.Contains(pullErr.Error(), filepath.Base(pack)) {
+ t.Fatalf("expected the selected corrupted residual pack's hash failure: pack=%s err=%v", filepath.Base(pack), pullErr)
+ }
+ if failedStats.Base != f.a.String() || failedStats.Reused == 0 {
+ t.Fatalf("failure preceded genuine consumption/preservation: %+v", failedStats)
+ }
+ if isDir(m.Dir(f.a)) || isDir(m.Dir(f.b)) {
+ t.Fatal("consumed donor or incomplete target remained published")
+ }
+ if entries, err := os.ReadDir(filepath.Join(m.Root, "tmp")); err != nil || len(entries) != 0 {
+ t.Fatalf("private/consumed cleanup failed: entries=%v err=%v", entries, err)
+ }
+ if err := os.WriteFile(pack, original, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ m.Pull = realPull
+ dir, err := m.Ensure(ctx, f.b)
+ if err != nil {
+ t.Fatal(err)
+ }
+ sameTree(t, f.srcB, dir)
+ if isDir(m.Dir(f.a)) {
+ t.Fatal("retry manufactured a replacement donor")
+ }
+ t.Logf("real consumed-base residual failure=%v reused=%d; no completed target; donor-free retry bytes verified", pullErr, failedStats.Reused)
+}
diff --git a/pkg/store/pull.go b/pkg/store/pull.go
index a90931d..fc05c13 100644
--- a/pkg/store/pull.go
+++ b/pkg/store/pull.go
@@ -37,8 +37,36 @@ type PullOptions struct {
Bases []Base
Claim func(Base) (dir string, ok bool)
MinSame float64
+ // VerifyKept makes a pull that takes a base over read back every chunk it leaves in place and
+ // check its hash, so nothing reaches the new artifact unchecked. A chunk that fails (the base
+ // was damaged after it was pulled) is fetched like any other.
+ VerifyKept bool
+ // BaseCopy says when a base that is not taken over is read where it is (#58).
+ BaseCopy BaseCopy
+ // ReadBases are artifacts on the node the pull may read where they are but never consumes
+ // (nil: Bases). They may be in use. Hold, if set, is called before one is read and keeps it
+ // from being taken over or evicted until the returned release is called (ok false: gone).
+ ReadBases []Base
+ Hold func(Base) (release func(), ok bool)
+ // BaseRange is the largest single read from a base's files (0: MaxRange). Chunks that follow
+ // one another are read together up to this; 1 reads chunk by chunk.
+ BaseRange int64
}
+// BaseCopy is the rule for reading a base that stays intact.
+type BaseCopy int
+
+const (
+ // BaseCopyAlways reads the base whenever it holds MinSame of the artifact anywhere.
+ BaseCopyAlways BaseCopy = iota
+ // BaseCopyRemote reads the base only if the store is not local. Packs on this machine are
+ // read in large ranges and decoded faster than a base is read and hashed chunk by chunk:
+ // on 15-18 GB artifacts a local store took 32-35 s where the base took 44-47 s.
+ BaseCopyRemote
+ // BaseCopyNever never reads a base that is not taken over.
+ BaseCopyNever
+)
+
// PullStats reports a pull.
type PullStats struct {
Files int `json:"files"`
@@ -50,10 +78,14 @@ type PullStats struct {
Cached int64 `json:"cached"` // stored bytes added to the node cache
Base string `json:"base,omitempty"` // the artifact this pull turned into the new one
Reused int64 `json:"reused"` // bytes left in place in the base's files
+ Verified int64 `json:"verified"` // of those, bytes read back and hash-checked (VerifyKept)
+ Repaired int64 `json:"repaired"` // bytes of the base that failed that check and were fetched instead
BaseBest int64 `json:"base_best"` // bytes the best offered base keeps at the same offsets, taken or not (#49)
LocalBase string `json:"local_base,omitempty"` // a base read where it is, not consumed (#58)
LocalBytes int64 `json:"local_bytes"` // bytes read from that base's files
Ranges int `json:"ranges"`
+ FetchedChunks int64 `json:"fetched_chunks"` // chunks taken from the chunk source (casync engine)
+ FetchedBytes int64 `json:"fetched_bytes"` // their logical size
Duration time.Duration `json:"duration_ns"`
}
@@ -104,6 +136,9 @@ func (s *Store) Pull(ctx context.Context, manifest digest.Digest, dest string, o
if o.MaxRange <= 0 {
o.MaxRange = 16 << 20
}
+ if o.BaseRange <= 0 {
+ o.BaseRange = o.MaxRange
+ }
m, err := s.ReadManifest(ctx, manifest)
if err != nil {
return PullStats{}, err
@@ -143,28 +178,50 @@ func (s *Store) Pull(ctx context.Context, manifest digest.Digest, dest string, o
return st, err
}
done := make([]bool, len(nodes)) // in place from a base, or written from peers
- if len(o.Bases) > 0 && o.Claim != nil {
+ readBases := o.ReadBases
+ if readBases == nil {
+ readBases = o.Bases
+ }
+ if len(o.Bases) > 0 && o.Claim != nil || len(readBases) > 0 {
minSame := o.MinSame
if minSame <= 0 {
minSame = DefaultMinSame
}
- p, ok, most, okMost := s.chooseBase(ctx, m, extentsOf(nodes, len(m.Files), false), o.Bases, minSame)
- st.BaseBest = p.same
+ newExt := extentsOf(nodes, len(m.Files), false)
claimed := false
- if ok {
- if dir, ok := o.Claim(p.base); ok {
- claimed = true
- st.Base = p.base.Manifest.String()
- if st.Reused, err = rebase(p, dir, m, nodes, handles, done); err != nil {
- return st, err
+ if len(o.Bases) > 0 && o.Claim != nil {
+ p, ok, _, _ := s.chooseBase(ctx, m, newExt, o.Bases, minSame)
+ st.BaseBest = p.same
+ if ok {
+ if dir, ok := o.Claim(p.base); ok {
+ claimed = true
+ st.Base = p.base.Manifest.String()
+ if st.Reused, err = rebase(p, dir, m, nodes, handles, done); err != nil {
+ return st, err
+ }
+ if o.VerifyKept {
+ st.Verified, st.Repaired = verifyKept(ctx, nodes, handles, done, o.Workers, o.BaseRange)
+ st.Reused -= st.Repaired
+ }
}
}
}
- // too little in place, but the base holds the bytes elsewhere: read them from it (#58)
- if !claimed && okMost {
- st.LocalBase = most.base.Manifest.String()
- if st.LocalBytes, err = fromBase(ctx, most, nodes, done, handles, o.Workers); err != nil {
- return st, err
+ // nothing taken over, but an artifact on the node holds the bytes: read them from it (#58)
+ copyBase := o.BaseCopy == BaseCopyAlways || o.BaseCopy == BaseCopyRemote && !s.B.Caps().Local
+ if !claimed && copyBase && len(readBases) > 0 {
+ if _, _, most, okMost := s.chooseBase(ctx, m, newExt, readBases, minSame); okMost {
+ release, held := func() {}, true
+ if o.Hold != nil {
+ release, held = o.Hold(most.base)
+ }
+ if held {
+ st.LocalBase = most.base.Manifest.String()
+ st.LocalBytes, err = fromBase(ctx, most, nodes, done, handles, o.Workers, o.BaseRange)
+ release()
+ if err != nil {
+ return st, err
+ }
+ }
}
}
}
diff --git a/pkg/store/push.go b/pkg/store/push.go
index 4620e11..c564a2a 100644
--- a/pkg/store/push.go
+++ b/pkg/store/push.go
@@ -408,7 +408,7 @@ func (s *Store) chunkFile(ctx context.Context, path string, twoLevel bool, mode
var list []ChunkRef
var nb int64
var fb *frameBuilder // new chunks of a sub-chunked file go out in compression frames (#52)
- if twoLevel && s.Config.Frame > 0 {
+ if s.Config.Frame > 0 && (twoLevel || s.Config.Sub == nil) {
fb = &frameBuilder{limit: int(s.Config.Frame), mode: mode}
}
for {
@@ -430,6 +430,14 @@ func (s *Store) chunkFile(ctx context.Context, path string, twoLevel bool, mode
continue
}
if !twoLevel {
+ if fb != nil { // a framed one-level store (Create.FrameFlat)
+ n, err := fb.add(ctx, ch, d, out)
+ if err != nil {
+ return nil, 0, err
+ }
+ nb += n
+ continue
+ }
n, err := send(ctx, ch, d, mode, out)
if err != nil {
return nil, 0, err
diff --git a/pkg/store/rebase.go b/pkg/store/rebase.go
index 76ab229..1b945d8 100644
--- a/pkg/store/rebase.go
+++ b/pkg/store/rebase.go
@@ -110,32 +110,22 @@ func (s *Store) chooseBase(ctx context.Context, m *Manifest, newExt [][]Extent,
}
// fromBase reads, from the base's files where they are, the stored chunks a pull still needs
-// that the base holds anywhere (#58). The base is not consumed. Each chunk is hash-checked; one
-// that cannot be read or does not match is left for the other sources. It returns the bytes read.
-func fromBase(ctx context.Context, p rebasePlan, nodes []node, done []bool, handles []*os.File, workers int) (int64, error) {
- type job struct {
- node int32
- file int
- off int64
- }
- var jobs []job
+// that the base holds anywhere (#58). The base is not consumed. Chunks that follow one another in
+// a base file are read together. Each chunk is hash-checked; one that cannot be read or does not
+// match is left for the other sources. It returns the bytes read.
+func fromBase(ctx context.Context, p rebasePlan, nodes []node, done []bool, handles []*os.File, workers int, maxRange int64) (int64, error) {
+ var jobs []rjob
for i := range nodes {
if !nodes[i].stored || done[i] {
continue
}
if f, e, ok := p.find(nodes[i].chunk); ok && e.Len == nodes[i].loc.Length {
- jobs = append(jobs, job{int32(i), f, e.Off})
+ jobs = append(jobs, rjob{int32(i), f, e.Off, int(e.Len)})
}
}
if len(jobs) == 0 {
return 0, nil
}
- slices.SortFunc(jobs, func(a, b job) int { // read each base file front to back
- if a.file != b.file {
- return a.file - b.file
- }
- return cmp.Compare(a.off, b.off)
- })
files := make([]*os.File, len(p.files))
defer func() {
for _, f := range files {
@@ -156,42 +146,53 @@ func fromBase(ctx context.Context, p rebasePlan, nodes []node, done []bool, hand
var n atomic.Int64
var mu sync.Mutex
var firstErr error
- var wg sync.WaitGroup
- w := max(workers, 1)
- per := (len(jobs) + w - 1) / w
- for a := 0; a < len(jobs); a += per {
- part := jobs[a:min(a+per, len(jobs))]
- wg.Add(1)
- go func() {
- defer wg.Done()
- var buf []byte
- for _, j := range part {
- if ctx.Err() != nil {
- return
- }
- nd := &nodes[j.node]
- if cap(buf) < int(nd.loc.Length) {
- buf = make([]byte, nd.loc.Length)
- }
- b := buf[:nd.loc.Length]
- if _, err := files[j.file].ReadAt(b, j.off); err != nil || digest.Of(b) != nd.chunk {
- continue
- }
- if err := emit(nodes, j.node, b, 0, handles); err != nil {
- mu.Lock()
- firstErr = cmp.Or(firstErr, err)
- mu.Unlock()
- return
- }
- done[j.node] = true
- n.Add(int64(len(b)))
- }
- }()
- }
- wg.Wait()
+ readSpans(ctx, files, jobs, workers, maxRange, func(j rjob, b []byte) {
+ if b == nil || digest.Of(b) != nodes[j.node].chunk {
+ return
+ }
+ if err := emit(nodes, j.node, b, 0, handles); err != nil {
+ mu.Lock()
+ firstErr = cmp.Or(firstErr, err)
+ mu.Unlock()
+ return
+ }
+ done[j.node] = true
+ n.Add(int64(len(b)))
+ })
return n.Load(), firstErr
}
+// verifyKept reads back, from the files of the new artifact, every chunk a taken-over base left
+// in place, and checks its hash. A chunk that fails in any of its places is no longer done, so
+// the pull fetches it and writes it everywhere it goes. It returns the bytes that passed and the
+// bytes that did not.
+func verifyKept(ctx context.Context, nodes []node, handles []*os.File, done []bool, workers int, maxRange int64) (ok, bad int64) {
+ var jobs []rjob
+ for i := range nodes {
+ if !nodes[i].stored || !done[i] {
+ continue
+ }
+ places(nodes, int32(i), 0, func(t target) {
+ jobs = append(jobs, rjob{int32(i), int(t.file), t.off, int(nodes[i].loc.Length)})
+ })
+ }
+ failed := make([]atomic.Bool, len(nodes))
+ readSpans(ctx, handles, jobs, workers, maxRange, func(j rjob, b []byte) {
+ if b == nil || digest.Of(b) != nodes[j.node].chunk {
+ failed[j.node].Store(true)
+ }
+ })
+ for _, j := range jobs {
+ if failed[j.node].Load() {
+ done[j.node] = false
+ bad += int64(j.n)
+ } else {
+ ok += int64(j.n)
+ }
+ }
+ return ok, bad
+}
+
// sameAt says whether base holds e's chunk at e's offset.
func sameAt(base []Extent, e Extent) bool {
k, ok := slices.BinarySearchFunc(base, e.Off, func(x Extent, off int64) int {
diff --git a/pkg/store/selector_test.go b/pkg/store/selector_test.go
new file mode 100644
index 0000000..c51ea5f
--- /dev/null
+++ b/pkg/store/selector_test.go
@@ -0,0 +1,255 @@
+package store
+
+import (
+ "context"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/ai-computing/onggi/pkg/backend"
+ bfs "github.com/ai-computing/onggi/pkg/backend/fs"
+ "github.com/ai-computing/onggi/pkg/chunker"
+ "github.com/ai-computing/onggi/pkg/codec"
+)
+
+// A base damaged after it was pulled is not carried into the new artifact: with VerifyKept the
+// chunk that no longer matches is fetched, and only the rest is left in place.
+func TestRebaseVerifiesWhatItKeeps(t *testing.T) {
+ ctx := context.Background()
+ s := openTwoLevel(t, t.TempDir())
+ w := random(41, 8<<20)
+ v := append([]byte(nil), w...)
+ copy(v[3<<20:], random(42, 200<<10))
+ b, nm, src := rebaseFixture(t, s,
+ map[string][]byte{"decode_B2.rbln": w},
+ map[string][]byte{"decode_B8.rbln": v})
+
+ // flip bytes of the base in a region both artifacts share
+ f, err := os.OpenFile(filepath.Join(b.Dir, "decode_B2.rbln"), os.O_RDWR, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := f.WriteAt([]byte("damaged after the pull"), 6<<20); err != nil {
+ t.Fatal(err)
+ }
+ f.Close()
+
+ claim := func(x Base) (string, bool) { return x.Dir, true }
+ out := t.TempDir()
+ st, err := s.Pull(ctx, nm, out, PullOptions{Bases: []Base{b}, Claim: claim, VerifyKept: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ sameTree(t, src, out)
+ t.Logf("%+v", st)
+ if st.Base != b.Manifest.String() || st.Repaired <= 0 || st.Repaired > 1<<20 || st.Verified < int64(len(v))*8/10 {
+ t.Fatalf("damaged chunk not repaired, or too much distrusted: %+v", st)
+ }
+ if st.Reused+st.Repaired > int64(len(v)) || st.Reused != st.Verified {
+ t.Fatalf("kept bytes and verified bytes disagree: %+v", st)
+ }
+}
+
+// Without VerifyKept the pull reads nothing back, as before.
+func TestRebaseDoesNotVerifyByDefault(t *testing.T) {
+ ctx := context.Background()
+ s := openTwoLevel(t, t.TempDir())
+ w := random(43, 4<<20)
+ v := append([]byte(nil), w...)
+ copy(v[1<<20:], random(44, 100<<10))
+ b, nm, src := rebaseFixture(t, s, map[string][]byte{"a.rbln": w}, map[string][]byte{"a.rbln": v})
+ out := t.TempDir()
+ st, err := s.Pull(ctx, nm, out, PullOptions{Bases: []Base{b}, Claim: func(x Base) (string, bool) { return x.Dir, true }})
+ if err != nil {
+ t.Fatal(err)
+ }
+ sameTree(t, src, out)
+ if st.Verified != 0 || st.Repaired != 0 || st.Reused < int64(len(v))*8/10 {
+ t.Fatalf("%+v", st)
+ }
+}
+
+// With a local store, BaseCopyRemote leaves a shifted base alone and reads the packs; the same
+// pull under BaseCopyAlways reads the base. Both produce the artifact.
+func TestBaseCopyRemoteSkipsALocalStore(t *testing.T) {
+ ctx := context.Background()
+ s := openTwoLevel(t, t.TempDir())
+ if !s.B.Caps().Local {
+ t.Fatal("the fs backend must report itself local")
+ }
+ w := random(93, 8<<20)
+ v := append(random(94, 17952), w...)
+ b, nm, src := rebaseFixture(t, s, map[string][]byte{"rank0.engine": w}, map[string][]byte{"rank0.engine": v})
+ never := func(Base) (string, bool) {
+ t.Fatal("consumed a base that keeps nothing in place")
+ return "", false
+ }
+ for _, c := range []struct {
+ name string
+ rule BaseCopy
+ base bool
+ }{{"remote", BaseCopyRemote, false}, {"never", BaseCopyNever, false}, {"always", BaseCopyAlways, true}} {
+ out := t.TempDir()
+ st, err := s.Pull(ctx, nm, out, PullOptions{Bases: []Base{b}, Claim: never, BaseCopy: c.rule})
+ if err != nil {
+ t.Fatal(c.name, err)
+ }
+ sameTree(t, src, out)
+ if got := st.LocalBase != ""; got != c.base {
+ t.Fatalf("%s: read the base = %v, want %v: %+v", c.name, got, c.base, st)
+ }
+ if !c.base && st.ReadBytes == 0 {
+ t.Fatalf("%s: nothing read from the store: %+v", c.name, st)
+ }
+ }
+ if _, err := os.Stat(filepath.Join(b.Dir, "rank0.engine")); err != nil {
+ t.Fatalf("the base was touched: %v", err)
+ }
+}
+
+// Chunks that follow one another in a file are read together, and each still reaches fn alone.
+func TestReadSpansJoinsNeighbours(t *testing.T) {
+ data := random(95, 1<<20)
+ p := filepath.Join(t.TempDir(), "f")
+ write(t, p, data)
+ f, err := os.Open(p)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer f.Close()
+ var jobs []rjob
+ for off := 0; off < len(data); off += 4096 { // contiguous, then a hole after each 64 KiB
+ if off/65536%2 == 1 {
+ continue
+ }
+ jobs = append(jobs, rjob{node: int32(off / 4096), off: int64(off), n: 4096})
+ }
+ jobs = append(jobs, rjob{node: -1, off: int64(len(data)) - 100, n: 4096}) // runs past the end
+ got := make(map[int32]bool)
+ bad := 0
+ done := make(chan struct{}, 1)
+ done <- struct{}{}
+ readSpans(context.Background(), []*os.File{f}, jobs, 4, 32<<10, func(j rjob, b []byte) {
+ <-done
+ defer func() { done <- struct{}{} }()
+ if b == nil {
+ bad++
+ return
+ }
+ if string(b) != string(data[j.off:j.off+int64(j.n)]) {
+ t.Errorf("chunk at %d has the wrong bytes", j.off)
+ }
+ got[j.node] = true
+ })
+ if bad != 1 || len(got) != len(jobs)-1 {
+ t.Fatalf("read %d of %d chunks, %d failed", len(got), len(jobs), bad)
+ }
+}
+
+// Reading a base chunk by chunk and in joined ranges gives the same artifact and the same bytes.
+func TestBaseRangeDoesNotChangeTheResult(t *testing.T) {
+ ctx := context.Background()
+ s := openTwoLevel(t, t.TempDir())
+ w := random(96, 8<<20)
+ v := append(random(97, 17952), w...)
+ b, nm, src := rebaseFixture(t, s, map[string][]byte{"rank0.engine": w}, map[string][]byte{"rank0.engine": v})
+ never := func(Base) (string, bool) { return "", false }
+ var local [2]int64
+ for i, r := range []int64{1, 0} {
+ out := t.TempDir()
+ st, err := s.Pull(ctx, nm, out, PullOptions{Bases: []Base{b}, Claim: never, BaseRange: r})
+ if err != nil {
+ t.Fatal(err)
+ }
+ sameTree(t, src, out)
+ local[i] = st.LocalBytes
+ }
+ if local[0] != local[1] || local[0] < int64(len(w))*9/10 {
+ t.Fatalf("bytes read from the base: chunk by chunk %d, joined %d", local[0], local[1])
+ }
+}
+
+// A base given only to read is never claimed; it is held while it is read and released after.
+func TestReadBasesAreHeldAndNeverClaimed(t *testing.T) {
+ ctx := context.Background()
+ s := openTwoLevel(t, t.TempDir())
+ w := random(51, 8<<20)
+ v := append([]byte(nil), w...)
+ copy(v[3<<20:], random(52, 200<<10))
+ b, nm, src := rebaseFixture(t, s, map[string][]byte{"decode_B2.rbln": w}, map[string][]byte{"decode_B8.rbln": v})
+ held, released := 0, 0
+ out := t.TempDir()
+ st, err := s.Pull(ctx, nm, out, PullOptions{ReadBases: []Base{b},
+ Claim: func(Base) (string, bool) { t.Fatal("claimed a base given only to read"); return "", false },
+ Hold: func(x Base) (func(), bool) {
+ if x.Manifest != b.Manifest {
+ t.Fatalf("held %s", x.Manifest)
+ }
+ held++
+ return func() { released++ }, true
+ }})
+ if err != nil {
+ t.Fatal(err)
+ }
+ sameTree(t, src, out)
+ if held != 1 || released != 1 || st.LocalBase != b.Manifest.String() || st.Base != "" || st.LocalBytes < int64(len(v))*8/10 {
+ t.Fatalf("held %d, released %d, %+v", held, released, st)
+ }
+ if _, err := os.Stat(filepath.Join(b.Dir, "decode_B2.rbln")); err != nil {
+ t.Fatalf("the base was touched: %v", err)
+ }
+
+ // a base that cannot be held is not read
+ out = t.TempDir()
+ st, err = s.Pull(ctx, nm, out, PullOptions{ReadBases: []Base{b}, Hold: func(Base) (func(), bool) { return nil, false }})
+ if err != nil {
+ t.Fatal(err)
+ }
+ sameTree(t, src, out)
+ if st.LocalBase != "" || st.LocalBytes != 0 {
+ t.Fatalf("read a base it could not hold: %+v", st)
+ }
+}
+
+// A one-level store with FrameFlat stores its chunks in frames and gives the same files back;
+// compressible data takes less room than without frames.
+func TestFlatStoreCanFrameItsChunks(t *testing.T) {
+ ctx := context.Background()
+ data := make([]byte, 6<<20)
+ for i := range data { // compressible, but not chunk by chunk identical
+ data[i] = byte(i>>9) ^ byte(i%7)
+ }
+ src := t.TempDir()
+ write(t, filepath.Join(src, "decode_B2.rbln"), data)
+ sizes := map[bool]int64{}
+ for _, framed := range []bool{false, true} {
+ b, err := bfs.Open(t.TempDir(), backend.DurabilityNone)
+ if err != nil {
+ t.Fatal(err)
+ }
+ s, err := OpenWith(ctx, b, Create{Chunker: chunker.Params{Min: 1 << 10, Avg: 4 << 10, Max: 16 << 10},
+ PackSize: 1 << 20, Frame: 256 << 10, FrameFlat: framed})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if (s.Config.Frame > 0) != framed || s.Config.Sub != nil {
+ t.Fatalf("framed %v: config %+v", framed, s.Config)
+ }
+ m, _, err := s.Push(ctx, src, PushOptions{Compress: codec.ModeAuto})
+ if err != nil {
+ t.Fatal(err)
+ }
+ out := t.TempDir()
+ if _, err := s.Pull(ctx, m, out, PullOptions{}); err != nil {
+ t.Fatal(err)
+ }
+ sameTree(t, src, out)
+ err = b.List(ctx, backend.Pack, func(_ string, n int64) error { sizes[framed] += n; return nil })
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+ if sizes[true] <= 0 || sizes[true] >= sizes[false] {
+ t.Fatalf("packs: %d B framed, %d B unframed", sizes[true], sizes[false])
+ }
+}
diff --git a/pkg/store/spans.go b/pkg/store/spans.go
new file mode 100644
index 0000000..d187281
--- /dev/null
+++ b/pkg/store/spans.go
@@ -0,0 +1,101 @@
+package store
+
+import (
+ "cmp"
+ "context"
+ "os"
+ "slices"
+ "sync"
+)
+
+// rjob is one chunk to read from a file: n bytes at off.
+type rjob struct {
+ node int32
+ file int
+ off int64
+ n int
+}
+
+// readSpans reads the jobs from files, joining chunks that follow one another in a file into
+// single reads of at most maxRange bytes, and calls fn with each chunk's bytes. A read of 15 GB
+// in 64 KiB chunks is 230 000 system calls one by one and under a thousand joined. fn gets nil
+// for a chunk whose read failed. It is called from several goroutines; b is only valid during
+// the call.
+func readSpans(ctx context.Context, files []*os.File, jobs []rjob, workers int, maxRange int64, fn func(j rjob, b []byte)) {
+ if len(jobs) == 0 {
+ return
+ }
+ if maxRange <= 0 {
+ maxRange = 16 << 20
+ }
+ slices.SortFunc(jobs, func(a, b rjob) int { // each file front to back
+ if a.file != b.file {
+ return a.file - b.file
+ }
+ return cmp.Compare(a.off, b.off)
+ })
+ type span struct{ a, b int } // jobs[a:b]
+ var spans []span
+ for k, j := range jobs {
+ if n := len(spans); n > 0 {
+ s := &spans[n-1]
+ first, prev := jobs[s.a], jobs[s.b-1]
+ if prev.file == j.file && prev.off+int64(prev.n) == j.off && j.off+int64(j.n)-first.off <= maxRange {
+ s.b = k + 1
+ continue
+ }
+ }
+ spans = append(spans, span{k, k + 1})
+ }
+ ch := make(chan span)
+ var wg sync.WaitGroup
+ for w := 0; w < max(workers, 1); w++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ var buf []byte
+ for s := range ch {
+ if ctx.Err() != nil {
+ continue
+ }
+ first, last := jobs[s.a], jobs[s.b-1]
+ size := int(last.off + int64(last.n) - first.off)
+ if cap(buf) < size {
+ buf = make([]byte, size)
+ }
+ b := buf[:size]
+ _, err := files[first.file].ReadAt(b, first.off)
+ for _, j := range jobs[s.a:s.b] {
+ if err != nil {
+ fn(j, nil)
+ continue
+ }
+ o := int(j.off - first.off)
+ fn(j, b[o:o+j.n])
+ }
+ }
+ }()
+ }
+ for _, s := range spans {
+ ch <- s
+ }
+ close(ch)
+ wg.Wait()
+}
+
+// places calls fn for every place chunk id goes: a file and the offset in it.
+func places(nodes []node, id int32, base int64, fn func(target)) {
+ nd := &nodes[id]
+ if nd.hasT {
+ fn(target{nd.firstT.file, nd.firstT.off + base})
+ for _, t := range nd.moreT {
+ fn(target{t.file, t.off + base})
+ }
+ }
+ if nd.hasP {
+ places(nodes, nd.firstP.node, base+int64(nd.firstP.off), fn)
+ for _, e := range nd.moreP {
+ places(nodes, e.node, base+int64(e.off), fn)
+ }
+ }
+}
diff --git a/pkg/store/store.go b/pkg/store/store.go
index 124161f..3cc274e 100644
--- a/pkg/store/store.go
+++ b/pkg/store/store.go
@@ -52,6 +52,9 @@ type Create struct {
SubKinds []layout.Kind // file kinds that get sub-chunks; empty: all
PackSize int64
Frame int64 // compression frames for sub-chunked files (#52); used only with Sub
+ // FrameFlat frames the new chunks of a one-level store too (Sub nil): Frame is then the frame
+ // size. It exists to compare flat chunking with two levels under the same compression.
+ FrameFlat bool
}
// DefaultSubKinds are the kinds whose compile variants rearrange the same bytes (#15).
@@ -85,6 +88,8 @@ func OpenWith(ctx context.Context, b backend.Backend, c Create) (*Store, error)
cfg = Config{Schema: "onggi.store.v1", Chunker: p, PackSize: packSize, Sub: c.Sub, Created: time.Now().UTC()}
if c.Sub != nil {
cfg.SubKinds, cfg.Frame = c.SubKinds, c.Frame
+ } else if c.FrameFlat {
+ cfg.Frame = c.Frame
}
data, _ := json.MarshalIndent(cfg, "", " ")
if err := b.Put(ctx, configKey, bytes.NewReader(data), int64(len(data))); err != nil {
diff --git a/tools/varmorph/README.md b/tools/varmorph/README.md
new file mode 100644
index 0000000..108064a
--- /dev/null
+++ b/tools/varmorph/README.md
@@ -0,0 +1,7 @@
+# VarMorph layout coding and device rebuild (prototype)
+
+- `classify.py`: classifies each 64 KiB window of a compiled artifact (weights by numeric type, code, padding, text) from byte statistics, without parsing the format.
+- `layout_coder.py`: codes a target variant against sibling(s) at any byte offset below chunk size (sub-block shift search, motion prediction with merge candidates, run merging, XOR residual, intra codec with bf16 byte planes, fill). It decodes every target and compares it byte for byte. With `VCODEC_DUMP=1` it writes the descriptor (`*.runs.npz`).
+- `device_rebuild.py`: rebuilds a target on the device from a resident sibling. `replay` is out of place. `inplace` uses a ring of saved windows, KEEP for unmoved runs, absorb below a measured threshold, folding, and batched copies on NPU-B.
+- `npu_d2d_bench.py`: device copy and upload costs per platform.
+- `rbln_vmshim.cpp`: C shim over `rbln_memcpy_v2v_multi` / `rbln_memcpy_h2v_multi` of the NPU-B runtime API.
diff --git a/tools/varmorph/classify.py b/tools/varmorph/classify.py
new file mode 100644
index 0000000..cd6db16
--- /dev/null
+++ b/tools/varmorph/classify.py
@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+"""Cohort 2026-11-12 (11): what a compiled binary holds, by content class, without parsing its format.
+
+Every 64 KiB window of a file is classified from byte statistics alone, so the same rule applies to
+any vendor's artifact:
+ fill one or two distinct byte values (padding, zero-initialized buffers)
+ text >= 85 % printable ASCII (metadata, names, JSON)
+ fp16/bf16 16-bit float weights: >= 90 % of 16-bit words have a high byte in 0x04-0x3F or
+ 0x84-0xBF (|x| < 2 as fp16 or bf16), the two signs are balanced within 10 %, and the
+ high byte has <= 5.5 bits of entropy. The median magnitude byte tells the two apart
+ (bf16 weights sit at 0x3A-0x3D, fp16 weights below 0x38).
+ fp32 the same test on the top byte of 32-bit words (fp32 tensors: norms, tables, biases);
+ every byte phase is tried, since tensors start at odd offsets in some containers
+ bf16_in_fp32 fp32 words whose low 16 bits are zero in >= 95 % of words: bf16 values widened
+ code byte entropy < 7.0 bits (instructions, tables, descriptors; structured, compressible)
+ dense the rest (high entropy, not weight-like: packed or quantized data, compressed sections)
+Per class: bytes, mean byte entropy, mean entropy of the high and low byte of 16-bit words.
+
+Optional join with a coding result: MODES.npy holds, per 64 KiB window of the target, the number of
+64-byte blocks coded as match / fill / residual / intra (from vcodec_server.py). With it, the
+output also gives how each class of the target was found in its sibling.
+argv: OUT.json FILE [FILE ...] [--modes FILE=MODES.npy ...]
+"""
+import json
+import os
+import sys
+import time
+
+import numpy as np
+
+WIN = 1 << 16
+CLASSES = ["fill", "text", "bf16", "fp32", "code", "dense", "fp16", "bf16_in_fp32"]
+
+
+def entropy_rows(counts):
+ tot = counts.sum(axis=1, keepdims=True).astype(np.float64)
+ p = counts / np.maximum(tot, 1)
+ with np.errstate(divide="ignore", invalid="ignore"):
+ e = -(np.where(p > 0, p * np.log2(p), 0.0)).sum(axis=1)
+ return e
+
+
+def hist_rows(a, nw):
+ """Per-row 256-bin histogram of a (nw, m) uint8 array."""
+ off = (np.arange(nw, dtype=np.int64) * 256)[:, None]
+ return np.bincount((a.astype(np.int64) + off).ravel(), minlength=nw * 256).reshape(nw, 256)
+
+
+def classify(path):
+ n = os.path.getsize(path) // WIN
+ mm = np.memmap(path, dtype=np.uint8, mode="r", shape=(n * WIN,))
+ cls = np.empty(n, dtype=np.uint8)
+ feats = np.empty((n, 3), dtype=np.float32)
+ step = (1 << 28) // WIN
+ wl_hi = np.zeros(256, dtype=bool)
+ wl_hi[0x04:0x40] = True
+ wl_hi[0x84:0xC0] = True
+ printable = np.zeros(256, dtype=bool)
+ printable[0x20:0x7F] = True
+ printable[[9, 10, 13]] = True
+ for i in range(0, n, step):
+ j = min(n, i + step)
+ a = np.asarray(mm[i * WIN:j * WIN]).reshape(j - i, WIN)
+ nw = j - i
+ hb = hist_rows(a, nw)
+ h_odd = hist_rows(a[:, 1::2], nw) # high byte of 16-bit words at phase 0
+ h_even = hist_rows(a[:, 0::2], nw) # high byte of 16-bit words at phase 1
+ e = entropy_rows(hb)
+ distinct = (hb > 0).sum(axis=1)
+ asc = (hb[:, printable].sum(axis=1)) / WIN
+
+ def test16(hh):
+ w = hh[:, wl_hi].sum(axis=1) / (WIN // 2)
+ sym = 1 - np.abs(hh[:, :128].sum(axis=1) - hh[:, 128:].sum(axis=1)) / (WIN // 2)
+ mag = hh[:, :128] + hh[:, 128:]
+ med = (np.cumsum(mag, axis=1) >= (WIN // 4)).argmax(axis=1)
+ return (w >= 0.9) & (sym >= 0.9) & (entropy_rows(hh) <= 5.5), med, entropy_rows(hh)
+ ok0, med0, e0 = test16(h_odd)
+ ok1, med1, e1 = test16(h_even)
+ is16 = ok0 | ok1
+ med = np.where(ok0, med0, med1)
+ ehi = np.where(ok1 & ~ok0, e1, e0)
+ elo = np.where(ok1 & ~ok0, e0, e1)
+ is32 = np.zeros(nw, dtype=bool)
+ zlow = np.zeros(nw)
+ for p in range(4):
+ h32 = hist_rows(a[:, p::4], nw)
+ w32 = h32[:, wl_hi].sum(axis=1) / (WIN // 4)
+ sym32 = 1 - np.abs(h32[:, :128].sum(axis=1) - h32[:, 128:].sum(axis=1)) / (WIN // 4)
+ ok = (w32 >= 0.9) & (sym32 >= 0.9)
+ if ok.any():
+ lo = (hist_rows(a[:, (p - 3) % 4::4], nw)[:, 0] + hist_rows(a[:, (p - 2) % 4::4], nw)[:, 0]) / (WIN // 2)
+ zlow = np.where(ok & ~is32, lo, zlow)
+ is32 |= ok
+ c = np.full(nw, 5, dtype=np.uint8)
+ c[e < 7.0] = 4
+ c[is32 & ~is16] = 3
+ c[is32 & ~is16 & (zlow >= 0.95)] = 7 # fp32 holding bf16 values (low half zero)
+ c[is16 & (med >= 0x38)] = 2
+ c[is16 & (med < 0x38)] = 6
+ c[asc >= 0.85] = 1
+ c[distinct <= 2] = 0
+ cls[i:j] = c
+ feats[i:j, 0] = e
+ feats[i:j, 1] = ehi
+ feats[i:j, 2] = elo
+ return cls, feats
+
+
+def summary(path, cls, feats, modes=None):
+ out = {"file": path, "bytes": os.path.getsize(path), "windows": int(len(cls)), "classes": {}}
+ for k, name in enumerate(CLASSES):
+ sel = cls == k
+ if not sel.any():
+ continue
+ row = {"bytes": int(sel.sum()) * WIN, "share": float(sel.mean()),
+ "entropy": float(feats[sel, 0].mean()), "entropy_hi": float(feats[sel, 1].mean()),
+ "entropy_lo": float(feats[sel, 2].mean())}
+ if modes is not None:
+ m = modes[: len(cls)][sel[: len(modes)]].sum(axis=0).astype(np.int64) * 64
+ row["coded_as"] = {"match": int(m[0]), "fill": int(m[1]), "residual": int(m[2]), "intra": int(m[3])}
+ out["classes"][name] = row
+ # contiguous extents of one class, to show the layout
+ brk = np.nonzero(np.diff(cls.astype(np.int16)))[0] + 1
+ st = np.concatenate([[0], brk])
+ en = np.concatenate([brk, [len(cls)]])
+ ext = [{"class": CLASSES[cls[s]], "offset": int(s) * WIN, "bytes": int(e - s) * WIN} for s, e in zip(st, en)]
+ big = sorted(ext, key=lambda x: -x["bytes"])[:12]
+ out["extents"] = int(len(ext))
+ out["largest_extents"] = sorted(big, key=lambda x: x["offset"])
+ return out
+
+
+if __name__ == "__main__":
+ args = sys.argv[1:]
+ out = args[0]
+ files, modes = [], {}
+ k = 1
+ while k < len(args):
+ if args[k] == "--modes":
+ f, m = args[k + 1].split("=", 1)
+ modes[f] = m
+ k += 2
+ else:
+ files.append(args[k])
+ k += 1
+ res = []
+ for f in files:
+ t = time.time()
+ cls, feats = classify(f)
+ md = np.load(modes[f]) if f in modes else None
+ res.append(summary(f, cls, feats, md))
+ res[-1]["seconds"] = round(time.time() - t, 1)
+ with open(out, "w") as fh:
+ json.dump(res, fh, indent=1)
+ print(time.strftime("%H:%M:%S"), f, {c: round(v["share"], 3) for c, v in res[-1]["classes"].items()}, flush=True)
+ print("ALL-DONE", flush=True)
diff --git a/tools/varmorph/device_rebuild.py b/tools/varmorph/device_rebuild.py
new file mode 100644
index 0000000..d2749cb
--- /dev/null
+++ b/tools/varmorph/device_rebuild.py
@@ -0,0 +1,563 @@
+#!/usr/bin/env python3
+"""Rebuild a target variant on the NPU from a resident sibling, with the coder's descriptor.
+
+The reference (sibling) bytes are uploaded once and left on the device: they stand for weights
+that are already resident. The timed part rebuilds the target on the device:
+ 1. folded copies: match runs of equal length whose target and reference offsets advance by
+ constant strides are one strided copy (as_strided). The rest are single-run copies.
+ 2. patch: the 64-byte blocks that match nothing (residual, new content, padding) are uploaded
+ as one contiguous buffer and put in place with one index_copy_.
+The baseline is an upload of the whole target from host memory, in the same 1 GiB pieces.
+The result is read back and its SHA-256 compared with the target file.
+
+Device memory is a list of 1 GiB tensors (int16); a copy never crosses a piece, and copies are
+cut at MAXCOPY bytes. Offsets must be even (int16 elements); odd runs are reported and fall
+back to the patch.
+argv: analyze RUNS.npz OUT.json (folding statistics only, no device)
+ inplace RUNS.npz OUT.json PLATFORM (in-place rebuild in the sibling's memory)
+ replay RUNS.npz OUT.json PLATFORM(rngd|rbln) (device replay)
+"""
+import hashlib
+import json
+import os
+import sys
+import time
+
+import numpy as np
+
+PIECE = 1 << 30 # bytes per device tensor
+MAXCOPY = 256 << 20 # bytes per copy call
+BLK = 64
+
+
+def split_runs(dst, src, ln):
+ """Cut runs at piece boundaries of both sides and at MAXCOPY."""
+ out_d, out_s, out_l = [], [], []
+ for d, s, l in zip(dst.tolist(), src.tolist(), ln.tolist()):
+ while l > 0:
+ take = min(l, PIECE - d % PIECE, PIECE - s % PIECE, MAXCOPY)
+ out_d.append(d)
+ out_s.append(s)
+ out_l.append(take)
+ d += take
+ s += take
+ l -= take
+ return np.array(out_d, np.int64), np.array(out_s, np.int64), np.array(out_l, np.int64)
+
+
+GAP = int(os.environ.get("REPLAY_GAP", str(1 << 20))) # bytes of target a merged copy may cover
+
+
+# Measured cost of one copy call and of host upload per platform (notes/npu-d2d-20261112):
+# rngd-a3: copy_ per call 120 us, upload 3.7 GB/s; rbln-b3: v2v_multi 1.6 us per copy, upload 21.6 GB/s.
+COSTS = {"rngd": dict(call_s=120e-6, h2d_Bps=3.7e9, d2d_Bps=24.4e9),
+ "rbln": dict(call_s=1.6e-6, h2d_Bps=21.6e9, d2d_Bps=21.7e9),
+ "rbln-unbatched": dict(call_s=5.0e-6, h2d_Bps=21.6e9, d2d_Bps=21.7e9),
+ "cpu": dict(call_s=1e-6, h2d_Bps=10e9, d2d_Bps=10e9)}
+
+
+def estimate(platform, n_copies, copy_bytes, patch_bytes, target_bytes):
+ """Predicted seconds of the rebuild and of a full upload, from the measured cost table."""
+ c = COSTS.get(platform, COSTS["cpu"])
+ rebuild = n_copies * c["call_s"] + copy_bytes / c["d2d_Bps"] + patch_bytes / c["h2d_Bps"]
+ upload = target_bytes / c["h2d_Bps"]
+ return rebuild, upload
+
+
+def absorb_threshold(platform):
+ """A run shorter than this is cheaper to upload in the patch than to copy with its own call:
+ the bytes the host link moves in the time of one call."""
+ c = COSTS.get(platform, COSTS["cpu"])
+ return int(c["call_s"] * c["h2d_Bps"]) // BLK * BLK
+
+
+ABSORB = int(os.environ.get("REPLAY_ABSORB", str(64 << 10))) # bytes: shorter off-motion runs go to the patch
+LOOK = int(os.environ.get("REPLAY_LOOK", str(1 << 20))) # bytes of target to look ahead for the motion
+
+
+def absorb(dst, src, ln, limit=ABSORB, look=LOOK):
+ """Rate-distortion step: a short run whose motion differs from the surrounding segment is not
+ copied on its own; the segment's copy runs over it and its blocks join the patch. Returns the
+ kept runs and the absorbed byte ranges (target offset, length)."""
+ if limit <= 0 or len(dst) == 0:
+ return dst, src, ln, []
+ o = np.argsort(dst, kind="stable")
+ dst, src, ln = dst[o].tolist(), src[o].tolist(), ln[o].tolist()
+ mv = [s_ - d for d, s_ in zip(dst, src)]
+ keep, gone = [], []
+ i, n = 0, len(dst)
+ while i < n:
+ keep.append(i)
+ cur = mv[i]
+ j = i + 1
+ while j < n:
+ if mv[j] == cur:
+ keep.append(j)
+ j += 1
+ continue
+ # skip short off-motion runs if the segment's motion resumes within LOOK bytes
+ k, skipped = j, 0
+ while k < n and mv[k] != cur and ln[k] <= limit and dst[k] - dst[j] <= look:
+ skipped += 1
+ k += 1
+ if k < n and mv[k] == cur and skipped:
+ gone.extend(range(j, k))
+ j = k
+ else:
+ break
+ i = j
+ ka = np.array(keep, dtype=np.int64)
+ d, s_, l = np.array(dst)[ka], np.array(src)[ka], np.array(ln)[ka]
+ return d, s_, l, [(dst[g], ln[g]) for g in gone]
+
+
+def coalesce(dst, src, ln, gap=GAP):
+ """Merge runs with the same motion (src - dst) whose target gap is <= gap: one copy covers the
+ gap, and the patch overwrites the gap blocks afterwards (skip + residual)."""
+ if gap <= 0 or len(dst) == 0:
+ return dst, src, ln
+ o = np.argsort(dst, kind="stable")
+ dst, src, ln = dst[o], src[o], ln[o]
+ out = [[dst[0], src[0], ln[0]]]
+ for d, s_, l in zip(dst[1:].tolist(), src[1:].tolist(), ln[1:].tolist()):
+ pd, ps, pl = out[-1]
+ if s_ - d == ps - pd and 0 <= d - (pd + pl) <= gap:
+ out[-1][2] = d + l - pd
+ else:
+ out.append([d, s_, l])
+ a = np.array(out, dtype=np.int64)
+ return a[:, 0], a[:, 1], a[:, 2]
+
+
+def fold(dst, src, ln):
+ """Greedy: consecutive runs (by target offset) with equal length and constant strides, inside
+ one piece on each side, form a group (d0, s0, len, count, dd, ds)."""
+ order = np.argsort(dst, kind="stable")
+ dst, src, ln = dst[order], src[order], ln[order]
+ groups = []
+ i, n = 0, len(dst)
+ while i < n:
+ j = i + 1
+ if j < n and ln[j] == ln[i] and dst[j] // PIECE == dst[i] // PIECE and src[j] // PIECE == src[i] // PIECE:
+ dd, ds = dst[j] - dst[i], src[j] - src[i]
+ if dd >= ln[i] and ds > 0:
+ while (j + 1 < n and ln[j + 1] == ln[i] and dst[j + 1] - dst[j] == dd and src[j + 1] - src[j] == ds
+ and dst[j + 1] // PIECE == dst[i] // PIECE and src[j + 1] // PIECE == src[i] // PIECE
+ and (j + 1 - i + 1) * ln[i] <= MAXCOPY):
+ j += 1
+ j += 1
+ else:
+ dd = ds = 0
+ j = i + 1
+ else:
+ dd = ds = 0
+ groups.append((int(dst[i]), int(src[i]), int(ln[i]), j - i, int(dd), int(ds)))
+ i = j
+ return groups
+
+
+def load(runs_path):
+ r = np.load(runs_path)
+ dst, src, ln = r["dst"].astype(np.int64), r["src"].astype(np.int64), r["len"].astype(np.int64)
+ odd = (dst % 2 != 0) | (src % 2 != 0)
+ return r, dst[~odd], src[~odd], ln[~odd], dst[odd], ln[odd]
+
+
+def analyze(runs_path, out):
+ r, dst, src, ln, odd_d, odd_l = load(runs_path)
+ dst, src, ln, gone = absorb(dst, src, ln)
+ dst, src, ln = coalesce(dst, src, ln)
+ d2, s2, l2 = split_runs(dst, src, ln)
+ g = fold(d2, s2, l2)
+ cnt = np.array([x[3] for x in g])
+ res = dict(gap=GAP, absorb=ABSORB, absorbed_runs=len(gone), absorbed_bytes=int(sum(x[1] for x in gone)),
+ coalesced_runs=int(len(dst)), runs=int(len(r["dst"])), odd_runs=int(len(odd_d)), odd_bytes=int(odd_l.sum()),
+ split_runs=int(len(d2)), groups=len(g), single_run_groups=int((cnt == 1).sum()),
+ bytes_in_multi_run_groups=int(sum(x[2] * x[3] for x in g if x[3] > 1)),
+ match_bytes=int(l2.sum()), other_blocks=int(len(r["other"])), v_bytes=int(r["v_bytes"]))
+ with open(out, "w") as f:
+ json.dump(res, f, indent=1)
+ print(json.dumps(res))
+ return r, g, odd_d, odd_l, gone
+
+
+def replay(runs_path, out, platform):
+ r, g, odd_d, odd_l, gone = analyze(runs_path, out + ".fold.json")
+ import torch
+ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+ from npu_d2d_bench import device
+ dev, sync, pk = device(platform)
+ ref_path, tgt_path = str(r["refs"][0]), str(r["target"])
+ vb = int(r["v_bytes"])
+ res = dict(schema="npu-replay.v1", platform=platform, device=dev, ref=ref_path, target=tgt_path, v_bytes=vb,
+ groups=len(g), load=(os.getloadavg() if hasattr(os, "getloadavg") else None))
+
+ def pieces_of(nbytes):
+ return [min(PIECE, nbytes - o) for o in range(0, nbytes, PIECE)]
+
+ def host_bytes(path, nbytes):
+ a = np.zeros(nbytes + (nbytes % 2), dtype=np.uint8)
+ with open(path, "rb") as f:
+ f.readinto(memoryview(a)[:os.path.getsize(path)])
+ return a
+
+ ref = host_bytes(ref_path, os.path.getsize(ref_path))
+ tgt = host_bytes(tgt_path, vb)
+ # resident reference (untimed upload)
+ t = time.perf_counter()
+ S = [torch.from_numpy(ref[o:o + n].view(np.int16).copy()).to(dev) for o, n in zip(range(0, len(ref), PIECE), pieces_of(len(ref)))]
+ sync()
+ res["ref_upload_s"] = time.perf_counter() - t
+ al = (vb // BLK) * BLK # the device holds the block-aligned part; the tail (< 64 B) stays on the host
+ D = [torch.empty(n // 2, dtype=torch.int16, device=dev) for n in pieces_of(al)]
+ # baseline: upload the whole target
+ host_t = [torch.from_numpy(tgt[o:o + n].view(np.int16).copy()) for o, n in zip(range(0, al, PIECE), pieces_of(al))]
+ sync()
+ t = time.perf_counter()
+ for d, h in zip(D, host_t):
+ d.copy_(h)
+ sync()
+ res["baseline_upload_s"] = time.perf_counter() - t
+ # patch: every block not covered by an even match run
+ cover = np.zeros(vb // BLK + 1, dtype=bool)
+ for d0, s0, l, c, dd, ds in g:
+ for k in range(c):
+ a = (d0 + k * dd) // BLK
+ cover[a:a + l // BLK] = True
+ need = ~cover[: vb // BLK]
+ need[r["other"][r["other"] < vb // BLK]] = True
+ for d0, l in gone: # absorbed runs: the segment's copy wrote over them
+ need[d0 // BLK:(d0 + l) // BLK] = True
+ patch_blocks = np.nonzero(need)[0]
+ pb = tgt[: (vb // BLK) * BLK].reshape(-1, BLK)[patch_blocks].copy()
+ by_piece = {}
+ for i, b in enumerate(patch_blocks.tolist()):
+ by_piece.setdefault((b * BLK) // PIECE, []).append(i)
+ res["patch_blocks"] = int(len(patch_blocks))
+ res["patch_bytes"] = int(len(patch_blocks) * BLK)
+ tail = tgt[(vb // BLK) * BLK:vb]
+ # clear the target so the replay cannot pass by accident
+ for d in D:
+ d.zero_()
+ sync()
+ path = {"strided": 0, "single": 0, "fallback": 0}
+ t0 = time.perf_counter()
+ for d0, s0, l, c, dd, ds in g:
+ dp, do = divmod(d0, PIECE)
+ sp, so = divmod(s0, PIECE)
+ le = l // 2
+ if c == 1:
+ D[dp][do // 2:do // 2 + le].copy_(S[sp][so // 2:so // 2 + le])
+ path["single"] += 1
+ continue
+ try:
+ D[dp].as_strided((c, le), (dd // 2, 1), do // 2).copy_(S[sp].as_strided((c, le), (ds // 2, 1), so // 2))
+ path["strided"] += 1
+ except Exception:
+ for k in range(c):
+ a, b = (do + k * dd) // 2, (so + k * ds) // 2
+ D[dp][a:a + le].copy_(S[sp][b:b + le])
+ path["fallback"] += 1
+ sync()
+ t1 = time.perf_counter()
+ for p, rows in by_piece.items():
+ sel = np.array(rows)
+ local = (patch_blocks[sel] * BLK - p * PIECE) // BLK
+ buf = torch.from_numpy(pb[sel].view(np.int16)).to(dev)
+ idx = torch.from_numpy(local.astype(np.int64)).to(dev)
+ D[p].view(-1, BLK // 2).index_copy_(0, idx, buf)
+ sync()
+ t2 = time.perf_counter()
+ res.update(copy_s=t1 - t0, patch_s=t2 - t1, rebuild_s=t2 - t0, paths=path)
+ # verify
+ h = hashlib.sha256()
+ got = np.concatenate([d.cpu().numpy().view(np.uint8) for d in D])[: (vb // BLK) * BLK]
+ h.update(got.tobytes())
+ h.update(tail.tobytes())
+ want = hashlib.sha256(tgt[:vb].tobytes()).hexdigest()
+ res.update(sha256=h.hexdigest(), target_sha256=want, correct=h.hexdigest() == want)
+ res["mismatch_bytes"] = int((got != tgt[: (vb // BLK) * BLK]).sum()) if not res["correct"] else 0
+ with open(out, "w") as f:
+ json.dump(res, f, indent=1)
+ print(json.dumps({k: v for k, v in res.items() if k != "load"}))
+
+
+WIN = int(os.environ.get("REPLAY_WIN", str(64 << 20))) # bytes of target written per window
+RING = int(os.environ.get("REPLAY_RING", str(256 << 20))) # bytes of original content kept behind the window
+
+
+def replay_inplace(runs_path, out, platform):
+ """Rebuild the target in the memory that holds the sibling, with O(RING) scratch.
+
+ One buffer holds the sibling's bytes and is overwritten with the target window by window, in
+ target order. Before a window is written, its original bytes are saved in a ring of
+ RING/WIN + 1 slots. A copy reads a source byte from:
+ - the live buffer if it lies after the current window (not yet overwritten);
+ - the ring if it lies in the RING bytes before the window, or in the window itself;
+ - a far scratch, filled before the first write, for the few sources further back.
+ Invariant: no original byte is overwritten before it is saved, and every copy that reads the
+ live buffer reads bytes after the current window. The patch is applied last."""
+ import torch
+ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+ from npu_d2d_bench import device
+ r, dst, src, ln, odd_d, odd_l = load(runs_path)
+ lim = ABSORB if "REPLAY_ABSORB" in os.environ else absorb_threshold(platform)
+ dst, src, ln, gone = absorb(dst, src, ln, limit=lim)
+ dst, src, ln = coalesce(dst, src, ln)
+ dev, sync, pk = device(platform)
+ ref_path, tgt_path = str(r["refs"][0]), str(r["target"])
+ vb = int(r["v_bytes"])
+ al = (vb // BLK) * BLK
+ ref = np.fromfile(ref_path, dtype=np.uint8)
+ tgt = np.fromfile(tgt_path, dtype=np.uint8)
+ total = max(len(ref), al)
+ total += (-total) % WIN
+ nslots = RING // WIN + 1
+ pieces = []
+ for d, s_, l in zip(dst.tolist(), src.tolist(), ln.tolist()):
+ while l > 0:
+ take = min(l, WIN - d % WIN)
+ pieces.append((d, s_, take))
+ d, s_, l = d + take, s_ + take, l - take
+ pieces.sort()
+ kept = [x for x in pieces if x[0] == x[1]] # KEEP: already in place, no copy
+ pieces_cp = [x for x in pieces if x[0] != x[1]]
+ segs = []
+ far = []
+ far_bytes = 0
+ need_ring = set() # windows whose original bytes a later copy reads: only these are saved
+ for d, s_, l in pieces_cp:
+ w = d // WIN
+ a = w * WIN
+ ring_lo = max(a - (nslots - 1) * WIN, 0)
+ x, dd = s_, d
+ while x < s_ + l:
+ if x >= a + WIN:
+ take = min(s_ + l - x, PIECE - x % PIECE)
+ segs.append((w, dd, "live", x // PIECE, x % PIECE, take))
+ elif x >= ring_lo:
+ take = min(s_ + l - x, WIN - x % WIN, a + WIN - x)
+ segs.append((w, dd, "ring", (x // WIN) % nslots, x % WIN, take))
+ need_ring.add(x // WIN)
+ else:
+ take = min(s_ + l - x, ring_lo - x, PIECE - (far_bytes % PIECE))
+ segs.append((w, dd, "far", far_bytes // PIECE, far_bytes % PIECE, take))
+ far.append((x, take))
+ far_bytes += take
+ x += take
+ dd += take
+ res = dict(schema="npu-replay-inplace.v1", platform=platform, device=dev, ref=ref_path, target=tgt_path,
+ v_bytes=vb, win=WIN, ring=RING, slots=nslots, copies=len(segs), far_bytes=far_bytes,
+ absorb_limit=lim, absorbed_bytes=int(sum(x[1] for x in gone)),
+ keep_bytes=int(sum(x[2] for x in kept)), keep_runs=len(kept),
+ far_copies=len(far), scratch_bytes=nslots * WIN + far_bytes, buffer_bytes=total,
+ load=(os.getloadavg() if hasattr(os, "getloadavg") else None))
+ host = np.zeros(total, dtype=np.uint8)
+ host[:len(ref)] = ref
+ Bf = [torch.from_numpy(host[o:o + PIECE].view(np.int16).copy()).to(dev) for o in range(0, total, PIECE)]
+ ringt = [torch.empty(WIN // 2, dtype=torch.int16, device=dev) for _ in range(nslots)]
+ fart = [torch.empty(min(PIECE, far_bytes - o) // 2, dtype=torch.int16, device=dev) for o in range(0, far_bytes, PIECE)]
+ sync()
+ del host
+ # NPU-B: batched copies through the public runtime API (rbln_memcpy_v2v_multi / h2v_multi);
+ # copies inside one call are unordered, so a call never mixes a window's ring save with its
+ # copies, and no copy in a call writes where another reads
+ shim = None
+ if platform == "rbln" and os.environ.get("RBLN_VMSHIM") and os.environ.get("REPLAY_BATCH", "1") != "0":
+ import ctypes
+ shim = ctypes.CDLL(os.environ["RBLN_VMSHIM"])
+ for t in ringt + fart:
+ shim.vmshim_layout_like(ctypes.c_uint64(t.data_ptr()), ctypes.c_uint64(Bf[0].data_ptr()))
+ if platform == "cpu" and os.environ.get("REPLAY_FAKE_BATCH"): # test the batched addressing on the host
+ import ctypes
+
+ class _Fake:
+ @staticmethod
+ def _run(a_, b_, c_, n):
+ for i in range(n):
+ ctypes.memmove(b_[i], a_[i], c_[i])
+ return 0
+ vmshim_v2v_multi = _run
+ vmshim_h2v_multi = _run
+ shim = _Fake()
+ MAXB = 1024
+ batch_calls = [0]
+
+ def flush(ops, h2v=False):
+ import ctypes
+ for i in range(0, len(ops), MAXB):
+ part = ops[i:i + MAXB]
+ U = ctypes.c_uint64 * len(part)
+ a_, b_, c_ = U(*[x[0] for x in part]), U(*[x[1] for x in part]), U(*[x[2] for x in part])
+ rc = (shim.vmshim_h2v_multi if h2v else shim.vmshim_v2v_multi)(a_, b_, c_, len(part))
+ if rc != 0:
+ raise RuntimeError("batched copy rc=%d" % rc)
+ batch_calls[0] += 1
+ need = np.ones(al // BLK, dtype=bool)
+ for d, s_, l in pieces:
+ need[d // BLK:(d + l) // BLK] = False
+ need[r["other"][r["other"] < al // BLK]] = True
+ for d0, l in gone:
+ need[d0 // BLK:(d0 + l) // BLK] = True
+ # runs at odd offsets cannot be addressed in 16-bit device elements: their blocks are uploaded
+ for d0, l in zip(odd_d.tolist(), odd_l.tolist()):
+ need[d0 // BLK:(d0 + l + BLK - 1) // BLK] = True
+ res["odd_runs"] = int(len(odd_d))
+ res["odd_bytes"] = int(odd_l.sum())
+ patch_blocks = np.nonzero(need)[0]
+ pb = tgt[:al].reshape(-1, BLK)[patch_blocks].copy()
+ res.update(patch_blocks=int(len(patch_blocks)), patch_bytes=int(len(patch_blocks) * BLK))
+ nwin = (max(x[0] for x in pieces) // WIN + 1) if pieces else 0
+ n_copies = len(segs) + len(far) + len(need_ring)
+ copy_bytes = sum(sg[5] for sg in segs) + sum(l for _, l in far) + len(need_ring) * WIN
+ res.update(windows=int(nwin), ring_saves=len(need_ring), relocate_bytes=int(sum(sg[5] for sg in segs)))
+ est_rebuild, est_upload = estimate(platform, n_copies, copy_bytes, res["patch_bytes"], al)
+ in_host = os.environ.get("REPLAY_TARGET_IN_HOST") == "1"
+ # the far scratch is known before the first write; beyond its budget the target is uploaded
+ far_budget = int(os.environ.get("REPLAY_FAR_BUDGET", str(1 << 30)))
+ res.update(est_rebuild_s=est_rebuild, est_upload_s=est_upload, target_in_host=in_host, far_budget=far_budget,
+ choice=("upload" if (in_host and est_upload < est_rebuild) or far_bytes > far_budget else "rebuild"))
+
+ def E(b):
+ return b // 2
+
+ def view(origin, key, off, l):
+ t = {"live": Bf, "ring": ringt, "far": fart}[origin][key]
+ return t[E(off):E(off) + E(l)]
+ by_w = {}
+ for sg in segs:
+ by_w.setdefault(sg[0], []).append(sg)
+ # fold, per window: copies from one source tensor with equal length and constant target and
+ # source strides become one strided copy (as_strided on both sides)
+ plan = {}
+ nfold = 0
+ for w, lst in by_w.items():
+ lst.sort(key=lambda x: x[1])
+ out_w, i = [], 0
+ while i < len(lst):
+ _, d0, o0, k0, f0, l0 = lst[i]
+ j = i + 1
+ if j < len(lst) and lst[j][2] == o0 and lst[j][3] == k0 and lst[j][5] == l0:
+ dd, ds = lst[j][1] - d0, lst[j][4] - f0
+ if dd >= l0 and ds > 0 and d0 // PIECE == lst[j][1] // PIECE:
+ while (j + 1 < len(lst) and lst[j + 1][2] == o0 and lst[j + 1][3] == k0 and lst[j + 1][5] == l0
+ and lst[j + 1][1] - lst[j][1] == dd and lst[j + 1][4] - lst[j][4] == ds
+ and lst[j + 1][1] // PIECE == d0 // PIECE):
+ j += 1
+ j += 1
+ else:
+ j = i + 1
+ out_w.append((d0, o0, k0, f0, l0, j - i, (lst[i + 1][1] - d0) if j - i > 1 else 0,
+ (lst[i + 1][4] - f0) if j - i > 1 else 0))
+ nfold += 1
+ i = j
+ plan[w] = out_w
+ res["calls"] = nfold
+ path = {"strided": 0, "single": 0, "fallback": 0}
+ t0 = time.perf_counter()
+ fo = 0
+ if shim is not None:
+ ops = []
+ for x, l in far:
+ ops.append((Bf[x // PIECE].data_ptr() + x % PIECE, fart[fo // PIECE].data_ptr() + fo % PIECE, l))
+ fo += l
+ flush(ops)
+ last_w = max(plan) if plan else -1
+ for w in range(0, last_w + 1):
+ a = w * WIN
+ if w in need_ring:
+ flush([(Bf[a // PIECE].data_ptr() + a % PIECE, ringt[w % nslots].data_ptr(), WIN)])
+ ops = []
+ for d, origin, key, off, l, c, dd, ds in plan.get(w, []):
+ st = {"live": Bf, "ring": ringt, "far": fart}[origin][key]
+ for k in range(c):
+ ops.append((st.data_ptr() + off + k * ds, Bf[d // PIECE].data_ptr() + d % PIECE + k * dd, l))
+ flush(ops)
+ path["batched"] = batch_calls[0]
+ plan = {}
+ for x, l in far if shim is None else []:
+ fart[fo // PIECE][E(fo % PIECE):E(fo % PIECE) + E(l)].copy_(Bf[x // PIECE][E(x % PIECE):E(x % PIECE) + E(l)])
+ fo += l
+ last_w = max(plan) if plan else -1
+ for w in range(0, last_w + 1):
+ a = w * WIN
+ if w in need_ring:
+ ringt[w % nslots].copy_(Bf[a // PIECE][E(a % PIECE):E(a % PIECE) + E(WIN)])
+ for d, origin, key, off, l, c, dd, ds in plan.get(w, []):
+ dt = Bf[d // PIECE]
+ st = {"live": Bf, "ring": ringt, "far": fart}[origin][key]
+ if c == 1:
+ dt[E(d % PIECE):E(d % PIECE) + E(l)].copy_(st[E(off):E(off) + E(l)])
+ path["single"] += 1
+ continue
+ try:
+ dt.as_strided((c, E(l)), (E(dd), 1), E(d % PIECE)).copy_(st.as_strided((c, E(l)), (E(ds), 1), E(off)))
+ path["strided"] += 1
+ except Exception:
+ for k in range(c):
+ a2, b2 = E(d % PIECE + k * dd), E(off + k * ds)
+ dt[a2:a2 + E(l)].copy_(st[b2:b2 + E(l)])
+ path["fallback"] += 1
+ sync()
+ res["paths"] = path
+ t1 = time.perf_counter()
+ if shim is not None and len(patch_blocks):
+ brk = np.nonzero(np.diff(patch_blocks) != 1)[0] + 1
+ starts = np.concatenate([[0], brk])
+ ends = np.concatenate([brk, [len(patch_blocks)]])
+ ops = []
+ base = pb.ctypes.data
+ for a_, b_ in zip(starts.tolist(), ends.tolist()):
+ off = int(patch_blocks[a_]) * BLK
+ n_ = (b_ - a_) * BLK
+ while n_ > 0: # do not cross a device piece
+ take = min(n_, PIECE - off % PIECE)
+ ops.append((base + a_ * BLK, Bf[off // PIECE].data_ptr() + off % PIECE, take))
+ a_ += take // BLK
+ off += take
+ n_ -= take
+ res["patch_ranges"] = len(ops)
+ flush(ops, h2v=True)
+ path["batched"] = batch_calls[0]
+ else:
+ by_piece = {}
+ for i, b in enumerate(patch_blocks.tolist()):
+ by_piece.setdefault((b * BLK) // PIECE, []).append(i)
+ for p_, rows in by_piece.items():
+ sel = np.array(rows)
+ local = (patch_blocks[sel] * BLK - p_ * PIECE) // BLK
+ Bf[p_].view(-1, BLK // 2).index_copy_(0, torch.from_numpy(local.astype(np.int64)).to(dev),
+ torch.from_numpy(pb[sel].view(np.int16)).to(dev))
+ sync()
+ t2 = time.perf_counter()
+ got = np.concatenate([b.cpu().numpy().view(np.uint8) for b in Bf])[:al]
+ want = tgt[:al]
+ ok = bool(np.array_equal(got, want))
+ res.update(copy_s=t1 - t0, patch_s=t2 - t1, rebuild_s=t2 - t0, correct=ok,
+ mismatch_bytes=0 if ok else int((got != want).sum()),
+ sha256=hashlib.sha256(got.tobytes() + tgt[al:vb].tobytes()).hexdigest(),
+ target_sha256=hashlib.sha256(tgt[:vb].tobytes()).hexdigest())
+ del got
+ if res["target_in_host"]:
+ # the other path: upload the whole target from host memory into the same buffer
+ hostp = [torch.from_numpy(tgt[o:min(o + PIECE, al)].view(np.int16).copy()) for o in range(0, al, PIECE)]
+ sync()
+ t3 = time.perf_counter()
+ for dst_t, src_t in zip(Bf, hostp):
+ dst_t[:src_t.numel()].copy_(src_t)
+ sync()
+ res["upload_s"] = time.perf_counter() - t3
+ res["upload_correct"] = bool(np.array_equal(np.concatenate([b.cpu().numpy().view(np.uint8) for b in Bf])[:al], want))
+ res["measured_choice"] = "upload" if res["upload_s"] < res["rebuild_s"] else "rebuild"
+ with open(out, "w") as f:
+ json.dump(res, f, indent=1)
+ print(json.dumps({k: v for k, v in res.items() if k != "load"}))
+
+
+if __name__ == "__main__":
+ if sys.argv[1] == "analyze":
+ analyze(sys.argv[2], sys.argv[3])
+ elif sys.argv[1] == "inplace":
+ replay_inplace(sys.argv[2], sys.argv[3], sys.argv[4])
+ else:
+ replay(sys.argv[2], sys.argv[3], sys.argv[4])
diff --git a/tools/varmorph/host_restore.py b/tools/varmorph/host_restore.py
new file mode 100644
index 0000000..8b78846
--- /dev/null
+++ b/tools/varmorph/host_restore.py
@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+"""Restore a target file on the host from a sibling file and a copy list (runs.npz), timed.
+
+Modes:
+ new write the target into a new file: copy every run from the sibling, then the patch
+ blocks (blocks no run covers), then fsync. Same output contract as `xdelta3 -d`.
+ inplace turn a private copy of the sibling into the target in place, window by window in
+ target order, saving each window's original bytes in a ring first (as the device
+ rebuild does); runs whose move is zero are not touched; then the patch; then fsync.
+The patch bytes are taken from the target file at the blocks no run covers (they stand for the
+residual and new content that the descriptor carries). The result is compared with the target's
+SHA-256. The copy of the sibling for `inplace` is made before the clock and is not timed.
+argv: MODE RUNS.npz WORKDIR OUT.json
+"""
+import hashlib
+import json
+import os
+import shutil
+import sys
+import time
+
+import numpy as np
+
+BLK = 64
+WIN = 64 << 20
+RING = 256 << 20
+
+
+def sha(path):
+ h = hashlib.sha256()
+ with open(path, "rb") as f:
+ while True:
+ b = f.read(64 << 20)
+ if not b:
+ break
+ h.update(b)
+ return h.hexdigest()
+
+
+def main():
+ mode, runs_path, work, out = sys.argv[1:5]
+ r = np.load(runs_path)
+ dst, src, ln = r["dst"].astype(np.int64), r["src"].astype(np.int64), r["len"].astype(np.int64)
+ ref, tgt = str(r["refs"][0]), str(r["target"])
+ vb = int(r["v_bytes"])
+ al = vb // BLK * BLK
+ T = np.memmap(tgt, dtype=np.uint8, mode="r")
+ need = np.ones(al // BLK, dtype=bool)
+ for d, l in zip(dst.tolist(), ln.tolist()):
+ need[d // BLK:(d + l) // BLK] = False
+ other = r["other"]
+ need[other[other < al // BLK]] = True
+ patch_blocks = np.nonzero(need)[0]
+ brk = np.nonzero(np.diff(patch_blocks) != 1)[0] + 1
+ pr = list(zip(np.concatenate([[0], brk]).tolist(), np.concatenate([brk, [len(patch_blocks)]]).tolist()))
+ if "patch_data" in r.files:
+ # bytes carried by the descriptor (vm_apply.py expand): the target file is not read
+ pd, oth = r["patch_data"], r["other"]
+ pos = {int(b): i for i, b in enumerate(oth.tolist())}
+ assert all(int(b) in pos for b in patch_blocks.tolist()), "descriptor does not cover every block"
+ patch = []
+ for a, b in pr:
+ i0 = pos[int(patch_blocks[a])]
+ patch.append((int(patch_blocks[a]) * BLK, pd[i0 * BLK:(i0 + (b - a)) * BLK]))
+ tail = r["tail"]
+ else:
+ patch = [(int(patch_blocks[a]) * BLK, np.array(T[int(patch_blocks[a]) * BLK:int(patch_blocks[b - 1]) * BLK + BLK]))
+ for a, b in pr]
+ tail = np.array(T[al:vb])
+ os.makedirs(work, exist_ok=True)
+ path = os.path.join(work, os.path.basename(tgt) + "." + mode)
+ res = dict(mode=mode, runs=runs_path, ref=ref, target=tgt, v_bytes=vb, n_runs=int(len(dst)),
+ patch_bytes=int(len(patch_blocks) * BLK), patch_ranges=len(pr),
+ load=os.getloadavg())
+ if mode == "inplace":
+ shutil.copyfile(ref, path)
+ os.truncate(path, max(vb, os.path.getsize(ref)))
+ with open(path, "rb") as f:
+ os.fsync(f.fileno())
+ order = np.argsort(dst, kind="stable")
+ dst, src, ln = dst[order], src[order], ln[order]
+ t0 = time.perf_counter()
+ written = 0
+ if mode == "new":
+ R = np.memmap(ref, dtype=np.uint8, mode="r")
+ O = np.memmap(path, dtype=np.uint8, mode="w+", shape=(vb,))
+ for d, s_, l in zip(dst.tolist(), src.tolist(), ln.tolist()):
+ O[d:d + l] = R[s_:s_ + l]
+ for off, b in patch:
+ O[off:off + len(b)] = b
+ O[al:vb] = tail
+ written = vb
+ else:
+ O = np.memmap(path, dtype=np.uint8, mode="r+")
+ nslots = RING // WIN + 1
+ ring = [None] * nslots
+ # far sources (before the ring) are saved before the first write
+ pieces = []
+ for d, s_, l in zip(dst.tolist(), src.tolist(), ln.tolist()):
+ while l > 0:
+ take = min(l, WIN - d % WIN)
+ pieces.append((d, s_, take))
+ d, s_, l = d + take, s_ + take, l - take
+ far = {}
+ for d, s_, l in pieces:
+ w = d // WIN
+ lo = max(w * WIN - (nslots - 1) * WIN, 0)
+ if s_ < lo and d != s_:
+ far[(s_, l)] = np.array(O[s_:s_ + l])
+ by_w = {}
+ for p in pieces:
+ by_w.setdefault(p[0] // WIN, []).append(p)
+ last = (al - 1) // WIN
+ for w in range(last + 1):
+ a = w * WIN
+ lst = [p for p in by_w.get(w, []) if p[0] != p[1]]
+ ring[w % nslots] = (a, np.array(O[a:a + WIN])) if lst or True else None
+ for d, s_, l in lst:
+ if s_ >= a + WIN:
+ O[d:d + l] = O[s_:s_ + l]
+ elif (s_, l) in far:
+ O[d:d + l] = far[(s_, l)]
+ else:
+ x, dd, rem = s_, d, l
+ while rem > 0:
+ if x >= a + WIN: # not yet overwritten
+ O[dd:dd + rem] = O[x:x + rem]
+ break
+ sw = x // WIN
+ base, buf = ring[sw % nslots]
+ take = min(rem, base + WIN - x)
+ O[dd:dd + take] = buf[x - base:x - base + take]
+ x, dd, rem = x + take, dd + take, rem - take
+ written += l
+ for off, b in patch:
+ O[off:off + len(b)] = b
+ written += len(b)
+ O[al:vb] = tail
+ O.flush()
+ del O
+ with open(path, "rb+") as f:
+ if mode == "inplace":
+ f.truncate(vb)
+ os.fsync(f.fileno())
+ t1 = time.perf_counter()
+ res.update(restore_s=t1 - t0, bytes_written=int(written))
+ res["sha256"] = sha(path)
+ res["target_sha256"] = sha(tgt)
+ res["correct"] = res["sha256"] == res["target_sha256"]
+ os.remove(path)
+ with open(out, "w") as f:
+ json.dump(res, f, indent=1)
+ print(json.dumps({k: v for k, v in res.items() if k != "load"}))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/varmorph/layout_coder.py b/tools/varmorph/layout_coder.py
new file mode 100644
index 0000000..2a7ce91
--- /dev/null
+++ b/tools/varmorph/layout_coder.py
@@ -0,0 +1,559 @@
+#!/usr/bin/env python3
+"""Cohort 2026-11-12 (10): a video-coding-style lossless coder for a compiled variant against siblings.
+
+A target variant v is coded against reference variants (the siblings a node may hold), the way a
+P-frame is coded against reference frames. Tools taken from video coding, and why:
+
+Matching
+- block matching: v is split into 64-byte blocks, and each block is searched in the references by
+ exact hash, verified byte by byte. Exact matching replaces a SAD search, since the result must be
+ lossless.
+- sub-block motion (the "sub-pel" step): content moves by offsets that are not multiples of 64
+ bytes (a first, aligned-only run matched 211 KB of a prefill pair that content-defined chunks
+ share in GB). A rolling hash over every byte position of the references finds a sample of target
+ blocks. The phases (offset mod 64) of those shifts are then indexed, so the main search stays one
+ lookup per block.
+- MV prediction: the block at the predicted position (the previous match plus the distance since)
+ is compared first. A hit is a "skip": no lookup and no new vector.
+- variable blocks: consecutive blocks whose references are consecutive merge into one run.
+
+Coding
+- MVD + entropy: each run is coded as its length and the difference of its motion vector from the
+ previous one (zigzag varints), then entropy-coded (zstd).
+- inter residual: a block that matches nothing exactly but differs from its predicted block in at
+ most T bytes is coded as the XOR with that block.
+- intra: any other block is raw content. Its bytes are coded with zstd, alone and after splitting
+ 2-byte words into byte planes (bf16).
+- fill (DC): a block of one repeated 8-byte word is coded as that word.
+- multi-reference: several references share one index; a vector names its reference through the
+ global offset.
+
+Not taken: transform and quantization (lossy) and loop filters.
+
+Every coded target is decoded and compared with v byte for byte before its size is reported. The
+decode is timed: a single-threaded rebuild into host memory from references already in memory.
+argv: OUT.json T REF[,REF...] TARGET [REF[,REF...] TARGET ...]
+"""
+import json
+import os
+import shutil
+import subprocess
+import sys
+import time
+
+import numpy as np
+
+try:
+ from numba import njit
+except ImportError: # slow fallback for small tests
+ def njit(*a, **k):
+ if a and callable(a[0]):
+ return a[0]
+ return lambda f: f
+
+B = 64
+W = B // 8
+K = (np.random.default_rng(12).integers(1, 2**63, size=W, dtype=np.uint64) | np.uint64(1))
+OP_MATCH, OP_FILL, OP_RES, OP_INTRA = 0, 1, 2, 3
+BASE = np.uint64(1099511628211)
+SAMPLES = 1 << 16
+NMERGE = int(os.environ.get("VCODEC_NMERGE", "4")) # merge candidates (0: off)
+NOMV = -(1 << 62)
+BITS = 26
+
+
+def log(*a):
+ print(time.strftime("%H:%M:%S"), *a, flush=True)
+
+
+def block_hashes(buf):
+ n = len(buf) // B
+ h = np.empty(n, dtype=np.uint64)
+ step = (1 << 28) // B
+ for i in range(0, n, step):
+ j = min(n, i + step)
+ w = np.frombuffer(buf[i * B:j * B].tobytes(), dtype=np.uint64).reshape(-1, W) if buf.ctypes.data % 8 else \
+ buf[i * B:j * B].view(np.uint64).reshape(-1, W)
+ h[i:j] = ((w ^ (w >> np.uint64(29))) * K).sum(axis=1, dtype=np.uint64)
+ return h
+
+
+@njit(cache=True)
+def is_fill(V, o):
+ for k in range(8, B):
+ if V[o + k] != V[o + (k & 7)]:
+ return False
+ return True
+
+
+@njit(cache=True)
+def rk(V, o):
+ h = np.uint64(0)
+ for k in range(B):
+ h = h * BASE + np.uint64(V[o + k])
+ return h
+
+
+@njit(cache=True)
+def find_shifts(R, V, sample_off, cap):
+ """Rolling hash over every byte of R; returns (ref_off, tgt_off) of verified sample hits."""
+ m = len(sample_off)
+ sh = np.empty(m, dtype=np.uint64)
+ for s in range(m):
+ sh[s] = rk(V, sample_off[s])
+ order = np.argsort(sh)
+ ss = sh[order]
+ so = sample_off[order]
+ bitmap = np.zeros(1 << (BITS - 3), dtype=np.uint8)
+ MIX = np.uint64(0x9E3779B97F4A7C15)
+ for s in range(m):
+ b = (ss[s] * MIX) >> np.uint64(64 - BITS)
+ bitmap[b >> np.uint64(3)] |= np.uint8(1) << np.uint8(b & np.uint64(7))
+ pw = np.uint64(1)
+ for k in range(B):
+ pw = pw * BASE
+ out_r = np.empty(cap, dtype=np.int64)
+ out_t = np.empty(cap, dtype=np.int64)
+ nout = 0
+ n = len(R)
+ if n < B:
+ return out_r[:0], out_t[:0]
+ h = rk(R, 0)
+ pos = 0
+ while True:
+ b = (h * MIX) >> np.uint64(64 - BITS)
+ if bitmap[b >> np.uint64(3)] & (np.uint8(1) << np.uint8(b & np.uint64(7))):
+ lo, hi = 0, m
+ while lo < hi:
+ mid = (lo + hi) >> 1
+ if ss[mid] < h:
+ lo = mid + 1
+ else:
+ hi = mid
+ c = lo
+ while c < m and ss[c] == h and nout < cap:
+ t = so[c]
+ eq = True
+ for k in range(B):
+ if R[pos + k] != V[t + k]:
+ eq = False
+ break
+ if eq:
+ out_r[nout] = pos
+ out_t[nout] = t
+ nout += 1
+ c += 1
+ if pos + B >= n:
+ break
+ h = h * BASE + np.uint64(R[pos + B]) - np.uint64(R[pos]) * pw
+ pos += 1
+ return out_r[:nout], out_t[:nout]
+
+
+@njit(cache=True)
+def scan(R, V, sr, spos, hv, T):
+ """Choose a mode per target block. ref holds a byte offset in R (or -1)."""
+ n = len(hv)
+ nr = len(sr)
+ nR = len(R)
+ mode = np.empty(n, dtype=np.uint8)
+ ref = np.full(n, -1, dtype=np.int64)
+ last_i = -1
+ last_r = -1
+ lookups = 0
+ skips = 0
+ merges = 0
+ mc = np.full(NMERGE, NOMV, dtype=np.int64)
+ for i in range(n):
+ o = i * B
+ if is_fill(V, o):
+ mode[i] = OP_FILL
+ continue
+ p = -1
+ if last_r >= 0:
+ p = last_r + (i - last_i) * B
+ if p + B > nR:
+ p = -1
+ nz = B + 1
+ if p >= 0:
+ nz = 0
+ for k in range(B):
+ if R[p + k] != V[o + k]:
+ nz += 1
+ if nz == 0:
+ mode[i] = OP_MATCH
+ ref[i] = p
+ last_i = i
+ last_r = p
+ skips += 1
+ continue
+ # merge candidates (as in HEVC merge mode): the motions of the last few runs, tried
+ # before any hash lookup so that a block found elsewhere does not break the motion field
+ mfound = -1
+ for c_ in range(NMERGE):
+ if mc[c_] == NOMV:
+ continue
+ q = o + mc[c_]
+ if q < 0 or q + B > nR or q == p:
+ continue
+ eq = True
+ for k in range(B):
+ if R[q + k] != V[o + k]:
+ eq = False
+ break
+ if eq:
+ mfound = q
+ break
+ if mfound >= 0:
+ mode[i] = OP_MATCH
+ ref[i] = mfound
+ last_i = i
+ last_r = mfound
+ merges += 1
+ continue
+ lookups += 1
+ lo, hi = 0, nr
+ x = hv[i]
+ while lo < hi:
+ mid = (lo + hi) >> 1
+ if sr[mid] < x:
+ lo = mid + 1
+ else:
+ hi = mid
+ found = -1
+ c = lo
+ while c < nr and c < lo + 8 and sr[c] == x:
+ q = spos[c]
+ eq = True
+ for k in range(B):
+ if R[q + k] != V[o + k]:
+ eq = False
+ break
+ if eq:
+ found = q
+ break
+ c += 1
+ if found >= 0:
+ mode[i] = OP_MATCH
+ ref[i] = found
+ last_i = i
+ last_r = found
+ d_ = found - o
+ if NMERGE > 0 and mc[0] != d_:
+ for c_ in range(NMERGE - 1, 0, -1):
+ mc[c_] = mc[c_ - 1]
+ mc[0] = d_
+ continue
+ if p >= 0 and nz <= T:
+ mode[i] = OP_RES
+ ref[i] = p
+ last_i = i
+ last_r = p
+ continue
+ mode[i] = OP_INTRA
+ return mode, ref, lookups, skips, merges
+
+
+@njit(cache=True)
+def put_varint(buf, pos, x):
+ while x >= 0x80:
+ buf[pos] = (x & 0x7F) | 0x80
+ x >>= 7
+ pos += 1
+ buf[pos] = x
+ return pos + 1
+
+
+@njit(cache=True)
+def build(R, V, mode, ref):
+ """Merge blocks into runs; write the op stream, the residual stream and the intra stream."""
+ n = len(mode)
+ ops = np.empty(n * 12 + 64, dtype=np.uint8)
+ res = np.empty(np.sum(mode == OP_RES) * B + 1, dtype=np.uint8)
+ intra = np.empty(np.sum(mode == OP_INTRA) * B + 1, dtype=np.uint8)
+ op = 0
+ rp = 0
+ ip = 0
+ prev_d = 0
+ runs = np.zeros(4, dtype=np.int64)
+ i = 0
+ while i < n:
+ m = mode[i]
+ j = i + 1
+ if m == OP_MATCH or m == OP_RES:
+ while j < n and mode[j] == m and ref[j] == ref[j - 1] + B:
+ j += 1
+ elif m == OP_FILL:
+ while j < n and mode[j] == OP_FILL:
+ eq = True
+ for k in range(8):
+ if V[j * B + k] != V[i * B + k]:
+ eq = False
+ break
+ if not eq:
+ break
+ j += 1
+ else:
+ while j < n and mode[j] == OP_INTRA:
+ j += 1
+ L = j - i
+ runs[m] += 1
+ ops[op] = m
+ op += 1
+ op = put_varint(ops, op, L)
+ if m == OP_MATCH or m == OP_RES:
+ d = ref[i] - i * B
+ dd = d - prev_d
+ op = put_varint(ops, op, (dd << 1) ^ (dd >> 63))
+ prev_d = d
+ if m == OP_RES:
+ for b in range(i, j):
+ for k in range(B):
+ res[rp] = R[ref[b] + k] ^ V[b * B + k]
+ rp += 1
+ elif m == OP_FILL:
+ for k in range(8):
+ ops[op] = V[i * B + k]
+ op += 1
+ else:
+ intra[ip:ip + L * B] = V[i * B:j * B]
+ ip += L * B
+ i = j
+ return ops[:op], res[:rp], intra[:ip], runs
+
+
+@njit(cache=True)
+def decode(R, ops, res, intra, n, tail):
+ out = np.empty(n * B + len(tail), dtype=np.uint8)
+ op = 0
+ rp = 0
+ ip = 0
+ prev_d = 0
+ i = 0
+ while op < len(ops):
+ m = ops[op]
+ op += 1
+ L = 0
+ s = 0
+ while True:
+ b = np.int64(ops[op])
+ op += 1
+ L |= (b & 0x7F) << s
+ s += 7
+ if b < 0x80:
+ break
+ a = i * B
+ e = a + L * B
+ if m == OP_MATCH or m == OP_RES:
+ z = 0
+ s = 0
+ while True:
+ b = np.int64(ops[op])
+ op += 1
+ z |= (b & 0x7F) << s
+ s += 7
+ if b < 0x80:
+ break
+ d = prev_d + ((z >> 1) ^ -(z & 1))
+ prev_d = d
+ out[a:e] = R[a + d:e + d]
+ if m == OP_RES:
+ for k in range(L * B):
+ out[a + k] ^= res[rp + k]
+ rp += L * B
+ elif m == OP_FILL:
+ for k in range(a, e):
+ out[k] = ops[op + ((k - a) & 7)]
+ op += 8
+ else:
+ out[a:e] = intra[ip:ip + L * B]
+ ip += L * B
+ i += L
+ out[n * B:] = tail
+ return out
+
+
+def zsize(arr, level=19):
+ if len(arr) == 0:
+ return 0
+ if shutil.which("zstd"):
+ lv = ["--ultra", "-22"] if level > 19 else ["-%d" % level]
+ p = subprocess.run(["zstd", "-q", "-c", "-T16", "--long=27"] + lv, input=arr.tobytes(), capture_output=True, check=True)
+ return len(p.stdout)
+ try:
+ import zstandard
+ return len(zstandard.ZstdCompressor(level=min(level, 19), threads=16).compress(arr.tobytes()))
+ except ImportError:
+ import lzma
+ return len(lzma.compress(arr.tobytes(), preset=6))
+
+
+def intra_size(arr):
+ """Best of zstd on the stream and zstd on its two byte planes (bf16 split)."""
+ plain = zsize(arr, 9)
+ if len(arr) < 2:
+ return plain, "zstd"
+ a = arr[: len(arr) // 2 * 2]
+ planes = zsize(np.ascontiguousarray(a[0::2]), 9) + zsize(np.ascontiguousarray(a[1::2]), 9) + (len(arr) - len(a))
+ return (planes, "planes") if planes < plain else (plain, "zstd")
+
+
+def load(paths):
+ sizes = [os.path.getsize(p) for p in paths]
+ pads = [(-s) % B for s in sizes]
+ R = np.empty(sum(sizes) + sum(pads), dtype=np.uint8)
+ o = 0
+ for p, s, pd in zip(paths, sizes, pads):
+ with open(p, "rb") as f:
+ f.readinto(memoryview(R)[o:o + s])
+ R[o + s:o + s + pd] = 0
+ o += s + pd
+ return R
+
+
+def run_stats(mode, ref, nontrivial_bytes):
+ idx = np.nonzero(mode == OP_MATCH)[0]
+ if len(idx) == 0:
+ return {}
+ r = ref[idx]
+ brk = np.ones(len(idx), dtype=bool)
+ brk[1:] = (idx[1:] != idx[:-1] + 1) | (r[1:] != r[:-1] + B)
+ st = np.nonzero(brk)[0]
+ run_bytes = np.diff(np.append(st, len(idx))) * B
+ d = r[st] - idx[st] * B
+ cov = {str(L): float(run_bytes[run_bytes >= L].sum() / nontrivial_bytes)
+ for L in [64, 256, 1024, 4096, 16384, 65536, 1 << 20]}
+ ud, inv = np.unique(d, return_inverse=True)
+ w = np.bincount(inv, weights=run_bytes)
+ o = np.argsort(-w)
+ cw = np.cumsum(w[o]) / run_bytes.sum()
+ jd = np.diff(d)
+ out = {"match_runs": int(len(st)), "median_run_bytes": float(np.median(run_bytes)),
+ "coverage_by_run_ge": cov, "distinct_deltas": int(len(ud)),
+ "deltas_for_90pct": int(np.searchsorted(cw, 0.9) + 1),
+ "top_deltas": [{"delta": int(ud[k]), "share": float(w[k] / run_bytes.sum())} for k in o[:10]]}
+ if len(jd):
+ uj, invj = np.unique(jd, return_inverse=True)
+ wj = np.bincount(invj, weights=run_bytes[1:])
+ oj = np.argsort(-wj)
+ cj = np.cumsum(wj[oj]) / max(run_bytes[1:].sum(), 1)
+ out.update({"distinct_jumps": int(len(uj)), "jumps_for_90pct": int(np.searchsorted(cj, 0.9) + 1),
+ "top_jumps": [{"jump": int(uj[k]), "share": float(wj[k] / run_bytes[1:].sum())} for k in oj[:10]]})
+ return out
+
+
+def code_one(refs, target, T, modes_path=None):
+ t0 = time.time()
+ R = load(refs)
+ V = np.fromfile(target, dtype=np.uint8)
+ nb = len(V) // B
+ tail = V[nb * B:].copy()
+ Vb = V[: nb * B]
+ # sub-block motion: phases of the shifts found for a sample of target blocks
+ cand = np.linspace(0, nb - 1, SAMPLES * 2).astype(np.int64)
+ cand = np.array([c * B for c in cand if not is_fill(Vb, c * B)][:SAMPLES], dtype=np.int64)
+ hr_, ht_ = find_shifts(R, Vb, cand, 1 << 22)
+ shift = hr_ - ht_
+ ph = np.bincount(((shift % B) + B) % B, minlength=B)
+ hit_blocks = len(np.unique(ht_))
+ order_ph = np.argsort(-ph)
+ phases = [0]
+ acc = ph[0]
+ for p in order_ph:
+ if os.environ.get("VCODEC_ALIGNED") == "1": # ablation: match only at block-aligned offsets
+ break
+ if ph[p] == 0 or len(phases) >= 8 or acc >= 0.99 * max(ph.sum(), 1):
+ break
+ if p != 0:
+ phases.append(int(p))
+ acc += ph[p]
+ us, cs = np.unique(shift, return_counts=True)
+ oc = np.argsort(-cs)[:10]
+ t1 = time.time()
+ log("phases", phases, "sample hits", int(hit_blocks), "of", len(cand))
+ hs, ps = [], []
+ for p in phases:
+ n = (len(R) - p) // B
+ hs.append(block_hashes(R[p:p + n * B]))
+ ps.append(p + np.arange(n, dtype=np.int64) * B)
+ hr = np.concatenate(hs)
+ pr = np.concatenate(ps)
+ del hs, ps
+ order = np.argsort(hr, kind="stable")
+ sr = hr[order]
+ spos = pr[order]
+ del hr, pr, order
+ hv = block_hashes(Vb)
+ t2 = time.time()
+ mode, ref, lookups, skips, merges = scan(R, Vb, sr, spos, hv, T)
+ ops, res, intra, runs = build(R, Vb, mode, ref)
+ t3 = time.time()
+ decode(R[:1 << 20], ops[:0], res, intra, 0, tail[:0]) # compile outside the timed call
+ t4 = time.time()
+ out = decode(R, ops, res, intra, nb, tail)
+ t5 = time.time()
+ ok = bool(np.array_equal(out, V))
+ del out
+ log("decoded", ok, round(t5 - t4, 2), "s")
+ cnt = np.bincount(mode, minlength=4) * B
+ shipped = None
+ if modes_path and os.environ.get("VCODEC_SAVE"):
+ # the descriptor as one file a node can fetch: the three streams and the tail, zstd -19
+ dpath = modes_path.replace(".modes.npy", ".vmd.npz")
+ np.savez(dpath, ops=ops, res=res, intra=intra, tail=tail, nb=np.int64(nb), v_bytes=np.int64(len(V)),
+ refs=np.array(refs), target=np.array(target))
+ subprocess.run(["zstd", "-q", "-f", "-19", "-T16", "--long=27", dpath, "-o", dpath + ".zst"], check=True)
+ shipped = os.path.getsize(dpath + ".zst")
+ if modes_path and os.environ.get("VCODEC_DUMP"):
+ # the descriptor itself, for a device-side replay: match runs (target byte offset,
+ # reference byte offset, length) and the other blocks (target block index, mode)
+ idx = np.nonzero(mode == OP_MATCH)[0]
+ rr = ref[idx]
+ brk = np.ones(len(idx), dtype=bool)
+ brk[1:] = (idx[1:] != idx[:-1] + 1) | (rr[1:] != rr[:-1] + B)
+ st = np.nonzero(brk)[0]
+ ln = np.diff(np.append(st, len(idx))) * B
+ other = np.nonzero(mode != OP_MATCH)[0]
+ np.savez(modes_path.replace(".modes.npy", ".runs.npz"), dst=idx[st] * B, src=rr[st], len=ln,
+ other=other, other_mode=mode[other], v_bytes=len(V), refs=np.array(refs), target=np.array(target))
+ if modes_path: # per 64 KiB window: blocks coded as match / fill / residual / intra
+ nw = nb // 1024
+ wm = np.bincount((np.arange(nw * 1024) // 1024) * 4 + mode[:nw * 1024], minlength=nw * 4)
+ np.save(modes_path, wm.reshape(nw, 4).astype(np.uint32))
+ s_ops = zsize(ops, 19)
+ s_res = zsize(res, 19)
+ s_intra, intra_how = intra_size(intra)
+ total = s_ops + s_res + s_intra + len(tail)
+ nontriv = int(len(V) - cnt[1])
+ return {
+ "refs": refs, "target": target, "T": T, "v_bytes": int(len(V)), "lossless": ok,
+ "shipped_bytes": shipped, "aligned_only": os.environ.get("VCODEC_ALIGNED") == "1",
+ "sample": {"blocks": int(len(cand)), "hit_blocks": int(hit_blocks), "hits": int(len(shift)),
+ "phase_hist_top": {int(p): int(ph[p]) for p in order_ph[:8]}, "phases_indexed": phases,
+ "top_shifts": [{"shift": int(us[k]), "count": int(cs[k])} for k in oc]},
+ "bytes_by_mode": {"match": int(cnt[0]), "fill": int(cnt[1]), "residual": int(cnt[2]), "intra": int(cnt[3])},
+ "runs_by_mode": {"match": int(runs[0]), "fill": int(runs[1]), "residual": int(runs[2]), "intra": int(runs[3])},
+ "lookups": int(lookups), "skips": int(skips), "merges": int(merges), "nmerge": NMERGE,
+ "match_structure": run_stats(mode, ref, max(nontriv, 1)),
+ "coded": {"ops_raw": int(len(ops)), "ops": s_ops, "residual_raw": int(len(res)), "residual": s_res,
+ "intra_raw": int(len(intra)), "intra": s_intra, "intra_codec": intra_how, "total": total},
+ "seconds": {"shift_search": round(t1 - t0, 1), "index": round(t2 - t1, 1), "encode": round(t3 - t2, 1),
+ "decode_host": round(t5 - t4, 2)},
+ "decode_GBps": round(len(V) / max(t5 - t4, 1e-9) / 1e9, 2),
+ }
+
+
+if __name__ == "__main__":
+ out, T = sys.argv[1], int(sys.argv[2])
+ a = sys.argv[3:]
+ res = []
+ for k in range(0, len(a), 2):
+ refs = a[k].split(",")
+ log("code", refs, "->", a[k + 1])
+ res.append(code_one(refs, a[k + 1], T, "%s.%d.modes.npy" % (out, k // 2)))
+ with open(out, "w") as f:
+ json.dump(res, f, indent=1)
+ r = res[-1]
+ log("result", json.dumps(r["coded"]), json.dumps(r["bytes_by_mode"]), json.dumps(r["sample"]["phases_indexed"]),
+ "decode", r["seconds"]["decode_host"])
+ log("ALL-DONE")
diff --git a/tools/varmorph/npu_d2d_bench.py b/tools/varmorph/npu_d2d_bench.py
new file mode 100644
index 0000000..4c19591
--- /dev/null
+++ b/tools/varmorph/npu_d2d_bench.py
@@ -0,0 +1,210 @@
+#!/usr/bin/env python3
+"""NPU micro-benchmark (NPU-A or NPU-B): can a target's weights be rebuilt on the device from
+resident weights?
+
+A video-style descriptor turns a variant into copies of runs from a sibling. This measures what
+such a decode costs on the card, against uploading the weights from the host. Cases:
+ h2d_ contiguous host -> device copy (the upload a restart pays)
+ d2d_ contiguous device -> device copy of one tensor into another
+ d2d_calls 512 device copies of one 32 KiB run each, one call per run
+ d2d_strided one call copying every other 32 KiB run of a 64 MiB region (a regular re-tiling
+ as one descriptor)
+ d2d_gather one call gathering the 32 KiB runs of a 64 MiB region in a random order
+ (index_select)
+ use_256 a device reduction over a 256 MiB tensor already on the device
+ d2duse_256 a device copy into that tensor, then the same reduction (timed together): if the
+ copy left the data on the host side, the reduction pays the upload
+ h2duse_256 a host upload into that tensor, then the reduction
+
+Every case runs in its own process, so a device error in one case does not leave the card in a bad
+state for the next. Each case syncs the card around the timed region and repeats REPS times after
+a warm-up. Values are finite bf16 (from a normal distribution), and results are compared as int16
+bit patterns: a fast path that changes the bytes is reported as wrong.
+argv: OUT.json PLATFORM(rngd|rbln) [REPS, default 5] (driver)
+ --case NAME PLATFORM REPS OUT_CASE.json (one case, internal)
+"""
+import json
+import os
+import subprocess
+import sys
+import time
+import traceback
+
+RUN = 16384 # bf16 elements per 32 KiB run
+CASES = ["h2d_64", "h2d_1024", "d2d_1", "d2d_64", "d2d_256", "d2d_1024",
+ "d2d_calls", "d2d_strided", "d2d_gather",
+ "use_256", "d2duse_256", "h2duse_256"]
+if os.environ.get("RBLN_VMSHIM"): # NPU-B batched copies through the public runtime API
+ CASES += ["v2vmulti_1", "v2vmulti_64", "v2vmulti_1024", "v2vmultis_1024", "h2vmulti_1024"]
+
+
+def device(platform):
+ import torch
+ if platform == "cpu": # local test of the drivers
+ return "cpu", (lambda: None), ("torch",)
+ if platform == "rngd":
+ from rngd_pd import devices
+ from rngd_pd.engines.fsdk import dev_str, open_device, sync
+ chip = devices.resolve_chip(-1)
+ dev = dev_str(chip)
+ open_device(8, 2)
+ return dev, (lambda: sync(chip)), ("torch", "furiosa-torch")
+ import rebel # noqa: F401
+ import torch_rbln # noqa: F401 registers the rbln device
+ dev = "rbln:%d" % int(os.environ.get("RBLN_BENCH_DEVICE", "0"))
+ return dev, (lambda: torch.rbln.synchronize(dev)), ("torch", "rebel-compiler")
+
+
+def run_case(name, platform, reps, out):
+ import torch
+ from importlib.metadata import version
+ torch.set_num_threads(4)
+ row = dict(case=name, s=[])
+ try:
+ dev, sync, pk = device(platform)
+ row.update(device=dev, packages={k: version(k) for k in pk})
+ g = torch.Generator().manual_seed(7)
+ bits = lambda t: t.view(torch.int16)
+
+ def host(nelem):
+ return torch.randn(nelem, generator=g, dtype=torch.float32).to(torch.bfloat16)
+
+ kind, _, arg = name.partition("_")
+ if kind in ("v2vmulti", "v2vmultis", "h2vmulti"):
+ import ctypes
+ lib = ctypes.CDLL(os.environ["RBLN_VMSHIM"])
+ k = int(arg)
+ rows = 2048 # 64 MiB region of 32 KiB runs
+ n = rows * RUN
+ h = host(n)
+ src = h.to(dev)
+ dst = torch.empty(n, dtype=torch.bfloat16, device=dev)
+ dst.copy_(src) # give dst a device view
+ sync()
+ if kind != "h2vmulti":
+ row["layout_like_rc"] = lib.vmshim_layout_like(ctypes.c_uint64(dst.data_ptr()), ctypes.c_uint64(src.data_ptr()))
+ perm = torch.randperm(rows, generator=g)[:k].tolist()
+ U64 = ctypes.c_uint64 * k
+ sz = U64(*([RUN * 2] * k))
+ dsta = U64(*[dst.data_ptr() + i * RUN * 2 * (2 if kind == "v2vmultis" else 1) for i in range(k)])
+ if kind == "h2vmulti":
+ hb = h.view(rows, RUN)
+ srca = U64(*[hb[r].data_ptr() for r in perm])
+ fn = lambda: lib.vmshim_h2v_multi(srca, dsta, sz, k)
+ else:
+ srca = U64(*[src.data_ptr() + r * RUN * 2 for r in perm])
+ fn = lambda: lib.vmshim_v2v_multi(srca, dsta, sz, k)
+ nbytes = k * RUN * 2
+ row["calls"] = 1
+ step = 2 if kind == "v2vmultis" else 1
+ check = lambda: all(torch.equal(bits(dst[(i * step) * RUN:(i * step + 1) * RUN].cpu()), bits(h[perm[i] * RUN:(perm[i] + 1) * RUN]))
+ for i in sorted({0, k // 2, k - 1}))
+ rc = fn()
+ row["rc"] = rc
+ elif kind in ("use", "d2duse", "h2duse"):
+ n = int(arg) * (1 << 20) // 2
+ h = host(n)
+ src = h.to(dev)
+ dst = torch.empty(n, dtype=torch.bfloat16, device=dev)
+ dst.copy_(src)
+ sync()
+ red = lambda: dst.sum()
+ if kind == "use":
+ fn = lambda: red()
+ elif kind == "d2duse":
+ fn = lambda: (dst.copy_(src), red())
+ else:
+ fn = lambda: (dst.copy_(h), red())
+ nbytes = n * 2
+ check = lambda: torch.equal(bits(dst[-RUN:].cpu()), bits(h[-RUN:]))
+ elif kind == "h2d":
+ n = int(arg) * (1 << 20) // 2
+ h = host(n)
+ dst = torch.empty(n, dtype=torch.bfloat16, device=dev)
+ fn = lambda: dst.copy_(h)
+ nbytes = n * 2
+ check = lambda: torch.equal(bits(dst[-RUN:].cpu()), bits(h[-RUN:]))
+ else:
+ n = (int(arg) if arg.isdigit() else 64) * (1 << 20) // 2
+ n = max(n // RUN, 2) * RUN
+ rows = n // RUN
+ h = host(n)
+ src = h.to(dev)
+ dst = torch.empty(n, dtype=torch.bfloat16, device=dev)
+ sync()
+ if arg.isdigit():
+ fn = lambda: dst.copy_(src)
+ nbytes = n * 2
+ check = lambda: torch.equal(bits(dst[-RUN:].cpu()), bits(h[-RUN:]))
+ elif arg == "calls":
+ k = min(512, rows)
+ perm = torch.randperm(rows, generator=g)[:k].tolist()
+
+ def fn():
+ for i, r in enumerate(perm):
+ dst[i * RUN:(i + 1) * RUN].copy_(src[r * RUN:(r + 1) * RUN])
+ nbytes = k * RUN * 2
+ row["calls"] = k
+ check = lambda: torch.equal(bits(dst[5 * RUN:6 * RUN].cpu()), bits(h[perm[5] * RUN:(perm[5] + 1) * RUN]))
+ elif arg == "strided":
+ s2, d2 = src.view(rows // 2, 2, RUN), dst.view(rows // 2, 2, RUN)
+ fn = lambda: d2[:, 0, :].copy_(s2[:, 1, :])
+ nbytes = (rows // 2) * RUN * 2
+ check = lambda: torch.equal(bits(d2[:, 0, :].cpu()), bits(h.view(rows // 2, 2, RUN)[:, 1, :]))
+ else: # gather
+ pidx = torch.randperm(rows, generator=g)
+ pdev = pidx.to(dev)
+ sv, dv = src.view(rows, RUN), dst.view(rows, RUN)
+ fn = lambda: dv.copy_(torch.index_select(sv, 0, pdev))
+ nbytes = n * 2
+ check = lambda: torch.equal(bits(dv[7].cpu()), bits(h.view(rows, RUN)[pidx[7]]))
+ row["bytes"] = nbytes
+ fn()
+ sync()
+ for _ in range(reps):
+ sync()
+ t = time.perf_counter()
+ fn()
+ sync()
+ row["s"].append(time.perf_counter() - t)
+ row["median_s"] = sorted(row["s"])[len(row["s"]) // 2]
+ row["GBps"] = nbytes / row["median_s"] / 1e9
+ if "calls" in row:
+ row["us_per_call"] = row["median_s"] / row["calls"] * 1e6
+ row["correct"] = bool(check())
+ except Exception as e:
+ row["error"] = repr(e)[:400]
+ row["trace"] = traceback.format_exc()[-1500:]
+ with open(out, "w") as f:
+ json.dump(row, f, indent=1)
+
+
+def driver(out, platform, reps):
+ res = dict(schema="npu-d2d.v2", platform=platform, reps=reps, load=os.getloadavg(), cases={})
+ for name in CASES:
+ part = out + "." + name + ".json"
+ t = time.time()
+ try:
+ p = subprocess.run([sys.executable, __file__, "--case", name, platform, str(reps), part],
+ capture_output=True, text=True, timeout=600)
+ row = json.load(open(part)) if os.path.exists(part) else dict(error="no output", rc=p.returncode)
+ if p.returncode:
+ row["rc"] = p.returncode
+ row["stderr"] = p.stderr[-800:]
+ except subprocess.TimeoutExpired:
+ row = dict(error="timeout 600 s")
+ row["wall_s"] = round(time.time() - t, 1)
+ res["cases"][name] = row
+ with open(out, "w") as f:
+ json.dump(res, f, indent=1)
+ print(name, {k: v for k, v in row.items() if k not in ("s", "trace", "stderr")}, flush=True)
+ res["finished"] = time.time()
+ with open(out, "w") as f:
+ json.dump(res, f, indent=1)
+
+
+if __name__ == "__main__":
+ if sys.argv[1] == "--case":
+ run_case(sys.argv[2], sys.argv[3], int(sys.argv[4]), sys.argv[5])
+ else:
+ driver(sys.argv[1], sys.argv[2], int(sys.argv[3]) if len(sys.argv) > 3 else 5)
diff --git a/tools/varmorph/rbln_vmshim.cpp b/tools/varmorph/rbln_vmshim.cpp
new file mode 100644
index 0000000..547599c
--- /dev/null
+++ b/tools/varmorph/rbln_vmshim.cpp
@@ -0,0 +1,38 @@
+// C shim over the public NPU-B runtime API (rebel/runtime/api/rbln_runtime_api.h, SDK 0.12.0) for
+// batched copies, which torch-rbln does not bind to Python. Loaded with ctypes inside a process
+// that has imported torch_rbln; tensor data pointers are vmem virtual addresses.
+// g++ -O2 -shared -fPIC -I$SP/rebel/include rbln_vmshim.cpp -o librbln_vmshim.so \
+// -L$SP/tvm -l:librbln.so -Wl,-rpath,$SP/tvm
+#include
+
+#include
+#include
+#include
+
+extern "C" {
+
+// n copies (src_vaddr[i] -> dst_vaddr[i], size[i]); on the device for <= 1024 per destination.
+int vmshim_v2v_multi(const uint64_t* src, const uint64_t* dst, const uint64_t* size, int n) {
+ std::vector> v;
+ v.reserve(n);
+ for (int i = 0; i < n; ++i) v.emplace_back(src[i], dst[i], size[i]);
+ return static_cast(rbln::rbln_memcpy_v2v_multi(v));
+}
+
+// n patches (host pointer -> dst_vaddr, size).
+int vmshim_h2v_multi(const uint64_t* host, const uint64_t* dst, const uint64_t* size, int n) {
+ std::vector> v;
+ v.reserve(n);
+ for (int i = 0; i < n; ++i) v.emplace_back(static_cast(host[i]), dst[i], size[i]);
+ return static_cast(rbln::rbln_memcpy_h2v_multi(v));
+}
+
+int vmshim_layout_like(uint64_t target, uint64_t ref) {
+ return static_cast(rbln::rbln_set_device_alloc_layout_like(target, ref));
+}
+
+int vmshim_sync(uint32_t torch_device_id) {
+ return static_cast(rbln::rbln_device_synchronize(torch_device_id));
+}
+
+} // extern "C"
diff --git a/tools/varmorph/reach.py b/tools/varmorph/reach.py
new file mode 100644
index 0000000..f577b73
--- /dev/null
+++ b/tools/varmorph/reach.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+"""Backward reach of the moves of a copy list and the scratch an in-place rebuild needs.
+argv: OUT.json RUNS.npz [RUNS.npz ...] (window 64 MiB, ring 256 MiB as in npu_replay.py)"""
+import json, sys
+import numpy as np
+WIN, RING = 64 << 20, 256 << 20
+res = []
+for p in sys.argv[2:]:
+ r = np.load(p)
+ d, s, l = r["dst"].astype(np.int64), r["src"].astype(np.int64), r["len"].astype(np.int64)
+ mv = s - d
+ moved = mv != 0
+ a = (d // WIN) * WIN # start of the window being written
+ back = np.where(moved, a - s, 0) # > 0: the source lies before the window (already overwritten)
+ ring_lo = np.maximum(a - RING, 0)
+ far = moved & (s < ring_lo)
+ far_bytes = int(np.minimum(l[far], ring_lo[far] - s[far]).sum())
+ res.append(dict(runs=p, target=str(r["target"]), n_runs=int(len(d)), keep_bytes=int(l[~moved].sum()),
+ moved_bytes=int(l[moved].sum()), max_back_bytes=int(back.max()) if len(back) else 0,
+ back_p99_bytes=int(np.percentile(back[moved], 99)) if moved.any() else 0,
+ far_runs=int(far.sum()), far_bytes=far_bytes, scratch_bytes=RING + WIN + far_bytes,
+ v_bytes=int(r["v_bytes"]), ref_bytes=None))
+json.dump(res, open(sys.argv[1], "w"), indent=1)
+for x in res: print(json.dumps(x))
diff --git a/tools/varmorph/vcdiff_to_runs.py b/tools/varmorph/vcdiff_to_runs.py
new file mode 100644
index 0000000..f90481f
--- /dev/null
+++ b/tools/varmorph/vcdiff_to_runs.py
@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+"""Turn an xdelta3 (VCDIFF) patch into the copy list that the device rebuild reads (runs.npz).
+
+Reads `xdelta3 printdelta PATCH` on stdin. printdelta prints the target offset and S@a
+(source offset) as absolute values, and T@a relative to the start of the current VCDIFF window
+("window at offset"); a T@ copy repeats bytes already decoded in that window. A copy from the target (T@) is resolved to the source bytes it repeats,
+piece by piece, so the result is generous to the delta: every byte a copy chain carries from the
+source becomes a run. ADD and RUN bytes, and copies that resolve to them, have no source and go
+to the patch. Runs are trimmed to whole 64-byte target blocks; blocks not fully covered form
+`other`. The instruction counts of the patch as written are stored too.
+argv: OUT.npz REF TARGET < printdelta.txt
+"""
+import bisect
+import json
+import os
+import sys
+
+import numpy as np
+
+BLK = 64
+
+
+class Window:
+ def __init__(self, t0, src0):
+ self.t0, self.src0 = t0, src0
+ self.st, self.en, self.kind, self.src = [], [], [], []
+
+ def emit(self, t, l, kind, s):
+ if l <= 0:
+ return
+ if self.st and self.en[-1] == t and self.kind[-1] == kind and \
+ (kind == 0 or self.src[-1] + (self.en[-1] - self.st[-1]) == s):
+ self.en[-1] = t + l
+ return
+ self.st.append(t)
+ self.en.append(t + l)
+ self.kind.append(kind)
+ self.src.append(s)
+
+
+def main():
+ out, ref, target = sys.argv[1], sys.argv[2], sys.argv[3]
+ vb = os.path.getsize(target)
+ st = dict(windows=0, cpy_s=0, cpy_t=0, add=0, run=0, cpy_s_bytes=0, cpy_t_bytes=0, add_bytes=0, run_bytes=0)
+ rd, rs, rl = [], [], []
+ w = None
+ next_t0, cw_off, tlen, win_at = 0, 0, 0, 0
+
+ def close(win):
+ if win is None:
+ return
+ for a, e, k, s in zip(win.st, win.en, win.kind, win.src):
+ if k == 1:
+ rd.append(win.t0 + a)
+ rs.append(win.src0 + s)
+ rl.append(e - a)
+
+ for line in sys.stdin:
+ if line.startswith("VCDIFF window number"):
+ close(w)
+ w = None
+ st["windows"] += 1
+ continue
+ if line.startswith("VCDIFF window at offset"):
+ win_at = int(line.split(":")[1])
+ continue
+ if line.startswith("VCDIFF copy window offset"):
+ cw_off = int(line.split(":")[1])
+ continue
+ if line.startswith("VCDIFF target window length"):
+ tlen = int(line.split(":")[1])
+ w = Window(0, 0) # printdelta prints absolute target offsets and addresses
+ next_t0 += tlen
+ cw_off = 0
+ continue
+ f = line.split()
+ if w is None or len(f) < 4 or not f[0].isdigit():
+ continue
+ t = int(f[0])
+ rest = f[2:]
+ while rest:
+ kind = rest[0]
+ if kind in ("ADD", "RUN"):
+ size, addr, rest = int(rest[1]), None, rest[2:]
+ st["add" if kind == "ADD" else "run"] += 1
+ st[("add" if kind == "ADD" else "run") + "_bytes"] += size
+ w.emit(t, size, 0, 0)
+ else:
+ size, addr, rest = int(rest[1]), rest[2], rest[3:]
+ if addr.startswith("S@"):
+ st["cpy_s"] += 1
+ st["cpy_s_bytes"] += size
+ w.emit(t, size, 1, int(addr[2:]))
+ else:
+ st["cpy_t"] += 1
+ st["cpy_t_bytes"] += size
+ p, done = win_at + int(addr[2:]), 0 # T@ is relative to the window
+ while done < size:
+ i = bisect.bisect_right(w.st, p) - 1
+ avail = min(w.en[i] - p, size - done, (t + done) - p)
+ if w.kind[i] == 1:
+ w.emit(t + done, avail, 1, w.src[i] + (p - w.st[i]))
+ else:
+ w.emit(t + done, avail, 0, 0)
+ p += avail
+ done += avail
+ t += size
+ close(w)
+ d = np.array(rd, np.int64)
+ s = np.array(rs, np.int64)
+ l = np.array(rl, np.int64)
+ st["source_runs_raw"] = int(len(d))
+ st["source_bytes"] = int(l.sum())
+ # merge across windows: consecutive runs with the same move
+ o = np.argsort(d, kind="stable")
+ d, s, l = d[o], s[o], l[o]
+ md, ms, ml = [], [], []
+ for a, b, c in zip(d.tolist(), s.tolist(), l.tolist()):
+ if md and md[-1] + ml[-1] == a and ms[-1] + ml[-1] == b:
+ ml[-1] += c
+ else:
+ md.append(a), ms.append(b), ml.append(c)
+ d, s, l = np.array(md, np.int64), np.array(ms, np.int64), np.array(ml, np.int64)
+ st["source_runs_merged"] = int(len(d))
+ # trim to whole target blocks
+ a = (d + BLK - 1) // BLK * BLK
+ e = (d + l) // BLK * BLK
+ keep = e > a
+ s = s + (a - d)
+ d, s, l = a[keep], s[keep], (e - a)[keep]
+ al = vb // BLK
+ covered = np.zeros(al, dtype=bool)
+ for a_, l_ in zip(d.tolist(), l.tolist()):
+ covered[a_ // BLK:(a_ + l_) // BLK] = True
+ other = np.nonzero(~covered)[0].astype(np.int64)
+ R = np.memmap(ref, dtype=np.uint8, mode="r")
+ T = np.memmap(target, dtype=np.uint8, mode="r")
+ bad = sum(1 for a_, b_, l_ in zip(d.tolist(), s.tolist(), l.tolist())
+ if not np.array_equal(R[b_:b_ + l_], T[a_:a_ + l_]))
+ st["runs_mismatched"] = int(bad)
+ st["runs"] = int(len(d))
+ st["run_bytes_trimmed"] = int(l.sum())
+ st["other_blocks"] = int(len(other))
+ st["v_bytes"] = vb
+ np.savez(out, dst=d, src=s, len=l, other=other, other_mode=np.zeros(len(other), np.uint8),
+ v_bytes=np.int64(vb), refs=np.array([ref]), target=np.array(target))
+ with open(out + ".json", "w") as fh:
+ json.dump(st, fh, indent=1)
+ print(json.dumps(st))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/varmorph/vm_apply.py b/tools/varmorph/vm_apply.py
new file mode 100644
index 0000000..55c6294
--- /dev/null
+++ b/tools/varmorph/vm_apply.py
@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+"""Apply a VarMorph descriptor file on a node: expand it, then restore the target file.
+
+ expand DESC.vmd.npz[.zst] REF OUT.runs.npz
+ Read the descriptor (ops, residual, intra, tail) and the sibling REF and write the copy list
+ the restorers read: match runs (dst, src, len) and, for every other block, its bytes
+ (fill, residual XOR the sibling block it predicts from, new content). Only the sibling
+ blocks that residuals predict from are read. Timed: decompress + expand.
+ new|inplace DESC REF WORKDIR OUT.json [COLD]
+ expand, then restore the target into a new file / over a private copy of the sibling
+ (host_restore.py), sync, and compare with the SHA-256 recorded in the descriptor's name
+ file (the target itself, read after the clock). COLD=1 evicts the sibling from the page
+ cache first (posix_fadvise DONTNEED).
+The copy of the sibling for `inplace` is made before the clock.
+"""
+import io
+import json
+import os
+import subprocess
+import sys
+import time
+
+import numpy as np
+from numba import njit
+
+B = 64
+OP_MATCH, OP_FILL, OP_RES, OP_INTRA = 0, 1, 2, 3
+
+
+@njit(cache=True)
+def count(ops):
+ """Number of match runs, and of non-match blocks."""
+ op, nm, nb_other = 0, 0, 0
+ while op < len(ops):
+ m = ops[op]
+ op += 1
+ L, s = 0, 0
+ while True:
+ b = np.int64(ops[op])
+ op += 1
+ L |= (b & 0x7F) << s
+ s += 7
+ if b < 0x80:
+ break
+ if m == OP_MATCH or m == OP_RES:
+ while True:
+ b = np.int64(ops[op])
+ op += 1
+ if b < 0x80:
+ break
+ if m == OP_MATCH:
+ nm += 1
+ else:
+ nb_other += L
+ elif m == OP_FILL:
+ op += 8
+ nb_other += L
+ else:
+ nb_other += L
+ return nm, nb_other
+
+
+@njit(cache=True)
+def expand(R, ops, res, intra, nm, nb_other):
+ dst = np.empty(nm, np.int64)
+ src = np.empty(nm, np.int64)
+ ln = np.empty(nm, np.int64)
+ other = np.empty(nb_other, np.int64)
+ data = np.empty(nb_other * B, np.uint8)
+ op, rp, ip, prev_d, i, km, ko = 0, 0, 0, 0, 0, 0, 0
+ while op < len(ops):
+ m = ops[op]
+ op += 1
+ L, s = 0, 0
+ while True:
+ b = np.int64(ops[op])
+ op += 1
+ L |= (b & 0x7F) << s
+ s += 7
+ if b < 0x80:
+ break
+ a = i * B
+ if m == OP_MATCH or m == OP_RES:
+ z, s = 0, 0
+ while True:
+ b = np.int64(ops[op])
+ op += 1
+ z |= (b & 0x7F) << s
+ s += 7
+ if b < 0x80:
+ break
+ d = prev_d + ((z >> 1) ^ -(z & 1))
+ prev_d = d
+ if m == OP_MATCH:
+ dst[km] = a
+ src[km] = a + d
+ ln[km] = L * B
+ km += 1
+ else:
+ for k in range(L * B):
+ data[ko * B + k] = R[a + d + k] ^ res[rp + k]
+ rp += L * B
+ for k in range(L):
+ other[ko + k] = i + k
+ ko += L
+ elif m == OP_FILL:
+ for k in range(L * B):
+ data[ko * B + k] = ops[op + (k & 7)]
+ op += 8
+ for k in range(L):
+ other[ko + k] = i + k
+ ko += L
+ else:
+ data[ko * B:(ko + L) * B] = intra[ip:ip + L * B]
+ ip += L * B
+ for k in range(L):
+ other[ko + k] = i + k
+ ko += L
+ i += L
+ return dst, src, ln, other, data
+
+
+def read_desc(path):
+ if path.endswith(".zst"):
+ raw = subprocess.run(["zstd", "-q", "-d", "-c", "--long=27", path], capture_output=True, check=True).stdout
+ return np.load(io.BytesIO(raw))
+ return np.load(path)
+
+
+def do_expand(desc, ref, out):
+ t0 = time.perf_counter()
+ d = read_desc(desc)
+ ops, res, intra, tail = d["ops"], d["res"], d["intra"], d["tail"]
+ t1 = time.perf_counter()
+ R = np.memmap(ref, dtype=np.uint8, mode="r")
+ nm, no = count(ops)
+ dst, src, ln, other, data = expand(R, ops, res, intra, nm, no)
+ t2 = time.perf_counter()
+ np.savez(out, dst=dst, src=src, len=ln, other=other, other_mode=np.zeros(len(other), np.uint8),
+ patch_data=data, tail=tail, v_bytes=d["v_bytes"], refs=np.array([ref]), target=d["target"])
+ return dict(desc_bytes=os.path.getsize(desc), decompress_s=t1 - t0, expand_s=t2 - t1, runs=int(nm),
+ other_blocks=int(no), patch_bytes=int(no * B))
+
+
+if __name__ == "__main__":
+ mode = sys.argv[1]
+ here = os.path.dirname(os.path.abspath(__file__))
+ if mode == "expand":
+ expand(np.zeros(64, np.uint8), np.zeros(0, np.uint8), np.zeros(0, np.uint8), np.zeros(0, np.uint8), 0, 0)
+ print(json.dumps(do_expand(sys.argv[2], sys.argv[3], sys.argv[4])))
+ else:
+ desc, ref, work, out = sys.argv[2:6]
+ cold = len(sys.argv) > 6 and sys.argv[6] == "1"
+ os.makedirs(work, exist_ok=True)
+ expand(np.zeros(64, np.uint8), np.zeros(0, np.uint8), np.zeros(0, np.uint8), np.zeros(0, np.uint8), 0, 0)
+ count(np.zeros(0, np.uint8))
+ runs = os.path.join(work, "apply.runs.npz")
+ if cold:
+ fd = os.open(ref, os.O_RDONLY)
+ os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED)
+ os.close(fd)
+ t0 = time.perf_counter()
+ e = do_expand(desc, ref, runs)
+ t1 = time.perf_counter()
+ p = subprocess.run([sys.executable, os.path.join(here, "host_restore.py"), mode, runs, work, out + ".restore"],
+ capture_output=True, text=True, check=True)
+ t2 = time.perf_counter()
+ r = json.load(open(out + ".restore"))
+ res = dict(mode=mode, cold=cold, expand=e, expand_total_s=t1 - t0, restore_s=r["restore_s"],
+ process_s=t2 - t1, apply_s=(t1 - t0) + r["restore_s"], correct=r["correct"],
+ bytes_written=r["bytes_written"], load=os.getloadavg())
+ json.dump(res, open(out, "w"), indent=1)
+ print(json.dumps(res))
]